hyperagents 1.2.3rc5.post3__py3-none-any.whl

civilization/store.py

@@ -0,0 +1,177 @@
+"""
+CivilizationStore — all DB operations for civilization nodes.
+Wraps asyncpg pool (or stub when pool=None for testing).
+"""
+from __future__ import annotations
+import json
+import logging
+from uuid import UUID
+from datetime import datetime
+from typing import Any
+
+from hyperclaw.taint import training_sink
+
+from .schema import CivilizationNode, NodeType, NodeStatus
+
+logger = logging.getLogger(__name__)
+
+
+class CivilizationStore:
+    def __init__(self, pool=None):
+        self.pool = pool  # asyncpg pool; None = no-DB mode (tests)
+
+    @training_sink("civilization.store.save")
+    async def save(self, node: CivilizationNode) -> CivilizationNode:
+        """Upsert a node into civilization_nodes."""
+        if not self.pool:
+            return node
+        content = node.model_dump(mode="json", exclude={"embedding"})
+        async with self.pool.acquire() as conn:
+            await conn.execute("""
+                INSERT INTO civilization_nodes
+                    (id, org_id, node_type, title, status, version, owner_id,
+                     tags, source, source_url, content, embedding, metadata,
+                     created_at, updated_at)
+                VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12::vector,$13,$14,$15)
+                ON CONFLICT (id) DO UPDATE SET
+                    title=EXCLUDED.title, status=EXCLUDED.status,
+                    version=EXCLUDED.version, content=EXCLUDED.content,
+                    embedding=EXCLUDED.embedding, updated_at=EXCLUDED.updated_at
+            """,
+            node.id, node.org_id, node.node_type.value, node.title,
+            node.status.value, node.version, node.owner_id,
+            node.tags, node.source, node.source_url,
+            json.dumps(content),
+            node.embedding,
+            json.dumps(node.metadata),
+            node.created_at, node.updated_at)
+        return node
+
+    async def get(self, node_id: UUID, org_id: str) -> CivilizationNode | None:
+        if not self.pool:
+            return None
+        async with self.pool.acquire() as conn:
+            row = await conn.fetchrow(
+                "SELECT * FROM civilization_nodes WHERE id=$1 AND org_id=$2",
+                node_id, org_id
+            )
+        if not row:
+            return None
+        return self._row_to_node(row)
+
+    async def list_by_type(self, org_id: str, node_type: NodeType,
+                           limit: int = 100) -> list[CivilizationNode]:
+        if not self.pool:
+            return []
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT * FROM civilization_nodes WHERE org_id=$1 AND node_type=$2 LIMIT $3",
+                org_id, node_type.value, limit
+            )
+        return [self._row_to_node(r) for r in rows]
+
+    async def list_all(self, org_id: str, limit: int = 500) -> list[CivilizationNode]:
+        if not self.pool:
+            return []
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT * FROM civilization_nodes WHERE org_id=$1 LIMIT $2",
+                org_id, limit
+            )
+        return [self._row_to_node(r) for r in rows]
+
+    async def search_by_embedding(self, org_id: str, embedding: list[float],
+                                   top_k: int = 10,
+                                   node_types: list[NodeType] | None = None) -> list[dict]:
+        """Vector similarity search."""
+        if not self.pool:
+            return []
+        type_filter = ""
+        args = [org_id, embedding, top_k]
+        if node_types:
+            placeholders = ",".join(f"${i+4}" for i in range(len(node_types)))
+            type_filter = f"AND node_type IN ({placeholders})"
+            args.extend([t.value for t in node_types])
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(f"""
+                SELECT *, 1 - (embedding <=> $2::vector) AS score
+                FROM civilization_nodes
+                WHERE org_id=$1 {type_filter}
+                ORDER BY embedding <=> $2::vector
+                LIMIT $3
+            """, *args)
+        return [dict(r) for r in rows]
+
+    async def search_by_tags(self, org_id: str, tags: list[str],
+                             limit: int = 50) -> list[CivilizationNode]:
+        """Find nodes that have any of the specified tags."""
+        if not self.pool:
+            return []
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT * FROM civilization_nodes WHERE org_id=$1 AND tags && $2 LIMIT $3",
+                org_id, tags, limit
+            )
+        return [self._row_to_node(r) for r in rows]
+
+    async def search_by_owner(self, org_id: str, owner_id: str,
+                              limit: int = 100) -> list[CivilizationNode]:
+        """Find all nodes owned by a specific person."""
+        if not self.pool:
+            return []
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT * FROM civilization_nodes WHERE org_id=$1 AND owner_id=$2 LIMIT $3",
+                org_id, owner_id, limit
+            )
+        return [self._row_to_node(r) for r in rows]
+
+    async def count_by_type(self, org_id: str) -> dict[str, int]:
+        if not self.pool:
+            return {}
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(
+                "SELECT node_type, COUNT(*) as cnt FROM civilization_nodes WHERE org_id=$1 GROUP BY node_type",
+                org_id
+            )
+        return {r["node_type"]: r["cnt"] for r in rows}
+
+    async def delete(self, node_id: UUID, org_id: str) -> bool:
+        if not self.pool:
+            return True
+        async with self.pool.acquire() as conn:
+            result = await conn.execute(
+                "DELETE FROM civilization_nodes WHERE id=$1 AND org_id=$2",
+                node_id, org_id
+            )
+        return result == "DELETE 1"
+
+    @training_sink("civilization.store.update_status")
+    async def update_status(self, node_id: UUID, org_id: str,
+                            new_status: NodeStatus) -> bool:
+        """Update a node's status."""
+        if not self.pool:
+            return True
+        async with self.pool.acquire() as conn:
+            result = await conn.execute(
+                "UPDATE civilization_nodes SET status=$3, updated_at=$4 WHERE id=$1 AND org_id=$2",
+                node_id, org_id, new_status.value, datetime.utcnow()
+            )
+        return "UPDATE 1" in result
+
+    @training_sink("civilization.store.update_embedding")
+    async def update_embedding(self, node_id: UUID, org_id: str,
+                               embedding: list[float]) -> bool:
+        """Update a node's embedding vector."""
+        if not self.pool:
+            return True
+        async with self.pool.acquire() as conn:
+            result = await conn.execute(
+                "UPDATE civilization_nodes SET embedding=$3::vector, updated_at=$4 WHERE id=$1 AND org_id=$2",
+                node_id, org_id, embedding, datetime.utcnow()
+            )
+        return "UPDATE 1" in result
+
+    def _row_to_node(self, row) -> CivilizationNode:
+        content = json.loads(row["content"]) if isinstance(row["content"], str) else row["content"]
+        return CivilizationNode(**{**content, "id": row["id"], "org_id": row["org_id"]})

civilization/sync/__init__.py

@@ -0,0 +1,16 @@
+"""
+Sync module for Civilization Knowledge Layer.
+Handles synchronization with external knowledge management systems.
+"""
+from .notion_sync import NotionSync, NotionConfig
+from .gdrive_sync import GDriveSync, GDriveConfig
+from .confluence_sync import ConfluenceSync, ConfluenceConfig
+
+__all__ = [
+    "NotionSync",
+    "NotionConfig",
+    "GDriveSync",
+    "GDriveConfig",
+    "ConfluenceSync",
+    "ConfluenceConfig",
+]

civilization/sync/confluence_sync.py

@@ -0,0 +1,317 @@
+"""
+Confluence integration for syncing organizational knowledge.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from ..schema import CivilizationNode, NodeType
+from ..ingestion.document_ingestor import DocumentIngestor
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ConfluenceConfig:
+    """Configuration for Confluence sync."""
+    base_url: str = ""  # e.g., "https://your-domain.atlassian.net/wiki"
+    username: str = ""
+    api_token: str = ""
+    space_keys: list[str] = field(default_factory=list)
+    space_type_mapping: dict[str, NodeType] = field(default_factory=dict)
+    sync_interval_minutes: int = 120
+    include_attachments: bool = False
+    label_filters: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ConfluencePage:
+    """Represents a Confluence page."""
+    page_id: str
+    title: str
+    space_key: str
+    web_url: str
+    body_html: str
+    body_storage: str = ""  # Confluence storage format
+    labels: list[str] = field(default_factory=list)
+    version: int = 1
+    last_modified: datetime | None = None
+    parent_id: str | None = None
+    ancestors: list[str] = field(default_factory=list)
+
+
+@dataclass
+class ConfluenceSyncResult:
+    """Result of a Confluence sync operation."""
+    success: bool
+    pages_synced: int = 0
+    pages_skipped: int = 0
+    nodes_created: int = 0
+    nodes_updated: int = 0
+    errors: list[str] = field(default_factory=list)
+    sync_time: datetime = field(default_factory=datetime.utcnow)
+
+
+class ConfluenceSync:
+    """
+    Synchronizes Confluence spaces/pages with Civilization Knowledge Layer.
+    """
+
+    LABEL_TYPE_MAPPING = {
+        "sop": NodeType.SOP,
+        "procedure": NodeType.SOP,
+        "checklist": NodeType.CHECKLIST,
+        "runbook": NodeType.RUNBOOK,
+        "playbook": NodeType.RUNBOOK,
+        "role": NodeType.ROLE,
+        "job-description": NodeType.JOB_DESCRIPTION,
+        "policy": NodeType.POLICY,
+        "workflow": NodeType.WORKFLOW,
+    }
+
+    def __init__(
+        self,
+        org_id: str,
+        config: ConfluenceConfig,
+        ingestor: DocumentIngestor | None = None,
+    ):
+        self.org_id = org_id
+        self.config = config
+        self.ingestor = ingestor
+        self._last_sync: datetime | None = None
+        self._page_cache: dict[str, int] = {}  # page_id -> version
+
+    async def sync_space(
+        self,
+        space_key: str,
+        node_type: NodeType | None = None,
+    ) -> ConfluenceSyncResult:
+        """
+        Sync a Confluence space to civilization nodes.
+
+        Args:
+            space_key: Confluence space key
+            node_type: Optional default node type
+
+        Returns:
+            ConfluenceSyncResult with sync statistics
+        """
+        result = ConfluenceSyncResult(success=False)
+
+        try:
+            # Fetch pages from space
+            pages = await self._list_space_pages(space_key)
+
+            for page in pages:
+                try:
+                    # Skip if not modified
+                    if self._should_skip_page(page):
+                        result.pages_skipped += 1
+                        continue
+
+                    # Apply label filters
+                    if self.config.label_filters:
+                        if not any(l in page.labels for l in self.config.label_filters):
+                            result.pages_skipped += 1
+                            continue
+
+                    # Process page
+                    await self._process_page(page, node_type, result)
+                    result.pages_synced += 1
+
+                except Exception as e:
+                    result.errors.append(f"Error processing page {page.title}: {e}")
+
+            result.success = True
+            self._last_sync = datetime.utcnow()
+
+        except Exception as e:
+            logger.exception("Confluence sync failed")
+            result.errors.append(str(e))
+
+        return result
+
+    async def sync_all_spaces(self) -> ConfluenceSyncResult:
+        """Sync all configured spaces."""
+        total_result = ConfluenceSyncResult(success=True)
+
+        for space_key in self.config.space_keys:
+            node_type = self.config.space_type_mapping.get(space_key)
+            result = await self.sync_space(space_key, node_type)
+
+            total_result.pages_synced += result.pages_synced
+            total_result.pages_skipped += result.pages_skipped
+            total_result.nodes_created += result.nodes_created
+            total_result.nodes_updated += result.nodes_updated
+            total_result.errors.extend(result.errors)
+
+            if not result.success:
+                total_result.success = False
+
+        return total_result
+
+    async def sync_page(self, page_id: str) -> CivilizationNode | None:
+        """Sync a single page by ID."""
+        page = await self._get_page(page_id)
+        if not page:
+            return None
+
+        result = ConfluenceSyncResult(success=False)
+        await self._process_page(page, None, result)
+
+        return None
+
+    async def sync_by_label(
+        self,
+        label: str,
+        node_type: NodeType | None = None,
+    ) -> ConfluenceSyncResult:
+        """Sync all pages with a specific label."""
+        result = ConfluenceSyncResult(success=False)
+
+        try:
+            pages = await self._search_by_label(label)
+
+            for page in pages:
+                try:
+                    if self._should_skip_page(page):
+                        result.pages_skipped += 1
+                        continue
+
+                    await self._process_page(page, node_type, result)
+                    result.pages_synced += 1
+
+                except Exception as e:
+                    result.errors.append(f"Error processing page {page.title}: {e}")
+
+            result.success = True
+
+        except Exception as e:
+            logger.exception("Confluence label sync failed")
+            result.errors.append(str(e))
+
+        return result
+
+    async def _list_space_pages(self, space_key: str) -> list[ConfluencePage]:
+        """
+        List all pages in a Confluence space.
+        This is a stub - actual implementation would use Confluence REST API.
+        """
+        # In real implementation:
+        # import httpx
+        # import base64
+        #
+        # auth = base64.b64encode(
+        #     f"{self.config.username}:{self.config.api_token}".encode()
+        # ).decode()
+        #
+        # async with httpx.AsyncClient() as client:
+        #     response = await client.get(
+        #         f"{self.config.base_url}/rest/api/content",
+        #         params={"spaceKey": space_key, "expand": "body.storage,version"},
+        #         headers={"Authorization": f"Basic {auth}"},
+        #     )
+
+        logger.info(f"Would list pages from Confluence space: {space_key}")
+        return []
+
+    async def _get_page(self, page_id: str) -> ConfluencePage | None:
+        """Fetch a single Confluence page with content."""
+        logger.info(f"Would fetch Confluence page: {page_id}")
+        return None
+
+    async def _search_by_label(self, label: str) -> list[ConfluencePage]:
+        """Search for pages with a specific label."""
+        logger.info(f"Would search Confluence for label: {label}")
+        return []
+
+    def _should_skip_page(self, page: ConfluencePage) -> bool:
+        """Check if page should be skipped (not modified)."""
+        cached_version = self._page_cache.get(page.page_id)
+        if cached_version and page.version <= cached_version:
+            return True
+        return False
+
+    async def _process_page(
+        self,
+        page: ConfluencePage,
+        node_type: NodeType | None,
+        result: ConfluenceSyncResult,
+    ) -> None:
+        """Process a Confluence page and create/update civilization node."""
+        if not self.ingestor:
+            logger.warning("No ingestor configured")
+            return
+
+        # Detect node type from labels if not specified
+        detected_type = node_type or self._detect_node_type(page)
+
+        # Convert HTML to markdown-ish text
+        content = self._convert_confluence_content(page)
+
+        # Ingest
+        from hyperclaw.taint import TaintedInput
+        try:
+            ingest_result = await self.ingestor.ingest(
+                content=content,
+                filename=f"{page.title}.md",
+                source="confluence",
+                source_url=page.web_url,
+                node_type_hint=detected_type,
+                tags=page.labels,
+            )
+        except TaintedInput as e:  # connector content never enters the knowledge layer
+            result.errors.append(e.user_message)
+            return
+
+        if ingest_result.success:
+            result.nodes_created += 1
+            self._page_cache[page.page_id] = page.version
+        else:
+            result.errors.extend(ingest_result.errors)
+
+    def _detect_node_type(self, page: ConfluencePage) -> NodeType:
+        """Detect node type from page labels."""
+        for label in page.labels:
+            label_lower = label.lower()
+            if label_lower in self.LABEL_TYPE_MAPPING:
+                return self.LABEL_TYPE_MAPPING[label_lower]
+        return NodeType.KNOWLEDGE_ARTICLE
+
+    def _convert_confluence_content(self, page: ConfluencePage) -> str:
+        """Convert Confluence storage format to plain text/markdown."""
+        import re
+
+        content = page.body_storage or page.body_html
+
+        # Simple HTML tag stripping (in production, use a proper HTML parser)
+        content = re.sub(r"<br\s*/?>", "\n", content)
+        content = re.sub(r"<p[^>]*>", "\n", content)
+        content = re.sub(r"</p>", "\n", content)
+        content = re.sub(r"<li[^>]*>", "- ", content)
+        content = re.sub(r"<h(\d)[^>]*>", r"\n#" + r"\1 ", content)
+        content = re.sub(r"</h\d>", "\n", content)
+        content = re.sub(r"<[^>]+>", "", content)
+
+        # Clean up whitespace
+        content = re.sub(r"\n{3,}", "\n\n", content)
+
+        # Add title
+        return f"# {page.title}\n\n{content.strip()}"
+
+    def get_sync_status(self) -> dict:
+        """Get current sync status."""
+        return {
+            "last_sync": self._last_sync.isoformat() if self._last_sync else None,
+            "configured_spaces": self.config.space_keys,
+            "sync_interval_minutes": self.config.sync_interval_minutes,
+            "cached_pages": len(self._page_cache),
+            "label_filters": self.config.label_filters,
+        }
+
+    def clear_cache(self) -> None:
+        """Clear page version cache (forces full re-sync)."""
+        self._page_cache.clear()