hyperagents 1.2.3rc5.post3__py3-none-any.whl

civilization/sync/gdrive_sync.py

@@ -0,0 +1,262 @@
+"""
+Google Drive integration for syncing organizational documents.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+from uuid import UUID
+
+from ..schema import CivilizationNode, NodeType
+from ..ingestion.document_ingestor import DocumentIngestor
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class GDriveConfig:
+    """Configuration for Google Drive sync."""
+    credentials_path: str = ""
+    folder_ids: list[str] = field(default_factory=list)
+    folder_type_mapping: dict[str, NodeType] = field(default_factory=dict)
+    sync_interval_minutes: int = 120
+    file_types: list[str] = field(default_factory=lambda: [".md", ".txt", ".docx", ".pdf"])
+    max_file_size_mb: int = 10
+
+
+@dataclass
+class GDriveFile:
+    """Represents a Google Drive file."""
+    file_id: str
+    name: str
+    mime_type: str
+    folder_id: str
+    web_view_link: str
+    modified_time: datetime
+    size_bytes: int = 0
+    content: bytes | None = None
+
+
+@dataclass
+class GDriveSyncResult:
+    """Result of a Google Drive sync operation."""
+    success: bool
+    files_processed: int = 0
+    files_skipped: int = 0
+    nodes_created: int = 0
+    nodes_updated: int = 0
+    errors: list[str] = field(default_factory=list)
+    sync_time: datetime = field(default_factory=datetime.utcnow)
+
+
+class GDriveSync:
+    """
+    Synchronizes Google Drive documents with Civilization Knowledge Layer.
+    """
+
+    SUPPORTED_MIME_TYPES = {
+        "application/vnd.google-apps.document": "gdoc",
+        "text/plain": "txt",
+        "text/markdown": "md",
+        "application/pdf": "pdf",
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document": "docx",
+    }
+
+    def __init__(
+        self,
+        org_id: str,
+        config: GDriveConfig,
+        ingestor: DocumentIngestor | None = None,
+    ):
+        self.org_id = org_id
+        self.config = config
+        self.ingestor = ingestor
+        self._last_sync: datetime | None = None
+        self._file_cache: dict[str, datetime] = {}  # file_id -> last_modified
+
+    async def sync_folder(
+        self,
+        folder_id: str,
+        node_type: NodeType | None = None,
+        recursive: bool = True,
+    ) -> GDriveSyncResult:
+        """
+        Sync a Google Drive folder to civilization nodes.
+
+        Args:
+            folder_id: Google Drive folder ID
+            node_type: Optional node type to assign to all files
+            recursive: Whether to sync subfolders
+
+        Returns:
+            GDriveSyncResult with sync statistics
+        """
+        result = GDriveSyncResult(success=False)
+
+        try:
+            # Fetch files from folder
+            files = await self._list_folder_files(folder_id, recursive)
+
+            for file in files:
+                try:
+                    # Skip if not modified since last sync
+                    if self._should_skip_file(file):
+                        result.files_skipped += 1
+                        continue
+
+                    # Process file
+                    await self._process_file(file, node_type, result)
+                    result.files_processed += 1
+
+                except Exception as e:
+                    result.errors.append(f"Error processing file {file.name}: {e}")
+
+            result.success = True
+            self._last_sync = datetime.utcnow()
+
+        except Exception as e:
+            logger.exception("Google Drive sync failed")
+            result.errors.append(str(e))
+
+        return result
+
+    async def sync_all_folders(self) -> GDriveSyncResult:
+        """Sync all configured folders."""
+        total_result = GDriveSyncResult(success=True)
+
+        for folder_id in self.config.folder_ids:
+            node_type = self.config.folder_type_mapping.get(folder_id)
+            result = await self.sync_folder(folder_id, node_type)
+
+            total_result.files_processed += result.files_processed
+            total_result.files_skipped += result.files_skipped
+            total_result.nodes_created += result.nodes_created
+            total_result.nodes_updated += result.nodes_updated
+            total_result.errors.extend(result.errors)
+
+            if not result.success:
+                total_result.success = False
+
+        return total_result
+
+    async def sync_file(self, file_id: str) -> CivilizationNode | None:
+        """Sync a single file by ID."""
+        file = await self._get_file(file_id)
+        if not file:
+            return None
+
+        result = GDriveSyncResult(success=False)
+        await self._process_file(file, None, result)
+
+        return None  # Would return created/updated node
+
+    async def _list_folder_files(
+        self,
+        folder_id: str,
+        recursive: bool,
+    ) -> list[GDriveFile]:
+        """
+        List files in a Google Drive folder.
+        This is a stub - actual implementation would use Google Drive API.
+        """
+        # In real implementation:
+        # from google.oauth2 import service_account
+        # from googleapiclient.discovery import build
+        #
+        # creds = service_account.Credentials.from_service_account_file(
+        #     self.config.credentials_path
+        # )
+        # service = build('drive', 'v3', credentials=creds)
+        # results = service.files().list(
+        #     q=f"'{folder_id}' in parents",
+        #     pageSize=100,
+        #     fields="files(id, name, mimeType, modifiedTime, size, webViewLink)"
+        # ).execute()
+
+        logger.info(f"Would list files from Google Drive folder: {folder_id}")
+        return []
+
+    async def _get_file(self, file_id: str) -> GDriveFile | None:
+        """Fetch a single file with content."""
+        logger.info(f"Would fetch Google Drive file: {file_id}")
+        return None
+
+    async def _download_file_content(self, file: GDriveFile) -> bytes | None:
+        """Download file content."""
+        # Check size limit
+        max_size = self.config.max_file_size_mb * 1024 * 1024
+        if file.size_bytes > max_size:
+            logger.warning(f"File {file.name} exceeds size limit")
+            return None
+
+        # In real implementation, would download file content
+        logger.info(f"Would download content for file: {file.name}")
+        return None
+
+    def _should_skip_file(self, file: GDriveFile) -> bool:
+        """Check if file should be skipped (not modified)."""
+        # Check mime type
+        if file.mime_type not in self.SUPPORTED_MIME_TYPES:
+            return True
+
+        # Check extension
+        ext = "." + file.name.split(".")[-1].lower() if "." in file.name else ""
+        if ext and ext not in self.config.file_types:
+            return True
+
+        # Check if modified since last seen
+        last_seen = self._file_cache.get(file.file_id)
+        if last_seen and file.modified_time <= last_seen:
+            return True
+
+        return False
+
+    async def _process_file(
+        self,
+        file: GDriveFile,
+        node_type: NodeType | None,
+        result: GDriveSyncResult,
+    ) -> None:
+        """Process a Google Drive file and create/update civilization node."""
+        if not self.ingestor:
+            logger.warning("No ingestor configured")
+            return
+
+        # Download content
+        content = await self._download_file_content(file)
+        if not content:
+            return
+
+        # Ingest
+        from hyperclaw.taint import TaintedInput
+        try:
+            ingest_result = await self.ingestor.ingest(
+                content=content,
+                filename=file.name,
+                source="gdrive",
+                source_url=file.web_view_link,
+                node_type_hint=node_type,
+            )
+        except TaintedInput as e:  # connector content never enters the knowledge layer
+            result.errors.append(e.user_message)
+            return
+
+        if ingest_result.success:
+            result.nodes_created += 1
+            self._file_cache[file.file_id] = file.modified_time
+        else:
+            result.errors.extend(ingest_result.errors)
+
+    def get_sync_status(self) -> dict:
+        """Get current sync status."""
+        return {
+            "last_sync": self._last_sync.isoformat() if self._last_sync else None,
+            "configured_folders": self.config.folder_ids,
+            "sync_interval_minutes": self.config.sync_interval_minutes,
+            "cached_files": len(self._file_cache),
+        }
+
+    def clear_cache(self) -> None:
+        """Clear the file modification cache (forces full re-sync)."""
+        self._file_cache.clear()

civilization/sync/notion_sync.py

@@ -0,0 +1,237 @@
+"""
+Notion integration for syncing organizational knowledge.
+"""
+from __future__ import annotations
+import logging
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+from uuid import UUID, uuid4
+
+from ..schema import CivilizationNode, NodeType, NodeStatus
+from ..ingestion.document_ingestor import DocumentIngestor
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class NotionConfig:
+    """Configuration for Notion sync."""
+    api_key: str = ""
+    workspace_id: str | None = None
+    database_ids: dict[NodeType, str] = field(default_factory=dict)
+    sync_interval_minutes: int = 60
+    auto_create_nodes: bool = True
+    bidirectional: bool = False  # If True, also push changes back to Notion
+
+
+@dataclass
+class NotionPage:
+    """Represents a Notion page."""
+    page_id: str
+    title: str
+    url: str
+    content: str
+    properties: dict = field(default_factory=dict)
+    last_edited: datetime | None = None
+    parent_id: str | None = None
+
+
+@dataclass
+class SyncResult:
+    """Result of a sync operation."""
+    success: bool
+    pages_synced: int = 0
+    nodes_created: int = 0
+    nodes_updated: int = 0
+    errors: list[str] = field(default_factory=list)
+    sync_time: datetime = field(default_factory=datetime.utcnow)
+
+
+class NotionSync:
+    """
+    Synchronizes Notion workspace content with Civilization Knowledge Layer.
+    """
+
+    def __init__(
+        self,
+        org_id: str,
+        config: NotionConfig,
+        ingestor: DocumentIngestor | None = None,
+    ):
+        self.org_id = org_id
+        self.config = config
+        self.ingestor = ingestor
+        self._last_sync: datetime | None = None
+        self._sync_cursor: dict[str, str] = {}
+
+    async def sync_database(
+        self,
+        database_id: str,
+        node_type: NodeType | None = None,
+    ) -> SyncResult:
+        """
+        Sync a Notion database to civilization nodes.
+
+        Args:
+            database_id: Notion database ID
+            node_type: Optional node type to assign
+
+        Returns:
+            SyncResult with sync statistics
+        """
+        result = SyncResult(success=False)
+
+        try:
+            # Fetch pages from Notion
+            pages = await self._fetch_database_pages(database_id)
+            result.pages_synced = len(pages)
+
+            # Process each page
+            for page in pages:
+                try:
+                    await self._process_page(page, node_type, result)
+                except Exception as e:
+                    result.errors.append(f"Error processing page {page.page_id}: {e}")
+
+            result.success = True
+            self._last_sync = datetime.utcnow()
+
+        except Exception as e:
+            logger.exception("Notion sync failed")
+            result.errors.append(str(e))
+
+        return result
+
+    async def sync_all_databases(self) -> SyncResult:
+        """Sync all configured databases."""
+        total_result = SyncResult(success=True)
+
+        for node_type, database_id in self.config.database_ids.items():
+            result = await self.sync_database(database_id, node_type)
+            total_result.pages_synced += result.pages_synced
+            total_result.nodes_created += result.nodes_created
+            total_result.nodes_updated += result.nodes_updated
+            total_result.errors.extend(result.errors)
+            if not result.success:
+                total_result.success = False
+
+        return total_result
+
+    async def sync_page(self, page_id: str) -> CivilizationNode | None:
+        """Sync a single Notion page."""
+        page = await self._fetch_page(page_id)
+        if not page:
+            return None
+
+        result = SyncResult(success=False)
+        await self._process_page(page, None, result)
+
+        return None  # Would return the created/updated node
+
+    async def _fetch_database_pages(self, database_id: str) -> list[NotionPage]:
+        """
+        Fetch all pages from a Notion database.
+        This is a stub - actual implementation would use Notion API.
+        """
+        # In real implementation:
+        # async with httpx.AsyncClient() as client:
+        #     response = await client.post(
+        #         f"https://api.notion.com/v1/databases/{database_id}/query",
+        #         headers={
+        #             "Authorization": f"Bearer {self.config.api_key}",
+        #             "Notion-Version": "2022-06-28",
+        #         },
+        #         json={"page_size": 100},
+        #     )
+        #     ...
+
+        logger.info(f"Would fetch pages from Notion database: {database_id}")
+        return []
+
+    async def _fetch_page(self, page_id: str) -> NotionPage | None:
+        """
+        Fetch a single Notion page with content.
+        This is a stub - actual implementation would use Notion API.
+        """
+        logger.info(f"Would fetch Notion page: {page_id}")
+        return None
+
+    async def _process_page(
+        self,
+        page: NotionPage,
+        node_type: NodeType | None,
+        result: SyncResult,
+    ) -> None:
+        """Process a Notion page and create/update civilization node."""
+        if not self.ingestor:
+            logger.warning("No ingestor configured, skipping page processing")
+            return
+
+        # Determine node type from page properties or use provided type
+        detected_type = node_type or self._detect_node_type(page)
+
+        # Ingest the page content
+        from hyperclaw.taint import TaintedInput
+        try:
+            ingest_result = await self.ingestor.ingest(
+                content=page.content,
+                filename=f"{page.title}.md",
+                source="notion",
+                source_url=page.url,
+                node_type_hint=detected_type,
+            )
+        except TaintedInput as e:  # connector content never enters the knowledge layer
+            result.errors.append(e.user_message)
+            return
+
+        if ingest_result.success:
+            result.nodes_created += 1
+        else:
+            result.errors.extend(ingest_result.errors)
+
+    def _detect_node_type(self, page: NotionPage) -> NodeType:
+        """Detect node type from page properties."""
+        # Check for type property
+        type_prop = page.properties.get("Type", {}).get("select", {}).get("name", "")
+        type_map = {
+            "SOP": NodeType.SOP,
+            "Procedure": NodeType.SOP,
+            "Checklist": NodeType.CHECKLIST,
+            "Runbook": NodeType.RUNBOOK,
+            "Role": NodeType.ROLE,
+            "Job Description": NodeType.JOB_DESCRIPTION,
+            "Policy": NodeType.POLICY,
+        }
+        return type_map.get(type_prop, NodeType.KNOWLEDGE_ARTICLE)
+
+    async def push_to_notion(
+        self,
+        node: CivilizationNode,
+        database_id: str | None = None,
+    ) -> str | None:
+        """
+        Push a civilization node back to Notion.
+        Returns the Notion page ID if successful.
+        """
+        if not self.config.bidirectional:
+            logger.warning("Bidirectional sync not enabled")
+            return None
+
+        db_id = database_id or self.config.database_ids.get(node.node_type)
+        if not db_id:
+            logger.warning(f"No database configured for node type: {node.node_type}")
+            return None
+
+        # In real implementation, would create/update Notion page
+        logger.info(f"Would push node {node.id} to Notion database {db_id}")
+        return None
+
+    def get_sync_status(self) -> dict:
+        """Get current sync status."""
+        return {
+            "last_sync": self._last_sync.isoformat() if self._last_sync else None,
+            "configured_databases": list(self.config.database_ids.keys()),
+            "sync_interval_minutes": self.config.sync_interval_minutes,
+            "bidirectional": self.config.bidirectional,
+        }

civilization/versioning/__init__.py

@@ -0,0 +1,19 @@
+"""
+Versioning module for Civilization Knowledge Layer.
+Handles version tracking, diffs, and staleness detection.
+"""
+from .version_manager import VersionManager, NodeVersion, VersionHistory
+from .diff_engine import DiffEngine, NodeDiff, DiffType
+from .staleness_detector import StalenessDetector, StalenessLevel, StalenessReport
+
+__all__ = [
+    "VersionManager",
+    "NodeVersion",
+    "VersionHistory",
+    "DiffEngine",
+    "NodeDiff",
+    "DiffType",
+    "StalenessDetector",
+    "StalenessLevel",
+    "StalenessReport",
+]