cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/confluence/__init__.py

@@ -0,0 +1,359 @@
+"""
+Confluence connector for CortexDB.
+
+Polls Confluence spaces for recently updated pages and ingests them as
+CortexDB Episodes.  Uses the ``atlassian-python-api`` library for API access.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="confluence")
+
+_PAGE_LIMIT = 50
+
+
+class ConfluenceConnector(CortexConnector):
+    """Connector that syncs Confluence pages into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    confluence_url:
+        Base URL of the Confluence instance (e.g.
+        ``https://myteam.atlassian.net/wiki``).
+    email:
+        Email address for Confluence authentication.
+    api_token:
+        Atlassian API token.
+    space_keys:
+        List of Confluence space keys to sync (e.g. ``["ENG", "OPS"]``).
+        If empty, all spaces visible to the authenticated user are synced.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "confluence"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        confluence_url: str,
+        email: str,
+        api_token: str,
+        space_keys: list[str] | None = None,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.confluence_url = confluence_url.rstrip("/")
+        self.email = email
+        self.api_token = api_token
+        self.space_keys = space_keys or []
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch recently updated pages from configured Confluence spaces.
+
+        Uses the CQL ``lastModified`` filter to retrieve incremental updates.
+        """
+        from atlassian import Confluence
+
+        client = Confluence(
+            url=self.confluence_url,
+            username=self.email,
+            password=self.api_token,
+            cloud=True,
+        )
+
+        spaces = self.space_keys or self._discover_spaces(client)
+        raw_events: list[RawEvent] = []
+
+        for space_key in spaces:
+            cql = f'space = "{space_key}" AND type = "page"'
+            if since:
+                cql_date = since.strftime("%Y-%m-%d %H:%M")
+                cql += f' AND lastModified >= "{cql_date}"'
+            cql += " ORDER BY lastModified DESC"
+
+            start = 0
+            while True:
+                try:
+                    results = client.cql(
+                        cql,
+                        start=start,
+                        limit=_PAGE_LIMIT,
+                        expand="content.body.storage,content.version,"
+                        "content.history.lastUpdated,"
+                        "content.metadata.labels,"
+                        "content.ancestors,"
+                        "content.restrictions.read.restrictions.user,"
+                        "content.restrictions.read.restrictions.group",
+                    )
+                except Exception:
+                    logger.exception(
+                        "CQL query failed for space %s", space_key
+                    )
+                    break
+
+                search_results = results.get("results", [])
+                if not search_results:
+                    break
+
+                for result in search_results:
+                    content_obj = result.get("content", result)
+                    page_data = self._extract_page_data(
+                        content_obj, space_key
+                    )
+                    updated_str = page_data.get("last_modified", "")
+                    try:
+                        updated_at = datetime.fromisoformat(
+                            updated_str.replace("Z", "+00:00")
+                        )
+                    except (ValueError, TypeError):
+                        updated_at = datetime.now(timezone.utc)
+
+                    restrictions = self._extract_restrictions(content_obj)
+
+                    raw_events.append(
+                        RawEvent(
+                            source="confluence",
+                            external_id=page_data.get("page_id", ""),
+                            timestamp=updated_at,
+                            data=page_data,
+                            permissions={
+                                "space_key": space_key,
+                                "restrictions": restrictions,
+                            },
+                        )
+                    )
+
+                total = results.get("totalSize", 0)
+                start += _PAGE_LIMIT
+                if start >= total:
+                    break
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a Confluence page ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        title = data.get("title", "Untitled")
+        excerpt = data.get("excerpt", "")
+        body_text = data.get("body_text", "")
+        space_key = data.get("space_key", "")
+
+        actor = self._extract_actor(data)
+        entities = self._extract_entities(data)
+
+        content = title
+        if excerpt:
+            content += f"\n\n{excerpt}"
+        elif body_text:
+            content += f"\n\n{body_text[:1000]}"
+
+        labels = data.get("labels", [])
+        tags = ["confluence", space_key.lower()] + [
+            lbl.get("name", lbl) if isinstance(lbl, dict) else str(lbl)
+            for lbl in labels
+        ]
+
+        ancestors = data.get("ancestors", [])
+        parent_id = None
+        if ancestors:
+            parent_id = f"confluence:{ancestors[-1].get('id', '')}"
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=content,
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace="confluence",
+            entities=entities,
+            tags=tags,
+            metadata={
+                "page_id": data.get("page_id", ""),
+                "space_key": space_key,
+                "version": data.get("version", 1),
+                "url": data.get("url", ""),
+            },
+            thread_id=f"confluence:{data.get('page_id', '')}",
+            parent_id=parent_id,
+            idempotency_key=(
+                f"{raw.external_id}:v{data.get('version', 1)}"
+            ),
+        )
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Confluence page/space restrictions to CortexDB visibility.
+
+        Pages with explicit read restrictions are marked ``Restricted``
+        with the allowed users/groups as principals.  Unrestricted pages
+        are ``Organization``.
+        """
+        restrictions = raw.permissions.get("restrictions", {})
+        users = restrictions.get("users", [])
+        groups = restrictions.get("groups", [])
+
+        if users or groups:
+            principals = tuple(users + groups)
+            return Visibility(
+                level=VisibilityLevel.RESTRICTED,
+                allowed_principals=principals,
+            )
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- helpers -------------------------------------------------------------
+
+    @staticmethod
+    def _discover_spaces(client: Any) -> list[str]:
+        """Auto-discover all spaces visible to the authenticated user."""
+        space_keys: list[str] = []
+        start = 0
+        while True:
+            results = client.get_all_spaces(start=start, limit=50)
+            spaces = results.get("results", [])
+            if not spaces:
+                break
+            for space in spaces:
+                space_keys.append(space["key"])
+            if len(spaces) < 50:
+                break
+            start += 50
+        return space_keys
+
+    @staticmethod
+    def _extract_page_data(
+        content_obj: dict[str, Any], space_key: str
+    ) -> dict[str, Any]:
+        """Flatten a Confluence content object into a plain dict."""
+        history = content_obj.get("history", {})
+        last_updated = history.get("lastUpdated", {})
+        version_info = content_obj.get("version", {})
+        body = content_obj.get("body", {})
+        storage = body.get("storage", {})
+        labels_data = content_obj.get("metadata", {}).get("labels", {})
+        labels = labels_data.get("results", []) if isinstance(labels_data, dict) else []
+        ancestors = content_obj.get("ancestors", [])
+        links = content_obj.get("_links", {})
+
+        modifier = last_updated.get("by", {})
+
+        return {
+            "page_id": content_obj.get("id", ""),
+            "title": content_obj.get("title", ""),
+            "space_key": space_key,
+            "body_text": storage.get("value", ""),
+            "excerpt": content_obj.get("excerpt", ""),
+            "version": version_info.get("number", 1) if isinstance(version_info, dict) else 1,
+            "last_modified": last_updated.get("when", ""),
+            "modifier": {
+                "account_id": modifier.get("accountId", ""),
+                "display_name": modifier.get("displayName", ""),
+                "email": modifier.get("email"),
+            },
+            "labels": labels,
+            "ancestors": [
+                {"id": a.get("id", ""), "title": a.get("title", "")}
+                for a in ancestors
+            ],
+            "url": links.get("webui", ""),
+        }
+
+    @staticmethod
+    def _extract_restrictions(content_obj: dict[str, Any]) -> dict[str, list[str]]:
+        """Extract read restrictions from a content object."""
+        restrictions_data = content_obj.get("restrictions", {})
+        read_restrictions = restrictions_data.get("read", {}).get(
+            "restrictions", {}
+        )
+
+        users: list[str] = []
+        groups: list[str] = []
+
+        for user in read_restrictions.get("user", {}).get("results", []):
+            account_id = user.get("accountId", "")
+            if account_id:
+                users.append(account_id)
+
+        for group in read_restrictions.get("group", {}).get("results", []):
+            group_name = group.get("name", "")
+            if group_name:
+                groups.append(group_name)
+
+        return {"users": users, "groups": groups}
+
+    @staticmethod
+    def _extract_actor(data: dict[str, Any]) -> Actor:
+        """Extract the actor (last modifier) from page data."""
+        modifier = data.get("modifier", {})
+        if modifier and modifier.get("account_id"):
+            return Actor(
+                id=modifier["account_id"],
+                name=modifier.get("display_name", "Unknown"),
+                email=modifier.get("email"),
+            )
+        return Actor(id="unknown", name="Unknown")
+
+    @staticmethod
+    def _extract_entities(data: dict[str, Any]) -> list[EntityRef]:
+        """Extract entity references from page data."""
+        entities: list[EntityRef] = []
+
+        space_key = data.get("space_key", "")
+        if space_key:
+            entities.append(
+                EntityRef(
+                    entity_type="space",
+                    entity_id=space_key,
+                    display_name=space_key,
+                )
+            )
+
+        page_id = data.get("page_id", "")
+        if page_id:
+            entities.append(
+                EntityRef(
+                    entity_type="page",
+                    entity_id=page_id,
+                    display_name=data.get("title", ""),
+                )
+            )
+
+        modifier = data.get("modifier", {})
+        if modifier and modifier.get("account_id"):
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=modifier["account_id"],
+                    display_name=modifier.get("display_name", ""),
+                )
+            )
+
+        return entities