cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/worker.py

@@ -0,0 +1,192 @@
+"""Background sync worker that periodically runs all configured connectors."""
+
+import asyncio
+import logging
+import os
+import signal
+import sys
+from datetime import datetime, timezone
+
+logger = logging.getLogger("cortexdb.connectors.worker")
+
+
+async def run_sync_cycle(cortex_url: str, api_key: str, tenant_id: str) -> None:
+    """Run one sync cycle for all configured connectors."""
+    connectors = []
+
+    # Slack
+    slack_token = os.environ.get("SLACK_BOT_TOKEN")
+    if slack_token:
+        try:
+            from cortexdb_connectors.slack import SlackConnector
+
+            channels = os.environ.get("SLACK_CHANNELS", "").split(",")
+            channels = [c.strip() for c in channels if c.strip()]
+            connector = SlackConnector(
+                cortex_url=cortex_url,
+                cortex_api_key=api_key,
+                tenant_id=tenant_id,
+                slack_bot_token=slack_token,
+                channels=channels or None,
+            )
+            connectors.append(("slack", connector))
+        except ImportError:
+            logger.warning("slack-sdk not installed, skipping Slack connector")
+
+    # GitHub
+    github_token = os.environ.get("GITHUB_TOKEN")
+    if github_token:
+        try:
+            from cortexdb_connectors.github import GitHubConnector
+
+            repos = os.environ.get("GITHUB_REPOS", "").split(",")
+            repos = [r.strip() for r in repos if r.strip()]
+            connector = GitHubConnector(
+                cortex_url=cortex_url,
+                cortex_api_key=api_key,
+                tenant_id=tenant_id,
+                github_token=github_token,
+                repos=repos,
+            )
+            connectors.append(("github", connector))
+        except ImportError:
+            logger.warning("pygithub not installed, skipping GitHub connector")
+
+    # PagerDuty
+    pd_key = os.environ.get("PAGERDUTY_API_KEY")
+    if pd_key:
+        try:
+            from cortexdb_connectors.pagerduty import PagerDutyConnector
+
+            service_ids = os.environ.get("PAGERDUTY_SERVICE_IDS", "").split(",")
+            service_ids = [s.strip() for s in service_ids if s.strip()]
+            connector = PagerDutyConnector(
+                cortex_url=cortex_url,
+                cortex_api_key=api_key,
+                tenant_id=tenant_id,
+                pagerduty_api_key=pd_key,
+                service_ids=service_ids or None,
+            )
+            connectors.append(("pagerduty", connector))
+        except ImportError:
+            logger.warning("pdpyras not installed, skipping PagerDuty connector")
+
+    # Jira
+    jira_url = os.environ.get("JIRA_URL")
+    jira_email = os.environ.get("JIRA_EMAIL")
+    jira_token = os.environ.get("JIRA_API_TOKEN")
+    if jira_url and jira_email and jira_token:
+        try:
+            from cortexdb_connectors.jira import JiraConnector
+
+            project_keys = os.environ.get("JIRA_PROJECT_KEYS", "").split(",")
+            project_keys = [k.strip() for k in project_keys if k.strip()]
+            connector = JiraConnector(
+                cortex_url=cortex_url,
+                cortex_api_key=api_key,
+                tenant_id=tenant_id,
+                jira_url=jira_url,
+                jira_email=jira_email,
+                jira_api_token=jira_token,
+                project_keys=project_keys,
+            )
+            connectors.append(("jira", connector))
+        except ImportError:
+            logger.warning("jira not installed, skipping Jira connector")
+
+    # Confluence
+    confluence_url = os.environ.get("CONFLUENCE_URL")
+    confluence_email = os.environ.get("CONFLUENCE_EMAIL")
+    confluence_token = os.environ.get("CONFLUENCE_API_TOKEN")
+    if confluence_url and confluence_email and confluence_token:
+        try:
+            from cortexdb_connectors.confluence import ConfluenceConnector
+
+            space_keys = os.environ.get("CONFLUENCE_SPACE_KEYS", "").split(",")
+            space_keys = [k.strip() for k in space_keys if k.strip()]
+            connector = ConfluenceConnector(
+                cortex_url=cortex_url,
+                cortex_api_key=api_key,
+                tenant_id=tenant_id,
+                confluence_url=confluence_url,
+                email=confluence_email,
+                api_token=confluence_token,
+                space_keys=space_keys,
+            )
+            connectors.append(("confluence", connector))
+        except ImportError:
+            logger.warning(
+                "atlassian-python-api not installed, skipping Confluence connector"
+            )
+
+    if not connectors:
+        logger.info("No connectors configured, skipping sync cycle")
+        return
+
+    for name, connector in connectors:
+        try:
+            result = await connector.sync()
+            logger.info(
+                "Connector %s: fetched=%d ingested=%d errors=%d duration=%.1fs",
+                name,
+                result.episodes_fetched,
+                result.episodes_ingested,
+                result.errors,
+                result.duration_seconds,
+            )
+        except Exception:
+            logger.exception("Connector %s failed", name)
+
+
+async def main_loop() -> None:
+    """Main worker loop."""
+    cortex_url = os.environ.get("CORTEXDB_URL", "http://localhost:3141")
+    api_key = os.environ.get("CORTEXDB_API_KEY", "cx_live_default_key")
+    tenant_id = os.environ.get("CORTEXDB_TENANT_ID", "default")
+    interval = int(os.environ.get("SYNC_INTERVAL", "300"))
+
+    logger.info(
+        "Starting connector worker: url=%s tenant=%s interval=%ds",
+        cortex_url,
+        tenant_id,
+        interval,
+    )
+
+    shutdown = asyncio.Event()
+
+    def handle_signal() -> None:
+        logger.info("Received shutdown signal")
+        shutdown.set()
+
+    loop = asyncio.get_running_loop()
+    for sig in (signal.SIGINT, signal.SIGTERM):
+        try:
+            loop.add_signal_handler(sig, handle_signal)
+        except NotImplementedError:
+            pass  # Windows
+
+    while not shutdown.is_set():
+        try:
+            await run_sync_cycle(cortex_url, api_key, tenant_id)
+        except Exception:
+            logger.exception("Sync cycle failed")
+
+        try:
+            await asyncio.wait_for(shutdown.wait(), timeout=interval)
+        except asyncio.TimeoutError:
+            pass
+
+    logger.info("Worker shut down cleanly")
+
+
+def main() -> None:
+    """Entry point."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(name)s %(levelname)s %(message)s",
+    )
+    asyncio.run(main_loop())
+
+
+if __name__ == "__main__":
+    main()

cortexdb_connectors/zendesk/__init__.py

@@ -0,0 +1,496 @@
+"""
+Zendesk connector for CortexDB.
+
+Polls Zendesk Support API v2 for Tickets, Comments, and Help Center
+Articles, converting them into CortexDB Episodes.  Uses ``httpx`` for
+direct HTTP access with cursor-based pagination.
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import httpx
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="zendesk")
+
+_RATE_LIMIT_RETRY_HEADER = "Retry-After"
+_MAX_RETRIES = 3
+
+
+class ZendeskConnector(CortexConnector):
+    """Connector that syncs Zendesk tickets, comments, and articles into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    subdomain:
+        Zendesk subdomain (e.g. ``mycompany`` for mycompany.zendesk.com).
+    email:
+        Zendesk agent email for API authentication.
+    api_token:
+        Zendesk API token.
+    include_help_center:
+        Whether to ingest Help Center articles.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    backfill_days:
+        Number of days of historical data to backfill on first sync.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "zendesk"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        subdomain: str,
+        email: str,
+        api_token: str,
+        include_help_center: bool = False,
+        namespace: str = "zendesk",
+        backfill_days: int = 90,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.subdomain = subdomain
+        self.email = email
+        self.api_token = api_token
+        self.include_help_center = include_help_center
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+        self._base_url = f"https://{subdomain}.zendesk.com"
+
+    # -- HTTP helpers --------------------------------------------------------
+
+    def _auth(self) -> tuple[str, str]:
+        """Return HTTP basic auth credentials (email/token)."""
+        return f"{self.email}/token", self.api_token
+
+    async def _request(
+        self,
+        client: httpx.AsyncClient,
+        method: str,
+        url: str,
+        **kwargs: Any,
+    ) -> httpx.Response:
+        """Execute an HTTP request with rate-limit retry."""
+        for attempt in range(_MAX_RETRIES):
+            resp = await client.request(method, url, **kwargs)
+            if resp.status_code == 429:
+                retry_after = int(
+                    resp.headers.get(_RATE_LIMIT_RETRY_HEADER, "5")
+                )
+                logger.warning(
+                    "Zendesk rate limited, retrying in %ds (attempt %d/%d)",
+                    retry_after,
+                    attempt + 1,
+                    _MAX_RETRIES,
+                )
+                import asyncio
+
+                await asyncio.sleep(retry_after)
+                continue
+            resp.raise_for_status()
+            return resp
+        raise httpx.HTTPStatusError(
+            "Rate limit retries exhausted",
+            request=resp.request,  # type: ignore[possibly-undefined]
+            response=resp,  # type: ignore[possibly-undefined]
+        )
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch tickets, comments, and optionally articles from Zendesk.
+
+        Uses cursor-based pagination via the incremental exports API for
+        tickets and standard pagination for comments and articles.
+        """
+        cutoff = since or datetime.now(timezone.utc) - timedelta(
+            days=self.backfill_days
+        )
+        raw_events: list[RawEvent] = []
+
+        async with httpx.AsyncClient(
+            auth=self._auth(), timeout=30.0
+        ) as client:
+            raw_events.extend(await self._fetch_tickets(client, cutoff))
+            if self.include_help_center:
+                raw_events.extend(await self._fetch_articles(client, cutoff))
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a Zendesk ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        event_kind = data.get("event_kind", "ticket")
+
+        if event_kind == "ticket":
+            return self._normalize_ticket(raw)
+        if event_kind == "comment":
+            return self._normalize_comment(raw)
+        if event_kind == "article":
+            return self._normalize_article(raw)
+        return self._normalize_ticket(raw)
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Zendesk visibility to CortexDB visibility.
+
+        Internal notes are ``Restricted``; public comments and articles
+        are ``Organization``; everything else is ``Organization``.
+        """
+        data = raw.data
+        if data.get("is_public") is False:
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- fetchers ------------------------------------------------------------
+
+    async def _fetch_tickets(
+        self,
+        client: httpx.AsyncClient,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch tickets and their comments using incremental export."""
+        events: list[RawEvent] = []
+        cutoff_unix = int(cutoff.timestamp())
+
+        url = (
+            f"{self._base_url}/api/v2/incremental/tickets/cursor.json"
+            f"?start_time={cutoff_unix}"
+        )
+
+        while url:
+            try:
+                resp = await self._request(client, "GET", url)
+            except Exception:
+                logger.exception("Failed to fetch Zendesk tickets")
+                break
+
+            body = resp.json()
+            tickets = body.get("tickets", [])
+
+            for ticket in tickets:
+                ts = self._parse_ts(ticket.get("updated_at"))
+                events.append(
+                    RawEvent(
+                        source="zendesk",
+                        external_id=str(ticket.get("id", "")),
+                        timestamp=ts,
+                        data={
+                            "event_kind": "ticket",
+                            "ticket": ticket,
+                            "is_public": True,
+                        },
+                        permissions={
+                            "requester_id": ticket.get("requester_id"),
+                            "organization_id": ticket.get("organization_id"),
+                        },
+                    )
+                )
+
+                # Fetch comments for this ticket.
+                events.extend(
+                    await self._fetch_comments(
+                        client, ticket.get("id", ""), ticket
+                    )
+                )
+
+            # Cursor-based pagination.
+            if body.get("end_of_stream"):
+                break
+            url = body.get("after_url", "")
+
+        return events
+
+    async def _fetch_comments(
+        self,
+        client: httpx.AsyncClient,
+        ticket_id: str | int,
+        ticket: dict[str, Any],
+    ) -> list[RawEvent]:
+        """Fetch all comments for a single ticket."""
+        events: list[RawEvent] = []
+        url: str | None = (
+            f"{self._base_url}/api/v2/tickets/{ticket_id}/comments.json"
+        )
+
+        while url:
+            try:
+                resp = await self._request(client, "GET", url)
+            except Exception:
+                logger.debug("Could not fetch comments for ticket %s", ticket_id)
+                break
+
+            body = resp.json()
+            for comment in body.get("comments", []):
+                ts = self._parse_ts(comment.get("created_at"))
+                events.append(
+                    RawEvent(
+                        source="zendesk",
+                        external_id=f"comment:{comment.get('id', '')}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "comment",
+                            "comment": comment,
+                            "ticket_id": str(ticket_id),
+                            "ticket_subject": ticket.get("subject", ""),
+                            "is_public": comment.get("public", True),
+                        },
+                        permissions={
+                            "author_id": comment.get("author_id"),
+                            "organization_id": ticket.get("organization_id"),
+                        },
+                    )
+                )
+
+            url = body.get("next_page")
+
+        return events
+
+    async def _fetch_articles(
+        self,
+        client: httpx.AsyncClient,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch Help Center articles updated since cutoff."""
+        events: list[RawEvent] = []
+        cutoff_str = cutoff.strftime("%Y-%m-%dT%H:%M:%SZ")
+        url: str | None = (
+            f"{self._base_url}/api/v2/help_center/articles.json"
+            f"?sort_by=updated_at&sort_order=desc&start_time={cutoff_str}"
+        )
+
+        while url:
+            try:
+                resp = await self._request(client, "GET", url)
+            except Exception:
+                logger.exception("Failed to fetch Zendesk articles")
+                break
+
+            body = resp.json()
+            for article in body.get("articles", []):
+                ts = self._parse_ts(article.get("updated_at"))
+                events.append(
+                    RawEvent(
+                        source="zendesk",
+                        external_id=f"article:{article.get('id', '')}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "article",
+                            "article": article,
+                            "is_public": not article.get("draft", False),
+                        },
+                        permissions={
+                            "author_id": article.get("author_id"),
+                            "section_id": article.get("section_id"),
+                        },
+                    )
+                )
+
+            url = body.get("next_page")
+
+        return events
+
+    # -- normalization helpers -----------------------------------------------
+
+    def _normalize_ticket(self, raw: RawEvent) -> Episode:
+        ticket = raw.data.get("ticket", {})
+        subject = ticket.get("subject", "Untitled ticket")
+        description = (ticket.get("description") or "")[:1000]
+        status = ticket.get("status", "")
+        priority = ticket.get("priority", "")
+
+        actor = Actor(
+            id=str(ticket.get("requester_id", "")),
+            name=str(ticket.get("requester_id", "Unknown")),
+        )
+
+        entities = self._extract_ticket_entities(ticket)
+        tags = ["zendesk", "ticket", status] + (ticket.get("tags") or [])
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.ISSUE,
+            occurred_at=raw.timestamp,
+            content=f"Ticket [{status}] [{priority}]: {subject}\n{description}".strip(),
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=tags,
+            metadata={
+                "zendesk_ticket_id": ticket.get("id"),
+                "status": status,
+                "priority": priority,
+                "type": ticket.get("type", ""),
+                "group_id": ticket.get("group_id"),
+                "assignee_id": ticket.get("assignee_id"),
+            },
+            thread_id=f"zd:ticket:{ticket.get('id', '')}",
+            idempotency_key=f"zd:ticket:{ticket.get('id', raw.external_id)}",
+        )
+
+    def _normalize_comment(self, raw: RawEvent) -> Episode:
+        data = raw.data
+        comment = data.get("comment", {})
+        ticket_id = data.get("ticket_id", "")
+        ticket_subject = data.get("ticket_subject", "")
+        body = (comment.get("body") or comment.get("plain_body") or "")[:1000]
+
+        actor = Actor(
+            id=str(comment.get("author_id", "")),
+            name=str(comment.get("author_id", "Unknown")),
+        )
+
+        visibility_tag = "public" if comment.get("public", True) else "internal"
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.COMMENT,
+            occurred_at=raw.timestamp,
+            content=f"Comment on [{ticket_subject}]: {body}",
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="ticket",
+                    entity_id=ticket_id,
+                    display_name=ticket_subject,
+                ),
+            ],
+            tags=["zendesk", "comment", visibility_tag],
+            metadata={
+                "ticket_id": ticket_id,
+                "is_public": comment.get("public", True),
+                "has_attachments": bool(comment.get("attachments")),
+            },
+            thread_id=f"zd:ticket:{ticket_id}",
+            parent_id=f"zd:ticket:{ticket_id}",
+            idempotency_key=raw.external_id,
+        )
+
+    def _normalize_article(self, raw: RawEvent) -> Episode:
+        article = raw.data.get("article", {})
+        title = article.get("title", "Untitled article")
+        body = (article.get("body") or "")[:2000]
+
+        actor = Actor(
+            id=str(article.get("author_id", "")),
+            name=str(article.get("author_id", "Unknown")),
+        )
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=f"Article: {title}\n{body}".strip(),
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="article",
+                    entity_id=str(article.get("id", "")),
+                    display_name=title,
+                ),
+            ],
+            tags=["zendesk", "article"] + (article.get("label_names") or []),
+            metadata={
+                "section_id": article.get("section_id"),
+                "locale": article.get("locale", ""),
+                "draft": article.get("draft", False),
+                "vote_sum": article.get("vote_sum", 0),
+            },
+            thread_id=f"zd:article:{article.get('id', '')}",
+            idempotency_key=raw.external_id,
+        )
+
+    # -- entity extraction ---------------------------------------------------
+
+    @staticmethod
+    def _extract_ticket_entities(
+        ticket: dict[str, Any],
+    ) -> list[EntityRef]:
+        entities: list[EntityRef] = []
+
+        ticket_id = ticket.get("id")
+        if ticket_id:
+            entities.append(
+                EntityRef(
+                    entity_type="ticket",
+                    entity_id=str(ticket_id),
+                    display_name=ticket.get("subject", str(ticket_id)),
+                )
+            )
+
+        org_id = ticket.get("organization_id")
+        if org_id:
+            entities.append(
+                EntityRef(
+                    entity_type="organization",
+                    entity_id=str(org_id),
+                    display_name=str(org_id),
+                )
+            )
+
+        assignee_id = ticket.get("assignee_id")
+        if assignee_id:
+            entities.append(
+                EntityRef(
+                    entity_type="user",
+                    entity_id=str(assignee_id),
+                    display_name=str(assignee_id),
+                )
+            )
+
+        group_id = ticket.get("group_id")
+        if group_id:
+            entities.append(
+                EntityRef(
+                    entity_type="group",
+                    entity_id=str(group_id),
+                    display_name=str(group_id),
+                )
+            )
+
+        return entities
+
+    # -- utilities -----------------------------------------------------------
+
+    @staticmethod
+    def _parse_ts(value: str | None) -> datetime:
+        if not value:
+            return datetime.now(timezone.utc)
+        try:
+            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+        except (ValueError, TypeError):
+            return datetime.now(timezone.utc)