cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/intercom/__init__.py

@@ -0,0 +1,631 @@
+"""
+Intercom connector for CortexDB.
+
+Polls Intercom API v2 for Conversations, Conversation parts, Articles,
+and Contacts, converting them into CortexDB Episodes.  Uses ``httpx``
+for direct HTTP access with cursor-based pagination (``starting_after``).
+"""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime, timedelta, timezone
+from typing import Any
+
+import httpx
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="intercom")
+_BASE_URL = "https://api.intercom.io"
+_MAX_RETRIES = 3
+
+
+class IntercomConnector(CortexConnector):
+    """Connector that syncs Intercom conversations and articles into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    access_token:
+        Intercom access token.
+    include_articles:
+        Whether to ingest Help Center articles.
+    namespace:
+        CortexDB namespace for ingested episodes.
+    backfill_days:
+        Number of days of historical data to backfill on first sync.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "intercom"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        access_token: str,
+        include_articles: bool = False,
+        namespace: str = "intercom",
+        backfill_days: int = 90,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.access_token = access_token
+        self.include_articles = include_articles
+        self.namespace = namespace
+        self.backfill_days = backfill_days
+
+    # -- HTTP helpers --------------------------------------------------------
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.access_token}",
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+            "Intercom-Version": "2.11",
+        }
+
+    async def _request(
+        self,
+        client: httpx.AsyncClient,
+        method: str,
+        path: str,
+        **kwargs: Any,
+    ) -> httpx.Response:
+        """Execute an HTTP request with rate-limit retry."""
+        url = f"{_BASE_URL}{path}"
+        for attempt in range(_MAX_RETRIES):
+            resp = await client.request(
+                method, url, headers=self._headers(), **kwargs
+            )
+            if resp.status_code == 429:
+                import asyncio
+
+                retry_after = int(resp.headers.get("Retry-After", "5"))
+                logger.warning(
+                    "Intercom rate limited, retrying in %ds (attempt %d/%d)",
+                    retry_after,
+                    attempt + 1,
+                    _MAX_RETRIES,
+                )
+                await asyncio.sleep(retry_after)
+                continue
+            resp.raise_for_status()
+            return resp
+        raise httpx.HTTPStatusError(
+            "Rate limit retries exhausted",
+            request=resp.request,  # type: ignore[possibly-undefined]
+            response=resp,  # type: ignore[possibly-undefined]
+        )
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch conversations, parts, contacts, and articles from Intercom."""
+        cutoff = since or datetime.now(timezone.utc) - timedelta(
+            days=self.backfill_days
+        )
+        raw_events: list[RawEvent] = []
+
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            raw_events.extend(
+                await self._fetch_conversations(client, cutoff)
+            )
+            raw_events.extend(await self._fetch_contacts(client, cutoff))
+            if self.include_articles:
+                raw_events.extend(await self._fetch_articles(client, cutoff))
+
+        return raw_events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert an Intercom ``RawEvent`` into a CortexDB ``Episode``."""
+        kind = raw.data.get("event_kind", "conversation")
+        if kind == "conversation":
+            return self._normalize_conversation(raw)
+        if kind == "conversation_part":
+            return self._normalize_part(raw)
+        if kind == "article":
+            return self._normalize_article(raw)
+        if kind == "contact":
+            return self._normalize_contact(raw)
+        return self._normalize_conversation(raw)
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Intercom visibility to CortexDB visibility.
+
+        Conversations are ``Organization`` by default.  Internal admin
+        notes are ``Restricted``.
+        """
+        data = raw.data
+        if data.get("is_admin_only"):
+            return Visibility(level=VisibilityLevel.RESTRICTED)
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- fetchers ------------------------------------------------------------
+
+    async def _fetch_conversations(
+        self,
+        client: httpx.AsyncClient,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch conversations and their parts using search + pagination."""
+        events: list[RawEvent] = []
+        cutoff_unix = int(cutoff.timestamp())
+
+        # Use search endpoint to filter by updated_at.
+        starting_after: str | None = None
+
+        while True:
+            payload: dict[str, Any] = {
+                "query": {
+                    "field": "updated_at",
+                    "operator": ">",
+                    "value": cutoff_unix,
+                },
+                "pagination": {"per_page": 50},
+            }
+            if starting_after:
+                payload["pagination"]["starting_after"] = starting_after
+
+            try:
+                resp = await self._request(
+                    client, "POST", "/conversations/search", json=payload
+                )
+            except Exception:
+                logger.exception("Failed to search Intercom conversations")
+                break
+
+            body = resp.json()
+            conversations = body.get("conversations", [])
+
+            for conv in conversations:
+                ts = self._epoch_to_dt(conv.get("updated_at"))
+                events.append(
+                    RawEvent(
+                        source="intercom",
+                        external_id=conv.get("id", ""),
+                        timestamp=ts,
+                        data={
+                            "event_kind": "conversation",
+                            "conversation": conv,
+                            "is_admin_only": False,
+                        },
+                        permissions={
+                            "admin_ids": [
+                                a.get("id", "")
+                                for a in conv.get("admins", {}).get(
+                                    "admins", []
+                                )
+                            ],
+                            "tags": [
+                                t.get("name", "")
+                                for t in conv.get("tags", {}).get("tags", [])
+                            ],
+                        },
+                    )
+                )
+
+                # Fetch conversation parts.
+                events.extend(
+                    await self._fetch_parts(client, conv)
+                )
+
+            # Cursor-based pagination.
+            pages = body.get("pages", {})
+            next_cursor = pages.get("next", {})
+            if next_cursor and next_cursor.get("starting_after"):
+                starting_after = next_cursor["starting_after"]
+            else:
+                break
+
+        return events
+
+    async def _fetch_parts(
+        self,
+        client: httpx.AsyncClient,
+        conversation: dict[str, Any],
+    ) -> list[RawEvent]:
+        """Fetch conversation parts (replies, notes) for a conversation."""
+        events: list[RawEvent] = []
+        conv_id = conversation.get("id", "")
+
+        try:
+            resp = await self._request(
+                client, "GET", f"/conversations/{conv_id}"
+            )
+        except Exception:
+            logger.debug("Could not fetch parts for conversation %s", conv_id)
+            return events
+
+        body = resp.json()
+        parts = body.get("conversation_parts", {}).get(
+            "conversation_parts", []
+        )
+
+        for part in parts:
+            ts = self._epoch_to_dt(part.get("updated_at") or part.get("created_at"))
+            is_admin_note = part.get("part_type") == "note"
+
+            events.append(
+                RawEvent(
+                    source="intercom",
+                    external_id=f"part:{part.get('id', '')}",
+                    timestamp=ts,
+                    data={
+                        "event_kind": "conversation_part",
+                        "part": part,
+                        "conversation_id": conv_id,
+                        "conversation_subject": conversation.get(
+                            "source", {}
+                        ).get("subject", ""),
+                        "is_admin_only": is_admin_note,
+                    },
+                    permissions={
+                        "author_type": part.get("author", {}).get("type", ""),
+                    },
+                )
+            )
+
+        return events
+
+    async def _fetch_contacts(
+        self,
+        client: httpx.AsyncClient,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch contacts updated since cutoff."""
+        events: list[RawEvent] = []
+        cutoff_unix = int(cutoff.timestamp())
+        starting_after: str | None = None
+
+        while True:
+            payload: dict[str, Any] = {
+                "query": {
+                    "field": "updated_at",
+                    "operator": ">",
+                    "value": cutoff_unix,
+                },
+                "pagination": {"per_page": 50},
+            }
+            if starting_after:
+                payload["pagination"]["starting_after"] = starting_after
+
+            try:
+                resp = await self._request(
+                    client, "POST", "/contacts/search", json=payload
+                )
+            except Exception:
+                logger.exception("Failed to search Intercom contacts")
+                break
+
+            body = resp.json()
+            for contact in body.get("data", []):
+                ts = self._epoch_to_dt(contact.get("updated_at"))
+                events.append(
+                    RawEvent(
+                        source="intercom",
+                        external_id=f"contact:{contact.get('id', '')}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "contact",
+                            "contact": contact,
+                            "is_admin_only": False,
+                        },
+                        permissions={},
+                    )
+                )
+
+            pages = body.get("pages", {})
+            next_cursor = pages.get("next", {})
+            if next_cursor and next_cursor.get("starting_after"):
+                starting_after = next_cursor["starting_after"]
+            else:
+                break
+
+        return events
+
+    async def _fetch_articles(
+        self,
+        client: httpx.AsyncClient,
+        cutoff: datetime,
+    ) -> list[RawEvent]:
+        """Fetch Help Center articles."""
+        events: list[RawEvent] = []
+        page_num = 1
+
+        while True:
+            try:
+                resp = await self._request(
+                    client,
+                    "GET",
+                    f"/articles?page={page_num}&per_page=50",
+                )
+            except Exception:
+                logger.exception("Failed to fetch Intercom articles")
+                break
+
+            body = resp.json()
+            articles = body.get("data", [])
+            if not articles:
+                break
+
+            for article in articles:
+                ts = self._epoch_to_dt(article.get("updated_at"))
+                if ts < cutoff:
+                    continue
+                events.append(
+                    RawEvent(
+                        source="intercom",
+                        external_id=f"article:{article.get('id', '')}",
+                        timestamp=ts,
+                        data={
+                            "event_kind": "article",
+                            "article": article,
+                            "is_admin_only": False,
+                        },
+                        permissions={
+                            "author_id": article.get("author_id"),
+                        },
+                    )
+                )
+
+            total_pages = body.get("pages", {}).get("total_pages", 1)
+            if page_num >= total_pages:
+                break
+            page_num += 1
+
+        return events
+
+    # -- normalization helpers -----------------------------------------------
+
+    def _normalize_conversation(self, raw: RawEvent) -> Episode:
+        conv = raw.data.get("conversation", {})
+        source_msg = conv.get("source", {})
+        subject = source_msg.get("subject", "")
+        body = (source_msg.get("body") or "")[:1000]
+        state = conv.get("state", "")
+
+        author = source_msg.get("author", {})
+        actor = Actor(
+            id=str(author.get("id", "")),
+            name=author.get("name") or author.get("email") or str(author.get("id", "Unknown")),
+            email=author.get("email"),
+        )
+
+        entities = self._extract_conversation_entities(conv)
+        tags_list = [
+            t.get("name", "")
+            for t in conv.get("tags", {}).get("tags", [])
+        ]
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.MESSAGE,
+            occurred_at=raw.timestamp,
+            content=f"Conversation [{state}]: {subject}\n{body}".strip(),
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=entities,
+            tags=["intercom", "conversation", state] + tags_list,
+            metadata={
+                "conversation_id": conv.get("id"),
+                "state": state,
+                "priority": conv.get("priority", ""),
+                "sla_applied": bool(conv.get("sla_applied")),
+                "statistics": conv.get("statistics", {}),
+            },
+            thread_id=f"ic:conv:{conv.get('id', '')}",
+            idempotency_key=f"ic:conv:{conv.get('id', raw.external_id)}",
+        )
+
+    def _normalize_part(self, raw: RawEvent) -> Episode:
+        data = raw.data
+        part = data.get("part", {})
+        conv_id = data.get("conversation_id", "")
+        conv_subject = data.get("conversation_subject", "")
+        body = (part.get("body") or "")[:1000]
+        part_type = part.get("part_type", "")
+
+        author = part.get("author", {})
+        actor = Actor(
+            id=str(author.get("id", "")),
+            name=author.get("name") or str(author.get("id", "Unknown")),
+            email=author.get("email"),
+        )
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.COMMENT,
+            occurred_at=raw.timestamp,
+            content=f"Reply on [{conv_subject}] ({part_type}): {body}",
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="conversation",
+                    entity_id=conv_id,
+                    display_name=conv_subject,
+                ),
+            ],
+            tags=["intercom", "conversation_part", part_type],
+            metadata={
+                "conversation_id": conv_id,
+                "part_type": part_type,
+                "author_type": author.get("type", ""),
+            },
+            thread_id=f"ic:conv:{conv_id}",
+            parent_id=f"ic:conv:{conv_id}",
+            idempotency_key=raw.external_id,
+        )
+
+    def _normalize_article(self, raw: RawEvent) -> Episode:
+        article = raw.data.get("article", {})
+        title = article.get("title", "Untitled article")
+        body = (article.get("body") or "")[:2000]
+        state = article.get("state", "")
+
+        return Episode(
+            actor=Actor(
+                id=str(article.get("author_id", "")),
+                name=str(article.get("author_id", "Unknown")),
+            ),
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=f"Article [{state}]: {title}\n{body}".strip(),
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="article",
+                    entity_id=str(article.get("id", "")),
+                    display_name=title,
+                ),
+            ],
+            tags=["intercom", "article", state],
+            metadata={
+                "article_id": article.get("id"),
+                "state": state,
+                "parent_id": article.get("parent_id"),
+                "parent_type": article.get("parent_type", ""),
+            },
+            thread_id=f"ic:article:{article.get('id', '')}",
+            idempotency_key=raw.external_id,
+        )
+
+    def _normalize_contact(self, raw: RawEvent) -> Episode:
+        contact = raw.data.get("contact", {})
+        name = contact.get("name") or contact.get("email") or "Unknown"
+        role = contact.get("role", "")
+        email = contact.get("email", "")
+
+        parts = [f"Contact: {name}"]
+        if role:
+            parts.append(f"Role: {role}")
+        if email:
+            parts.append(f"Email: {email}")
+        if contact.get("phone"):
+            parts.append(f"Phone: {contact['phone']}")
+
+        company = contact.get("companies", {}).get("data", [])
+        company_names = [c.get("name", "") for c in company if c.get("name")]
+        if company_names:
+            parts.append(f"Companies: {', '.join(company_names)}")
+
+        return Episode(
+            actor=Actor(
+                id=str(contact.get("id", "")),
+                name=name,
+                email=email or None,
+            ),
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=" | ".join(parts),
+            raw_payload=raw.data,
+            tenant_id=self.tenant_id,
+            namespace=self.namespace,
+            entities=[
+                EntityRef(
+                    entity_type="contact",
+                    entity_id=str(contact.get("id", "")),
+                    display_name=name,
+                ),
+            ],
+            tags=["intercom", "contact", role],
+            metadata={
+                "contact_id": contact.get("id"),
+                "role": role,
+                "has_hard_bounced": contact.get("has_hard_bounced", False),
+                "unsubscribed": contact.get(
+                    "unsubscribed_from_emails", False
+                ),
+            },
+            idempotency_key=raw.external_id,
+        )
+
+    # -- entity extraction ---------------------------------------------------
+
+    @staticmethod
+    def _extract_conversation_entities(
+        conv: dict[str, Any],
+    ) -> list[EntityRef]:
+        entities: list[EntityRef] = []
+
+        conv_id = conv.get("id", "")
+        if conv_id:
+            entities.append(
+                EntityRef(
+                    entity_type="conversation",
+                    entity_id=conv_id,
+                    display_name=conv.get("source", {}).get("subject", conv_id),
+                )
+            )
+
+        for admin in conv.get("admins", {}).get("admins", []):
+            if admin.get("id"):
+                entities.append(
+                    EntityRef(
+                        entity_type="admin",
+                        entity_id=str(admin["id"]),
+                        display_name=admin.get("name", str(admin["id"])),
+                    )
+                )
+
+        contacts = conv.get("contacts", {}).get("contacts", [])
+        for contact in contacts:
+            if contact.get("id"):
+                entities.append(
+                    EntityRef(
+                        entity_type="contact",
+                        entity_id=str(contact["id"]),
+                        display_name=contact.get("name", str(contact["id"])),
+                    )
+                )
+
+        for tag in conv.get("tags", {}).get("tags", []):
+            if tag.get("id"):
+                entities.append(
+                    EntityRef(
+                        entity_type="tag",
+                        entity_id=str(tag["id"]),
+                        display_name=tag.get("name", str(tag["id"])),
+                    )
+                )
+
+        return entities
+
+    # -- utilities -----------------------------------------------------------
+
+    @staticmethod
+    def _epoch_to_dt(value: int | str | None) -> datetime:
+        if not value:
+            return datetime.now(timezone.utc)
+        try:
+            return datetime.fromtimestamp(int(value), tz=timezone.utc)
+        except (ValueError, TypeError, OSError):
+            return datetime.now(timezone.utc)