cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/slack/__init__.py

@@ -0,0 +1,331 @@
+"""
+Slack connector for CortexDB.
+
+Ingests messages (including threaded replies) from Slack channels into
+CortexDB as Episodes.  Uses the ``slack_sdk`` WebClient to interact with
+the Slack API and respects Tier-3 rate limits (~50 requests/minute).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    Visibility,
+    VisibilityLevel,
+)
+
+logger = logging.getLogger(__name__)
+
+_SOURCE = Source(system="slack")
+
+# Slack Tier-3 rate limit: ~50 req/min -> ~1.2 s between requests.
+_RATE_LIMIT_DELAY = 1.3
+
+
+class SlackConnector(CortexConnector):
+    """Connector that syncs Slack channel messages into CortexDB.
+
+    Parameters
+    ----------
+    cortex_url:
+        Base URL of the CortexDB API.
+    cortex_api_key:
+        Bearer token for CortexDB authentication.
+    slack_bot_token:
+        Slack Bot User OAuth Token (``xoxb-...``).
+    channels:
+        Explicit list of channel IDs to sync.  If empty, the connector
+        will auto-discover all channels the bot has been added to.
+    tenant_id:
+        CortexDB tenant identifier.
+    """
+
+    connector_name: str = "slack"
+
+    def __init__(
+        self,
+        cortex_url: str,
+        cortex_api_key: str,
+        slack_bot_token: str,
+        channels: list[str] | None = None,
+        tenant_id: str = "default",
+    ) -> None:
+        super().__init__(cortex_url, cortex_api_key, tenant_id)
+        self.slack_bot_token = slack_bot_token
+        self.channels = channels or []
+        self._user_cache: dict[str, dict[str, Any]] = {}
+
+    # -- CortexConnector interface -------------------------------------------
+
+    async def fetch_events(
+        self, since: datetime | None = None
+    ) -> list[RawEvent]:
+        """Fetch messages from configured Slack channels.
+
+        If *since* is provided, only messages newer than that timestamp
+        are returned.  Pagination is handled automatically.
+        """
+        from slack_sdk import WebClient
+        from slack_sdk.errors import SlackApiError
+
+        client = WebClient(token=self.slack_bot_token)
+        channels = self.channels or self._discover_channels(client)
+        oldest = str(since.timestamp()) if since else "0"
+        events: list[RawEvent] = []
+
+        for channel_id in channels:
+            cursor: str | None = None
+            while True:
+                try:
+                    kwargs: dict[str, Any] = {
+                        "channel": channel_id,
+                        "oldest": oldest,
+                        "limit": 200,
+                    }
+                    if cursor:
+                        kwargs["cursor"] = cursor
+
+                    resp = client.conversations_history(**kwargs)
+                except SlackApiError as exc:
+                    if exc.response.get("error") == "ratelimited":
+                        retry_after = int(
+                            exc.response.headers.get("Retry-After", 5)
+                        )
+                        logger.warning(
+                            "Slack rate-limited; sleeping %d s", retry_after
+                        )
+                        await asyncio.sleep(retry_after)
+                        continue
+                    raise
+
+                channel_info = self._get_channel_info(client, channel_id)
+
+                for msg in resp.get("messages", []):
+                    raw = RawEvent(
+                        source="slack",
+                        external_id=f"{channel_id}:{msg.get('ts', '')}",
+                        timestamp=datetime.fromtimestamp(
+                            float(msg.get("ts", "0")), tz=timezone.utc
+                        ),
+                        data={**msg, "_channel_id": channel_id},
+                        permissions={
+                            "is_private": channel_info.get("is_private", False),
+                            "channel_members": channel_info.get("members", []),
+                        },
+                    )
+                    events.append(raw)
+
+                    # Fetch thread replies if this message has a thread.
+                    if msg.get("reply_count", 0) > 0:
+                        thread_events = self._fetch_thread_replies(
+                            client, channel_id, msg["ts"], oldest, channel_info
+                        )
+                        events.extend(thread_events)
+
+                next_cursor = (
+                    resp.get("response_metadata", {}).get("next_cursor")
+                )
+                if not next_cursor:
+                    break
+                cursor = next_cursor
+                await asyncio.sleep(_RATE_LIMIT_DELAY)
+
+            await asyncio.sleep(_RATE_LIMIT_DELAY)
+
+        return events
+
+    def normalize(self, raw: RawEvent) -> Episode:
+        """Convert a Slack message ``RawEvent`` into a CortexDB ``Episode``."""
+        data = raw.data
+        user_id = data.get("user", "unknown")
+        actor = self._make_actor(user_id)
+        channel_id = data.get("_channel_id", "")
+
+        thread_ts = data.get("thread_ts")
+        parent_ts = None
+        if thread_ts and thread_ts != data.get("ts"):
+            parent_ts = f"{channel_id}:{thread_ts}"
+
+        return Episode(
+            actor=actor,
+            source=_SOURCE,
+            episode_type=EpisodeType.MESSAGE,
+            occurred_at=raw.timestamp,
+            content=data.get("text", ""),
+            raw_payload=data,
+            tenant_id=self.tenant_id,
+            namespace="slack",
+            entities=[
+                EntityRef(
+                    entity_type="channel",
+                    entity_id=channel_id,
+                    display_name=data.get("_channel_name", channel_id),
+                ),
+            ],
+            tags=["slack"],
+            metadata={
+                "subtype": data.get("subtype", "message"),
+                "reactions": data.get("reactions", []),
+            },
+            thread_id=(
+                f"{channel_id}:{thread_ts}" if thread_ts else None
+            ),
+            parent_id=parent_ts,
+            idempotency_key=raw.external_id,
+        )
+
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        """Map Slack channel privacy to CortexDB visibility.
+
+        Public channels map to ``Organization``; private channels map to
+        ``Restricted`` with the channel member list as allowed principals.
+        """
+        perms = raw.permissions
+        if perms.get("is_private"):
+            members = tuple(str(m) for m in perms.get("channel_members", []))
+            return Visibility(
+                level=VisibilityLevel.RESTRICTED,
+                allowed_principals=members,
+            )
+        return Visibility(level=VisibilityLevel.ORGANIZATION)
+
+    # -- helpers -------------------------------------------------------------
+
+    def _discover_channels(self, client: Any) -> list[str]:
+        """Auto-discover channels the bot is a member of."""
+        channel_ids: list[str] = []
+        cursor: str | None = None
+        while True:
+            kwargs: dict[str, Any] = {
+                "types": "public_channel,private_channel",
+                "limit": 200,
+            }
+            if cursor:
+                kwargs["cursor"] = cursor
+            resp = client.conversations_list(**kwargs)
+            for ch in resp.get("channels", []):
+                if ch.get("is_member"):
+                    channel_ids.append(ch["id"])
+            next_cursor = resp.get("response_metadata", {}).get("next_cursor")
+            if not next_cursor:
+                break
+            cursor = next_cursor
+        return channel_ids
+
+    def _get_channel_info(self, client: Any, channel_id: str) -> dict[str, Any]:
+        """Return cached channel metadata."""
+        try:
+            resp = client.conversations_info(channel=channel_id)
+            return resp.get("channel", {})
+        except Exception:
+            logger.debug("Could not fetch channel info for %s", channel_id)
+            return {}
+
+    def _fetch_thread_replies(
+        self,
+        client: Any,
+        channel_id: str,
+        thread_ts: str,
+        oldest: str,
+        channel_info: dict[str, Any],
+    ) -> list[RawEvent]:
+        """Fetch all replies within a Slack thread."""
+        from slack_sdk.errors import SlackApiError
+
+        events: list[RawEvent] = []
+        cursor: str | None = None
+        while True:
+            try:
+                kwargs: dict[str, Any] = {
+                    "channel": channel_id,
+                    "ts": thread_ts,
+                    "oldest": oldest,
+                    "limit": 200,
+                }
+                if cursor:
+                    kwargs["cursor"] = cursor
+                resp = client.conversations_replies(**kwargs)
+            except SlackApiError as exc:
+                if exc.response.get("error") == "ratelimited":
+                    retry_after = int(
+                        exc.response.headers.get("Retry-After", 5)
+                    )
+                    logger.warning(
+                        "Slack rate-limited (thread); sleeping %d s",
+                        retry_after,
+                    )
+                    import time
+                    time.sleep(retry_after)
+                    continue
+                raise
+
+            for msg in resp.get("messages", []):
+                # Skip the parent message itself (already captured).
+                if msg.get("ts") == thread_ts and not msg.get("parent_user_id"):
+                    continue
+                raw = RawEvent(
+                    source="slack",
+                    external_id=f"{channel_id}:{msg.get('ts', '')}",
+                    timestamp=datetime.fromtimestamp(
+                        float(msg.get("ts", "0")), tz=timezone.utc
+                    ),
+                    data={**msg, "_channel_id": channel_id},
+                    permissions={
+                        "is_private": channel_info.get("is_private", False),
+                        "channel_members": channel_info.get("members", []),
+                    },
+                )
+                events.append(raw)
+
+            next_cursor = resp.get("response_metadata", {}).get("next_cursor")
+            if not next_cursor:
+                break
+            cursor = next_cursor
+        return events
+
+    def _make_actor(self, user_id: str) -> Actor:
+        """Build an Actor from a Slack user ID (uses in-memory cache)."""
+        if user_id in self._user_cache:
+            info = self._user_cache[user_id]
+        else:
+            info = {"id": user_id, "name": user_id}
+            self._user_cache[user_id] = info
+        return Actor(
+            id=info.get("id", user_id),
+            name=info.get("real_name", info.get("name", user_id)),
+            email=info.get("profile", {}).get("email"),
+            avatar_url=info.get("profile", {}).get("image_72"),
+        )
+
+    async def populate_user_cache(self) -> None:
+        """Pre-populate the user cache by calling ``users.list``.
+
+        Call this before ``sync`` to enrich actor metadata on episodes.
+        """
+        from slack_sdk import WebClient
+
+        client = WebClient(token=self.slack_bot_token)
+        cursor: str | None = None
+        while True:
+            kwargs: dict[str, Any] = {"limit": 200}
+            if cursor:
+                kwargs["cursor"] = cursor
+            resp = client.users_list(**kwargs)
+            for member in resp.get("members", []):
+                self._user_cache[member["id"]] = member
+            next_cursor = resp.get("response_metadata", {}).get("next_cursor")
+            if not next_cursor:
+                break
+            cursor = next_cursor
+            await asyncio.sleep(_RATE_LIMIT_DELAY)

cortexdb_connectors/state.py

@@ -0,0 +1,145 @@
+"""
+Sync state persistence for CortexDB connectors.
+
+Stores per-connector, per-tenant cursor positions in a local JSON file so
+that incremental syncs can resume from where they left off.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from cortexdb_connectors.base import SyncState
+
+
+_DEFAULT_STATE_PATH = Path.home() / ".cortexdb" / "sync_state.json"
+
+
+class SyncStateStore:
+    """Persists connector sync cursors to a JSON file on disk.
+
+    The file layout is::
+
+        {
+          "<connector_name>::<tenant_id>": {
+            "connector_name": "...",
+            "tenant_id": "...",
+            "last_synced_at": "ISO-8601 | null",
+            "cursor": "...",
+            "extra": {}
+          }
+        }
+    """
+
+    def __init__(self, path: str | Path | None = None) -> None:
+        self.path = Path(path) if path else _DEFAULT_STATE_PATH
+        self._ensure_dir()
+
+    # -- public API ----------------------------------------------------------
+
+    def get(self, connector_name: str, tenant_id: str = "default") -> SyncState:
+        """Load the sync state for *connector_name* and *tenant_id*.
+
+        Returns a fresh ``SyncState`` with ``last_synced_at=None`` if no
+        previous state exists.
+        """
+        data = self._load()
+        key = self._key(connector_name, tenant_id)
+        entry = data.get(key)
+
+        if entry is None:
+            return SyncState(connector_name=connector_name, tenant_id=tenant_id)
+
+        last_synced: datetime | None = None
+        if entry.get("last_synced_at"):
+            last_synced = datetime.fromisoformat(entry["last_synced_at"])
+
+        return SyncState(
+            connector_name=connector_name,
+            tenant_id=tenant_id,
+            last_synced_at=last_synced,
+            cursor=entry.get("cursor"),
+            extra=entry.get("extra", {}),
+        )
+
+    def save(self, state: SyncState) -> None:
+        """Persist the given ``SyncState`` to disk."""
+        data = self._load()
+        key = self._key(state.connector_name, state.tenant_id)
+        data[key] = {
+            "connector_name": state.connector_name,
+            "tenant_id": state.tenant_id,
+            "last_synced_at": (
+                state.last_synced_at.isoformat() if state.last_synced_at else None
+            ),
+            "cursor": state.cursor,
+            "extra": state.extra,
+        }
+        self._write(data)
+
+    def update_last_synced(
+        self,
+        connector_name: str,
+        tenant_id: str = "default",
+        timestamp: datetime | None = None,
+    ) -> SyncState:
+        """Convenience: set ``last_synced_at`` to *timestamp* (default: now UTC)."""
+        state = self.get(connector_name, tenant_id)
+        state.last_synced_at = timestamp or datetime.now(timezone.utc)
+        self.save(state)
+        return state
+
+    def list_all(self) -> list[SyncState]:
+        """Return every stored ``SyncState`` entry."""
+        data = self._load()
+        states: list[SyncState] = []
+        for entry in data.values():
+            last_synced: datetime | None = None
+            if entry.get("last_synced_at"):
+                last_synced = datetime.fromisoformat(entry["last_synced_at"])
+            states.append(
+                SyncState(
+                    connector_name=entry["connector_name"],
+                    tenant_id=entry["tenant_id"],
+                    last_synced_at=last_synced,
+                    cursor=entry.get("cursor"),
+                    extra=entry.get("extra", {}),
+                )
+            )
+        return states
+
+    def clear(self, connector_name: str, tenant_id: str = "default") -> None:
+        """Remove the stored state for a specific connector/tenant pair."""
+        data = self._load()
+        key = self._key(connector_name, tenant_id)
+        data.pop(key, None)
+        self._write(data)
+
+    # -- internals -----------------------------------------------------------
+
+    @staticmethod
+    def _key(connector_name: str, tenant_id: str) -> str:
+        return f"{connector_name}::{tenant_id}"
+
+    def _ensure_dir(self) -> None:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+
+    def _load(self) -> dict[str, Any]:
+        if not self.path.exists():
+            return {}
+        with open(self.path, "r", encoding="utf-8") as fh:
+            return json.load(fh)
+
+    def _write(self, data: dict[str, Any]) -> None:
+        tmp = self.path.with_suffix(".tmp")
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump(data, fh, indent=2, default=str)
+        # Atomic-ish rename (works on POSIX; on Windows replaces if exists in
+        # Python 3.12+, but we guard with a remove for older versions).
+        if os.name == "nt" and tmp.exists() and self.path.exists():
+            self.path.unlink()
+        tmp.rename(self.path)