cortexdb-connectors 0.2.0__py3-none-any.whl

cortexdb_connectors/__init__.py

@@ -0,0 +1,40 @@
+"""cortexdb_connectors — data connector SDK for the CortexDB v1 API.
+
+Each connector subclass implements `fetch_events`, `normalize`, and
+`map_permissions`; the base class POSTs every normalised Episode to
+`/v1/experience` with the required Authorization + X-Cortex-Actor
+headers. Use the `cortexdb-sync` CLI to run a connector once or in a
+poll loop; or `from cortexdb_connectors.slack import SlackConnector`
+to drive it programmatically.
+"""
+
+__version__ = "0.2.0"
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    SyncResult,
+    SyncState,
+    Visibility,
+    VisibilityLevel,
+)
+
+__all__ = [
+    "__version__",
+    "Actor",
+    "CortexConnector",
+    "EntityRef",
+    "Episode",
+    "EpisodeType",
+    "RawEvent",
+    "Source",
+    "SyncResult",
+    "SyncState",
+    "Visibility",
+    "VisibilityLevel",
+]

cortexdb_connectors/base.py

@@ -0,0 +1,364 @@
+"""CortexDB Connector SDK — base classes and data models (v1 surface).
+
+Each connector subclass implements `fetch_events`, `normalize`, and
+`map_permissions`. The base class orchestrates a fetch → normalize →
+ingest cycle and POSTs each Episode to `POST /v1/experience` as a fully
+formed envelope. v1 requires `Authorization: Bearer <PASETO>` AND
+`X-Cortex-Actor: <actor>` on every request, both of which the base
+class stamps for you.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import uuid
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+
+DEFAULT_API_URL = "https://api-v1.cortexdb.ai"
+
+
+# ---------------------------------------------------------------------------
+# Enums
+# ---------------------------------------------------------------------------
+
+
+class EpisodeType(str, Enum):
+    """Canonical episode types understood by CortexDB.
+
+    These map to v1 `modality` values during ingest. Most messaging-like
+    events become `conversation`; documents become `document`; events
+    without a natural author become `observation`.
+    """
+
+    MESSAGE = "message"
+    CODE_CHANGE = "code_change"
+    ISSUE = "issue"
+    INCIDENT = "incident"
+    DOCUMENT = "document"
+    REVIEW = "review"
+    COMMENT = "comment"
+    DEPLOYMENT = "deployment"
+    ALERT = "alert"
+    MEETING = "meeting"
+    CUSTOM = "custom"
+
+
+# How each EpisodeType lands as a v1 modality. The connector can override
+# by setting `episode.metadata["modality"]` explicitly.
+_TYPE_TO_MODALITY: dict[EpisodeType, str] = {
+    EpisodeType.MESSAGE: "conversation",
+    EpisodeType.COMMENT: "conversation",
+    EpisodeType.REVIEW: "feedback",
+    EpisodeType.CODE_CHANGE: "document",
+    EpisodeType.ISSUE: "document",
+    EpisodeType.DOCUMENT: "document",
+    EpisodeType.INCIDENT: "observation",
+    EpisodeType.DEPLOYMENT: "observation",
+    EpisodeType.ALERT: "observation",
+    EpisodeType.MEETING: "conversation",
+    EpisodeType.CUSTOM: "observation",
+}
+
+
+class VisibilityLevel(str, Enum):
+    """Controls who may access an episode within CortexDB.
+
+    Kept for connector-side modelling; the v1 surface uses scope paths
+    and capabilities, not these labels. We currently fold visibility
+    into `context.labels` for traceability rather than enforcement.
+    """
+
+    PUBLIC = "public"
+    ORGANIZATION = "organization"
+    RESTRICTED = "restricted"
+    PRIVATE = "private"
+
+
+# ---------------------------------------------------------------------------
+# Data-transfer objects
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class Source:
+    system: str
+    connector_version: str = "0.2.0"
+
+
+@dataclass(frozen=True)
+class Actor:
+    id: str
+    name: str
+    email: str | None = None
+    avatar_url: str | None = None
+
+
+@dataclass(frozen=True)
+class EntityRef:
+    entity_type: str
+    entity_id: str
+    display_name: str = ""
+
+
+@dataclass(frozen=True)
+class Visibility:
+    level: VisibilityLevel
+    allowed_principals: tuple[str, ...] = ()
+
+
+@dataclass
+class RawEvent:
+    source: str
+    external_id: str
+    timestamp: datetime
+    data: dict[str, Any]
+    permissions: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class Episode:
+    """A normalized unit of knowledge ready for ingestion into CortexDB."""
+
+    id: str = field(default_factory=lambda: str(uuid.uuid4()))
+    actor: Actor | None = None
+    source: Source | None = None
+    episode_type: EpisodeType = EpisodeType.CUSTOM
+    occurred_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    content: str = ""
+    raw_payload: dict[str, Any] = field(default_factory=dict)
+    visibility: Visibility = field(
+        default_factory=lambda: Visibility(level=VisibilityLevel.ORGANIZATION)
+    )
+    tenant_id: str = "default"
+    namespace: str = "default"
+    entities: list[EntityRef] = field(default_factory=list)
+    tags: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    thread_id: str | None = None
+    parent_id: str | None = None
+    idempotency_key: str | None = None
+    ttl_seconds: int | None = None
+
+
+@dataclass
+class SyncResult:
+    episodes_fetched: int = 0
+    episodes_ingested: int = 0
+    errors: list[str] = field(default_factory=list)
+    duration_seconds: float = 0.0
+
+
+@dataclass
+class SyncState:
+    connector_name: str
+    tenant_id: str
+    last_synced_at: datetime | None = None
+    cursor: str | None = None
+    extra: dict[str, Any] = field(default_factory=dict)
+
+
+# ---------------------------------------------------------------------------
+# Abstract base connector
+# ---------------------------------------------------------------------------
+
+
+class CortexConnector(ABC):
+    """Base class for CortexDB data connectors.
+
+    Concrete connectors implement `fetch_events`, `normalize`, and
+    `map_permissions`. The base orchestrates a sync cycle and ingests
+    each normalised Episode via `POST /v1/experience`.
+
+    Constructor accepts the v0 positional shape `(cortex_url,
+    cortex_api_key, tenant_id)` so existing connector classes that pass
+    `super().__init__(cortex_url, cortex_api_key, tenant_id)` keep
+    working without modification. Pass `actor=` / `scope_template=` (or
+    set the matching env vars) to target a specific scope on v1.
+    """
+
+    def __init__(
+        self,
+        cortex_url: str | None = None,
+        cortex_api_key: str | None = None,
+        tenant_id: str = "default",
+        *,
+        actor: str | None = None,
+        scope_template: str | None = None,
+    ) -> None:
+        self.cortex_url = (cortex_url or os.environ.get("CORTEXDB_URL") or DEFAULT_API_URL).rstrip("/")
+        self.cortex_api_key = cortex_api_key or os.environ.get("CORTEXDB_API_KEY") or ""
+        self.tenant_id = tenant_id
+        # v1-specific fields. Fall back to env so a freshly signed-up actor
+        # (state.json from `cortexdb init` or the MCP server) is picked up
+        # without any code change in the connector subclass.
+        self.actor = actor or os.environ.get("CORTEXDB_ACTOR") or ""
+        self.scope_template = (
+            scope_template
+            or os.environ.get("CORTEXDB_SCOPE_TEMPLATE")
+            or os.environ.get("CORTEXDB_SCOPE")
+            or self.actor
+        )
+
+    # -- abstract interface --------------------------------------------------
+
+    @abstractmethod
+    async def fetch_events(self, since: datetime | None = None) -> list[RawEvent]:
+        ...
+
+    @abstractmethod
+    def normalize(self, raw: RawEvent) -> Episode:
+        ...
+
+    @abstractmethod
+    def map_permissions(self, raw: RawEvent) -> Visibility:
+        ...
+
+    # -- default sync loop ---------------------------------------------------
+
+    async def sync(self, since: datetime | None = None) -> SyncResult:
+        import time
+
+        start = time.monotonic()
+        result = SyncResult()
+        raw_events = await self.fetch_events(since=since)
+        result.episodes_fetched = len(raw_events)
+
+        for raw in raw_events:
+            try:
+                episode = self.normalize(raw)
+                episode.visibility = self.map_permissions(raw)
+                await self.ingest_episode(episode)
+                result.episodes_ingested += 1
+            except Exception as exc:  # noqa: BLE001
+                result.errors.append(
+                    f"Error processing event {raw.external_id}: {exc}"
+                )
+
+        result.duration_seconds = round(time.monotonic() - start, 3)
+        return result
+
+    # -- v1 ingest -----------------------------------------------------------
+
+    def _resolve_scope(self, episode: Episode) -> str:
+        """Resolve the scope template against this episode.
+
+        Placeholders: `{tenant}`, `{source}`, `{namespace}`, `{actor}`,
+        plus any `{entity.<type>}` shorthand for the first entity of that
+        type attached to the episode (e.g. `{entity.channel}`).
+        """
+        tpl = self.scope_template or self.actor or "org:default"
+        if not ("{" in tpl):
+            return tpl
+
+        replacements: dict[str, str] = {
+            "tenant": episode.tenant_id or self.tenant_id,
+            "source": (episode.source.system if episode.source else episode.namespace),
+            "namespace": episode.namespace,
+            "actor": self.actor,
+        }
+        for ent in episode.entities:
+            replacements[f"entity.{ent.entity_type}"] = ent.entity_id
+
+        for key, value in replacements.items():
+            tpl = tpl.replace("{" + key + "}", value or "")
+        return tpl
+
+    def _envelope(self, episode: Episode) -> dict[str, Any]:
+        """Build the v1 ExperienceItem JSON for this episode."""
+        scope = self._resolve_scope(episode)
+        modality = (
+            episode.metadata.get("modality")
+            or _TYPE_TO_MODALITY.get(episode.episode_type, "observation")
+        )
+
+        labels = list(episode.tags)
+        if episode.source and episode.source.system:
+            labels.append(f"source:{episode.source.system}")
+        if episode.visibility and episode.visibility.level:
+            labels.append(f"visibility:{episode.visibility.level.value}")
+        for ent in episode.entities:
+            labels.append(f"{ent.entity_type}:{ent.entity_id}")
+
+        content: dict[str, Any] = {
+            "kind": "message",
+            "role": "user" if modality == "conversation" else "system",
+            "text": episode.content,
+        }
+        context: dict[str, Any] = {
+            "observed_at": episode.occurred_at.isoformat(),
+            "labels": labels,
+        }
+        if episode.thread_id:
+            context["thread_id"] = episode.thread_id
+        if episode.parent_id:
+            context["preceded_by"] = [episode.parent_id]
+        if episode.metadata:
+            # Forward non-reserved metadata under a single key — the v1
+            # surface doesn't reject unknown context keys but we tuck
+            # everything under `meta` to keep the envelope tidy.
+            context["meta"] = {k: v for k, v in episode.metadata.items() if k != "modality"}
+
+        envelope: dict[str, Any] = {
+            "scope": scope,
+            "modality": modality,
+            "content": content,
+            "context": context,
+            "idempotency_key": (
+                episode.idempotency_key
+                or self._fallback_idem(episode, scope)
+            ),
+        }
+        if episode.actor:
+            # v1 wants ActorRef = {id: "kind:local"} not a bare string.
+            envelope["observed_actor"] = {"id": episode.actor.id}
+
+        return envelope
+
+    @staticmethod
+    def _fallback_idem(episode: Episode, scope: str) -> str:
+        """Deterministic idempotency_key when the connector didn't set one."""
+        src = episode.source.system if episode.source else "cx"
+        seed = f"{scope}|{src}|{episode.id}|{episode.content[:512]}"
+        return f"conn:{src}:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:24]}"
+
+    async def ingest_episode(self, episode: Episode) -> str:
+        """POST `/v1/experience` with the v1 envelope.
+
+        Returns the server-side event_id on success.
+        """
+        import httpx
+
+        if not self.cortex_api_key:
+            raise RuntimeError(
+                "CortexDB API key missing — set CORTEXDB_API_KEY or pass "
+                "cortex_api_key explicitly. Run `cortexdb auth signup --save` "
+                "for a free-tier token."
+            )
+        if not self.actor:
+            raise RuntimeError(
+                "CortexDB actor missing — set CORTEXDB_ACTOR or pass actor= "
+                "explicitly. The actor must match the token's `sub` claim."
+            )
+
+        envelope = self._envelope(episode)
+        headers = {
+            "Authorization": f"Bearer {self.cortex_api_key}",
+            "X-Cortex-Actor": self.actor,
+            "Content-Type": "application/json",
+            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
+        }
+
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/experience",
+                json=envelope,
+                headers=headers,
+            )
+            resp.raise_for_status()
+            body = resp.json()
+        return body.get("event_id") or body.get("id") or episode.id