cortexdb-connectors 0.2.9__tar.gz → 0.2.11__tar.gz

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.9
+Version: 0.2.11
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0
@@ -32,6 +32,7 @@ Provides-Extra: pagerduty
 Requires-Dist: pdpyras>=5.0; extra == "pagerduty"
 Provides-Extra: jira
 Requires-Dist: jira>=3.0; extra == "jira"
+Requires-Dist: tzdata; extra == "jira"
 Provides-Extra: confluence
 Requires-Dist: atlassian-python-api>=3.0; extra == "confluence"
 Provides-Extra: notion
@@ -62,6 +63,7 @@ Requires-Dist: slack-sdk>=3.0; extra == "all"
 Requires-Dist: pygithub>=2.0; extra == "all"
 Requires-Dist: pdpyras>=5.0; extra == "all"
 Requires-Dist: jira>=3.0; extra == "all"
+Requires-Dist: tzdata; extra == "all"
 Requires-Dist: atlassian-python-api>=3.0; extra == "all"
 Requires-Dist: notion-client>=2.0; extra == "all"
 Requires-Dist: aiohttp>=3.9; extra == "all"
@@ -156,7 +158,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.11}/README.md

@@ -78,7 +78,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.11}/cortexdb_connectors/base.py

@@ -36,6 +36,11 @@ _ACTOR_LOCAL_ILLEGAL = re.compile(r"[^A-Za-z0-9_.@-]")
 # never drop content silently.
 _MAX_CONTENT_BYTES = 10 * 1024 * 1024  # 10 MB
 
+# Server-enforced ceiling on `idempotency_key` (cortex-types
+# `EnvelopeError::IdempotencyKeyTooLong`). Over it, the write is refused with
+# 422 INVALID_BODY — which stalls a sync cursor — so keys are built to fit.
+_MAX_IDEM_BYTES = 64
+
 
 # ---------------------------------------------------------------------------
 # Enums
@@ -102,7 +107,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.9"
+    connector_version: str = "0.2.11"
 
 
 @dataclass(frozen=True)
@@ -126,6 +131,25 @@ class Visibility:
     allowed_principals: tuple[str, ...] = ()
 
 
+@dataclass
+class Attachment:
+    """A binary attachment destined for `POST /v1/blobs` + a `blob_ref`
+    experience, so CortexDB's multimodal pipeline (Tika/vision/ASR content
+    processors) extracts and indexes it.
+
+    `data` carries the raw bytes (downloaded by the connector's
+    `fetch_events`); when `data` is None the attachment is metadata-only
+    (e.g. oversized skip) and only its filename line survives in text.
+    """
+
+    filename: str
+    mime_type: str
+    size_bytes: int = 0
+    data: bytes | None = None
+    source_url: str | None = None
+    attachment_id: str = ""
+
+
 @dataclass
 class RawEvent:
     source: str
@@ -158,6 +182,11 @@ class Episode:
     parent_id: str | None = None
     idempotency_key: str | None = None
     ttl_seconds: int | None = None
+    # When set, this episode is a BINARY attachment: `ingest_episode` uploads
+    # `blob.data` to `POST /v1/blobs` and sends the experience with
+    # `content.kind="blob_ref"` (the multimodal path) instead of text. The
+    # `content` field then serves only as the no-bytes fallback text.
+    blob: Attachment | None = None
 
 
 @dataclass
@@ -234,6 +263,14 @@ class CortexConnector(ABC):
             or self.scope
             or self.actor
         )
+        # idempotency_key → server blob_id for every blob uploaded this run,
+        # and → event_id for everything ingested this run. A connector that
+        # reconciles source deletions persists these so a LATER run can name
+        # the event and purge the bytes of an item deleted at the source;
+        # neither id is discoverable after the fact (the blob id is only
+        # visible at upload time, and idempotency records expire in 24h).
+        self._uploaded_blobs: dict[str, str] = {}
+        self._ingested_events: dict[str, str] = {}
 
     def bind(self, *, actor: str | None = None, scope_template: str | None = None) -> "CortexConnector":
         """Set v1-only fields after construction. Returns self for chaining."""
@@ -338,8 +375,13 @@ class CortexConnector(ABC):
             )
         return tpl
 
-    def _envelope(self, episode: Episode) -> dict[str, Any]:
-        """Build the v1 ExperienceItem JSON for this episode."""
+    def _envelope(self, episode: Episode, blob_id: str | None = None) -> dict[str, Any]:
+        """Build the v1 ExperienceItem JSON for this episode.
+
+        With `blob_id` (an id returned by `upload_blob`) the content becomes
+        `kind="blob_ref"` — the server fetches the stored bytes and runs the
+        per-modality content processor on them. Otherwise plain text.
+        """
         scope = self._resolve_scope(episode)
         modality = (
             episode.metadata.get("modality")
@@ -354,11 +396,19 @@ class CortexConnector(ABC):
         for ent in episode.entities:
             labels.append(f"{ent.entity_type}:{ent.entity_id}")
 
-        content: dict[str, Any] = {
-            "kind": "message",
-            "role": "user" if modality == "conversation" else "system",
-            "text": self._guard_content(episode.content, episode),
-        }
+        if blob_id is not None:
+            content: dict[str, Any] = {"kind": "blob_ref", "blob_id": blob_id}
+            # Filename/MIME ride labels — blob_ref content carries no text,
+            # and labels are the queryable metadata surface.
+            if episode.blob is not None:
+                labels.append(f"filename={episode.blob.filename}")
+                labels.append(f"mime={episode.blob.mime_type}")
+        else:
+            content = {
+                "kind": "message",
+                "role": "user" if modality == "conversation" else "system",
+                "text": self._guard_content(episode.content, episode),
+            }
         # The v1 write envelope is STRICT (deny_unknown_fields) as of v0.8.0:
         # thread_id / meta / preceded_by are NOT envelope context fields, and
         # sending them returns 422 INVALID_BODY (which stalls the sync cursor
@@ -387,10 +437,7 @@ class CortexConnector(ABC):
             "modality": modality,
             "content": content,
             "context": context,
-            "idempotency_key": (
-                episode.idempotency_key
-                or self._fallback_idem(episode, scope)
-            ),
+            "idempotency_key": self._idem_key(episode, scope),
         }
         if episode.actor and episode.actor.id:
             # Source actor ids are not guaranteed bare: Jira Cloud accountIds
@@ -414,6 +461,38 @@ class CortexConnector(ABC):
         seed = f"{scope}|{src}|{episode.id}|{episode.content[:512]}"
         return f"conn:{src}:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:24]}"
 
+    def _idem_key(self, episode: Episode, scope: str) -> str:
+        """The key actually sent — the connector's key QUALIFIED BY SCOPE.
+
+        C4: the server namespaces idempotency records by the scope ROOT (the
+        first path segment) plus the caller, while the envelope it hashes
+        carries the FULL scope. So the same source item synced into two
+        sibling scopes under one root re-used one record with a different
+        body — `409 IDEMPOTENCY_CONFLICT`, and every event after the first
+        scope was silently dropped. Qualifying the key with the resolved
+        scope makes it mean what it always claimed: "this item, in this
+        scope".
+
+        The scope rides as an 8-hex-char digest because the wire field is
+        capped at `_MAX_IDEM_BYTES` and scope paths are unbounded; a key that
+        can't fit the suffix is replaced by a digest of (scope, key) rather
+        than truncated, since a truncated key would collide across items.
+
+        Same-scope semantics are unchanged: one item still maps to exactly
+        one key, so re-syncs replay and poll/webhook paths (which resolve to
+        the same scope) still share one key. `_fallback_idem` keys already
+        hash the scope in, so they pass through untouched.
+        """
+        key = episode.idempotency_key
+        if not key:
+            return self._fallback_idem(episode, scope)
+        digest = hashlib.sha256(scope.encode("utf-8")).hexdigest()
+        suffix = f"@{digest[:8]}"
+        if len(key) + len(suffix) <= _MAX_IDEM_BYTES:
+            return f"{key}{suffix}"
+        seed = f"{scope}|{key}".encode("utf-8")
+        return f"cx:{hashlib.sha256(seed).hexdigest()[:40]}"
+
     @staticmethod
     def _guard_content(text: str, episode: Episode) -> str:
         """Hard 10 MB ceiling on content. Real items never hit this; if one
@@ -433,10 +512,168 @@ class CortexConnector(ABC):
         )
         return capped
 
+    def _headers(self, content_type: str = "application/json") -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.cortex_api_key}",
+            "X-Cortex-Actor": self.actor,
+            "Content-Type": content_type,
+            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
+        }
+
+    async def upload_blob(self, data: bytes, mime_type: str) -> str:
+        """POST raw bytes to `/v1/blobs`; returns the server `blob_id`."""
+        import httpx
+
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/blobs",
+                content=data,
+                headers=self._headers(mime_type or "application/octet-stream"),
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "blob upload REJECTED: status=%s bytes=%d mime=%s body=%s",
+                    resp.status_code, len(data), mime_type, resp.text[:300],
+                )
+            resp.raise_for_status()
+            return resp.json()["blob_id"]
+
+    async def experience_status(
+        self, idempotency_key: str, scope: str
+    ) -> dict[str, Any]:
+        """`GET /v1/experience/status` for a prior write with this key.
+
+        Returns the server's status document (`{found, event_id, status,
+        …}`), or `{"found": False}` when the probe fails — callers treat an
+        unreachable probe as "unknown", never as proof of absence.
+        """
+        import httpx
+
+        try:
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.get(
+                    f"{self.cortex_url}/v1/experience/status",
+                    params={"idempotency_key": idempotency_key, "scope": scope},
+                    headers=self._headers(),
+                )
+                if resp.status_code != 200:
+                    return {"found": False}
+                body = resp.json()
+                return body if isinstance(body, dict) else {"found": False}
+        except Exception:  # noqa: BLE001
+            return {"found": False}
+
+    async def experience_exists(self, idempotency_key: str, scope: str) -> bool:
+        """Did a write with this key already land?
+
+        Used before attachment blob uploads so a re-sync doesn't store a
+        fresh (orphaned) blob copy for an experience that will just replay.
+        Best-effort: any probe failure returns False (upload proceeds — the
+        experience write itself still dedups via its idempotency key).
+        """
+        status = await self.experience_status(idempotency_key, scope)
+        return bool(status.get("found"))
+
+    # -- v1 deletion propagation ---------------------------------------------
+
+    async def forget_events(
+        self, event_ids: list[str], scope: str, *, reason: str = ""
+    ) -> None:
+        """Redact specific events via `POST /v1/forget` (`memory_ids` selector).
+
+        The precise form of deletion propagation: it names exactly the events
+        to remove, so nothing else in the scope is touched. Used when a source
+        item that was ingested as its own event disappears at the source.
+        """
+        import httpx
+
+        ids = [e for e in event_ids if e]
+        if not ids:
+            return
+        body = {
+            "scope": scope,
+            "selector": {"memory_ids": ids},
+            "cascade": "redact_events",
+            "reason": reason or "source deletion reconciled by connector",
+        }
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/forget", json=body, headers=self._headers()
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "forget REJECTED by server: status=%s ids=%s body=%s",
+                    resp.status_code, ids, resp.text[:300],
+                )
+            resp.raise_for_status()
+
+    async def forget_entity(
+        self, entity: str, scope: str, *, reason: str = ""
+    ) -> None:
+        """Redact everything about a deleted source entity (`about_entity`).
+
+        Used when the source hard-deletes a whole item (a Jira issue, a
+        Notion page) and every event mentioning it must go.
+        """
+        import httpx
+
+        body = {
+            "scope": scope,
+            "selector": {"about_entity": entity},
+            "cascade": "redact_events",
+            "reason": reason or f"source deleted {entity}",
+        }
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/forget", json=body, headers=self._headers()
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "forget REJECTED by server: status=%s entity=%s body=%s",
+                    resp.status_code, entity, resp.text[:300],
+                )
+            resp.raise_for_status()
+
+    async def delete_blob(self, blob_id: str) -> bool:
+        """`DELETE /v1/blobs/{id}` — purge uploaded bytes.
+
+        Redacting the blob_ref experience removes the memory; it does not
+        remove the stored object. A source-side file deletion has to purge
+        BOTH, or the deleted file stays byte-retrievable from CortexDB.
+        Returns True when the bytes are gone (deleted now, or already
+        absent — 404 is success for a delete). Raises on anything else so
+        the caller reports a failed purge instead of assuming one.
+        """
+        import httpx
+
+        if not blob_id:
+            return False
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.delete(
+                f"{self.cortex_url}/v1/blobs/{blob_id}", headers=self._headers()
+            )
+            if resp.status_code == 404:
+                return True
+            if resp.status_code >= 400:
+                logger.error(
+                    "blob delete REJECTED by server: status=%s blob_id=%s body=%s",
+                    resp.status_code, blob_id, resp.text[:300],
+                )
+            resp.raise_for_status()
+            return True
+
     async def ingest_episode(self, episode: Episode) -> str:
         """POST `/v1/experience` with the v1 envelope.
 
-        Returns the server-side event_id on success.
+        Blob-carrying episodes (`episode.blob` set with bytes) first upload
+        the bytes to `/v1/blobs` and go in as `content.kind="blob_ref"` so the
+        server's content processors extract them. A status probe on the
+        episode's idempotency key skips the upload when the experience already
+        exists (re-sync), avoiding orphaned duplicate blobs. Bytes-less blob
+        episodes (oversized skips) fall back to their text content.
+
+        Returns the server-side event_id on success ("exists" when the
+        status probe short-circuited a replay).
         """
         import httpx
 
@@ -452,13 +689,24 @@ class CortexConnector(ABC):
                 "explicitly. The actor must match the token's `sub` claim."
             )
 
-        envelope = self._envelope(episode)
-        headers = {
-            "Authorization": f"Bearer {self.cortex_api_key}",
-            "X-Cortex-Actor": self.actor,
-            "Content-Type": "application/json",
-            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
-        }
+        scope = self._resolve_scope(episode)
+        key = self._idem_key(episode, scope)
+        blob_id: str | None = None
+        if episode.blob is not None and episode.blob.data:
+            if await self.experience_exists(key, scope):
+                logger.info(
+                    "attachment already ingested (idempotency_key=%s); "
+                    "skipping blob upload",
+                    key,
+                )
+                return "exists"
+            blob_id = await self.upload_blob(
+                episode.blob.data, episode.blob.mime_type
+            )
+            self._uploaded_blobs[key] = blob_id
+
+        envelope = self._envelope(episode, blob_id=blob_id)
+        headers = self._headers()
 
         async with httpx.AsyncClient(timeout=30.0) as client:
             resp = await client.post(
@@ -480,4 +728,12 @@ class CortexConnector(ABC):
                 )
             resp.raise_for_status()
             body = resp.json()
-        return body.get("event_id") or body.get("id") or episode.id
+        event_id = body.get("event_id") or body.get("id")
+        if event_id:
+            # Remembered so a connector that reconciles source deletions can
+            # name this exact event later. The server's idempotency records
+            # expire after 24h, so a status probe cannot resolve the id of
+            # something ingested days ago — which is exactly when a source
+            # deletion shows up.
+            self._ingested_events[key] = event_id
+        return event_id or episode.id

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.11}/cortexdb_connectors/cli.py

@@ -80,7 +80,12 @@ CONNECTORS: dict[str, dict[str, Any]] = {
             ("jira_email", "JIRA_EMAIL"),
             ("jira_api_token", "JIRA_API_TOKEN"),
         ],
-        "options": [("project_keys", "JIRA_PROJECT_KEYS", "list")],
+        "options": [
+            ("project_keys", "JIRA_PROJECT_KEYS", "list"),
+            ("reconcile_deletions", "JIRA_RECONCILE_DELETIONS", "bool"),
+            ("reconcile_interval_seconds", "JIRA_RECONCILE_INTERVAL_SECONDS", "int"),
+            ("ledger_path", "JIRA_LEDGER_PATH", "str"),
+        ],
     },
     "freshdesk": {
         "import": "cortexdb_connectors.freshdesk:FreshdeskConnector",
@@ -260,6 +265,7 @@ def _load_cortex_creds(args: argparse.Namespace) -> dict[str, str]:
 
 _OPTION_PARSERS: dict[str, Callable[[str], Any]] = {
     "str": lambda v: v,
+    "int": int,
     "list": lambda v: [x.strip() for x in v.split(",") if x.strip()],
     "list-int": lambda v: [int(x.strip()) for x in v.split(",") if x.strip()],
     "bool": lambda v: v.lower() in ("1", "true", "yes", "on"),