cortexdb-connectors 0.2.10__tar.gz → 0.2.12__tar.gz

{cortexdb_connectors-0.2.10 → cortexdb_connectors-0.2.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.10
+Version: 0.2.12
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0
@@ -32,6 +32,7 @@ Provides-Extra: pagerduty
 Requires-Dist: pdpyras>=5.0; extra == "pagerduty"
 Provides-Extra: jira
 Requires-Dist: jira>=3.0; extra == "jira"
+Requires-Dist: tzdata; extra == "jira"
 Provides-Extra: confluence
 Requires-Dist: atlassian-python-api>=3.0; extra == "confluence"
 Provides-Extra: notion
@@ -62,6 +63,7 @@ Requires-Dist: slack-sdk>=3.0; extra == "all"
 Requires-Dist: pygithub>=2.0; extra == "all"
 Requires-Dist: pdpyras>=5.0; extra == "all"
 Requires-Dist: jira>=3.0; extra == "all"
+Requires-Dist: tzdata; extra == "all"
 Requires-Dist: atlassian-python-api>=3.0; extra == "all"
 Requires-Dist: notion-client>=2.0; extra == "all"
 Requires-Dist: aiohttp>=3.9; extra == "all"

{cortexdb_connectors-0.2.10 → cortexdb_connectors-0.2.12}/cortexdb_connectors/base.py

@@ -36,6 +36,11 @@ _ACTOR_LOCAL_ILLEGAL = re.compile(r"[^A-Za-z0-9_.@-]")
 # never drop content silently.
 _MAX_CONTENT_BYTES = 10 * 1024 * 1024  # 10 MB
 
+# Server-enforced ceiling on `idempotency_key` (cortex-types
+# `EnvelopeError::IdempotencyKeyTooLong`). Over it, the write is refused with
+# 422 INVALID_BODY — which stalls a sync cursor — so keys are built to fit.
+_MAX_IDEM_BYTES = 64
+
 
 # ---------------------------------------------------------------------------
 # Enums
@@ -102,7 +107,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.10"
+    connector_version: str = "0.2.12"
 
 
 @dataclass(frozen=True)
@@ -258,6 +263,14 @@ class CortexConnector(ABC):
             or self.scope
             or self.actor
         )
+        # idempotency_key → server blob_id for every blob uploaded this run,
+        # and → event_id for everything ingested this run. A connector that
+        # reconciles source deletions persists these so a LATER run can name
+        # the event and purge the bytes of an item deleted at the source;
+        # neither id is discoverable after the fact (the blob id is only
+        # visible at upload time, and idempotency records expire in 24h).
+        self._uploaded_blobs: dict[str, str] = {}
+        self._ingested_events: dict[str, str] = {}
 
     def bind(self, *, actor: str | None = None, scope_template: str | None = None) -> "CortexConnector":
         """Set v1-only fields after construction. Returns self for chaining."""
@@ -313,8 +326,12 @@ class CortexConnector(ABC):
                 if result.latest_success is None or raw.timestamp > result.latest_success:
                     result.latest_success = raw.timestamp
             except Exception as exc:  # noqa: BLE001
+                # Include the exception TYPE: httpx timeout/transport errors
+                # stringify to "" and produced blank error lines that hid the
+                # real failure from operators.
                 result.errors.append(
-                    f"Error processing event {raw.external_id}: {exc}"
+                    f"Error processing event {raw.external_id}: "
+                    f"{type(exc).__name__}: {exc}"
                 )
                 if result.earliest_failure is None or raw.timestamp < result.earliest_failure:
                     result.earliest_failure = raw.timestamp
@@ -380,6 +397,15 @@ class CortexConnector(ABC):
             labels.append(f"source:{episode.source.system}")
         if episode.visibility and episode.visibility.level:
             labels.append(f"visibility:{episode.visibility.level.value}")
+            # Non-org visibility carries WHO may see it (bounded — labels are
+            # metadata, not an unbounded roster). Recall metadata filters can
+            # then scope RESTRICTED/PRIVATE items to their principals.
+            if episode.visibility.level in (
+                VisibilityLevel.RESTRICTED,
+                VisibilityLevel.PRIVATE,
+            ):
+                for principal in episode.visibility.allowed_principals[:16]:
+                    labels.append(f"visible_to={principal}")
         for ent in episode.entities:
             labels.append(f"{ent.entity_type}:{ent.entity_id}")
 
@@ -424,10 +450,7 @@ class CortexConnector(ABC):
             "modality": modality,
             "content": content,
             "context": context,
-            "idempotency_key": (
-                episode.idempotency_key
-                or self._fallback_idem(episode, scope)
-            ),
+            "idempotency_key": self._idem_key(episode, scope),
         }
         if episode.actor and episode.actor.id:
             # Source actor ids are not guaranteed bare: Jira Cloud accountIds
@@ -451,12 +474,58 @@ class CortexConnector(ABC):
         seed = f"{scope}|{src}|{episode.id}|{episode.content[:512]}"
         return f"conn:{src}:{hashlib.sha256(seed.encode('utf-8')).hexdigest()[:24]}"
 
+    def _idem_key(self, episode: Episode, scope: str) -> str:
+        """The key actually sent — the connector's key QUALIFIED BY SCOPE.
+
+        C4: the server namespaces idempotency records by the scope ROOT (the
+        first path segment) plus the caller, while the envelope it hashes
+        carries the FULL scope. So the same source item synced into two
+        sibling scopes under one root re-used one record with a different
+        body — `409 IDEMPOTENCY_CONFLICT`, and every event after the first
+        scope was silently dropped. Qualifying the key with the resolved
+        scope makes it mean what it always claimed: "this item, in this
+        scope".
+
+        The scope rides as an 8-hex-char digest because the wire field is
+        capped at `_MAX_IDEM_BYTES` and scope paths are unbounded; a key that
+        can't fit the suffix is replaced by a digest of (scope, key) rather
+        than truncated, since a truncated key would collide across items.
+
+        Same-scope semantics are unchanged: one item still maps to exactly
+        one key, so re-syncs replay and poll/webhook paths (which resolve to
+        the same scope) still share one key. `_fallback_idem` keys already
+        hash the scope in, so they pass through untouched.
+        """
+        key = episode.idempotency_key
+        if not key:
+            return self._fallback_idem(episode, scope)
+        digest = hashlib.sha256(scope.encode("utf-8")).hexdigest()
+        suffix = f"@{digest[:8]}"
+        if len(key) + len(suffix) <= _MAX_IDEM_BYTES:
+            return f"{key}{suffix}"
+        seed = f"{scope}|{key}".encode("utf-8")
+        return f"cx:{hashlib.sha256(seed).hexdigest()[:40]}"
+
     @staticmethod
     def _guard_content(text: str, episode: Episode) -> str:
         """Hard 10 MB ceiling on content. Real items never hit this; if one
         does we truncate on a UTF-8 boundary and log LOUDLY (never silent), so
-        an operator can find out what was too big and why."""
-        raw = text.encode("utf-8")
+        an operator can find out what was too big and why.
+
+        Also sanitizes text that cannot be UTF-8-encoded (lone surrogates —
+        e.g. from a source emitting broken UTF-16 pairs): unencodable code
+        points are replaced instead of raising a client-side
+        ``UnicodeEncodeError`` that fails the whole item (tl;dv T10)."""
+        try:
+            raw = text.encode("utf-8")
+        except UnicodeEncodeError:
+            logger.warning(
+                "content for idempotency_key=%s contained unencodable code "
+                "points (lone surrogates?) — replaced",
+                episode.idempotency_key,
+            )
+            text = text.encode("utf-8", errors="replace").decode("utf-8")
+            raw = text.encode("utf-8")
         if len(raw) <= _MAX_CONTENT_BYTES:
             return text
         capped = raw[:_MAX_CONTENT_BYTES].decode("utf-8", errors="ignore")
@@ -496,13 +565,14 @@ class CortexConnector(ABC):
             resp.raise_for_status()
             return resp.json()["blob_id"]
 
-    async def experience_exists(self, idempotency_key: str, scope: str) -> bool:
-        """Probe `GET /v1/experience/status` for a prior write with this key.
+    async def experience_status(
+        self, idempotency_key: str, scope: str
+    ) -> dict[str, Any]:
+        """`GET /v1/experience/status` for a prior write with this key.
 
-        Used before attachment blob uploads so a re-sync doesn't store a
-        fresh (orphaned) blob copy for an experience that will just replay.
-        Best-effort: any probe failure returns False (upload proceeds — the
-        experience write itself still dedups via its idempotency key).
+        Returns the server's status document (`{found, event_id, status,
+        …}`), or `{"found": False}` when the probe fails — callers treat an
+        unreachable probe as "unknown", never as proof of absence.
         """
         import httpx
 
@@ -514,10 +584,110 @@ class CortexConnector(ABC):
                     headers=self._headers(),
                 )
                 if resp.status_code != 200:
-                    return False
-                return bool(resp.json().get("found"))
+                    return {"found": False}
+                body = resp.json()
+                return body if isinstance(body, dict) else {"found": False}
         except Exception:  # noqa: BLE001
+            return {"found": False}
+
+    async def experience_exists(self, idempotency_key: str, scope: str) -> bool:
+        """Did a write with this key already land?
+
+        Used before attachment blob uploads so a re-sync doesn't store a
+        fresh (orphaned) blob copy for an experience that will just replay.
+        Best-effort: any probe failure returns False (upload proceeds — the
+        experience write itself still dedups via its idempotency key).
+        """
+        status = await self.experience_status(idempotency_key, scope)
+        return bool(status.get("found"))
+
+    # -- v1 deletion propagation ---------------------------------------------
+
+    async def forget_events(
+        self, event_ids: list[str], scope: str, *, reason: str = ""
+    ) -> None:
+        """Redact specific events via `POST /v1/forget` (`memory_ids` selector).
+
+        The precise form of deletion propagation: it names exactly the events
+        to remove, so nothing else in the scope is touched. Used when a source
+        item that was ingested as its own event disappears at the source.
+        """
+        import httpx
+
+        ids = [e for e in event_ids if e]
+        if not ids:
+            return
+        body = {
+            "scope": scope,
+            "selector": {"memory_ids": ids},
+            "cascade": "redact_events",
+            "reason": reason or "source deletion reconciled by connector",
+        }
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/forget", json=body, headers=self._headers()
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "forget REJECTED by server: status=%s ids=%s body=%s",
+                    resp.status_code, ids, resp.text[:300],
+                )
+            resp.raise_for_status()
+
+    async def forget_entity(
+        self, entity: str, scope: str, *, reason: str = ""
+    ) -> None:
+        """Redact everything about a deleted source entity (`about_entity`).
+
+        Used when the source hard-deletes a whole item (a Jira issue, a
+        Notion page) and every event mentioning it must go.
+        """
+        import httpx
+
+        body = {
+            "scope": scope,
+            "selector": {"about_entity": entity},
+            "cascade": "redact_events",
+            "reason": reason or f"source deleted {entity}",
+        }
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/forget", json=body, headers=self._headers()
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "forget REJECTED by server: status=%s entity=%s body=%s",
+                    resp.status_code, entity, resp.text[:300],
+                )
+            resp.raise_for_status()
+
+    async def delete_blob(self, blob_id: str) -> bool:
+        """`DELETE /v1/blobs/{id}` — purge uploaded bytes.
+
+        Redacting the blob_ref experience removes the memory; it does not
+        remove the stored object. A source-side file deletion has to purge
+        BOTH, or the deleted file stays byte-retrievable from CortexDB.
+        Returns True when the bytes are gone (deleted now, or already
+        absent — 404 is success for a delete). Raises on anything else so
+        the caller reports a failed purge instead of assuming one.
+        """
+        import httpx
+
+        if not blob_id:
             return False
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            resp = await client.delete(
+                f"{self.cortex_url}/v1/blobs/{blob_id}", headers=self._headers()
+            )
+            if resp.status_code == 404:
+                return True
+            if resp.status_code >= 400:
+                logger.error(
+                    "blob delete REJECTED by server: status=%s blob_id=%s body=%s",
+                    resp.status_code, blob_id, resp.text[:300],
+                )
+            resp.raise_for_status()
+            return True
 
     async def ingest_episode(self, episode: Episode) -> str:
         """POST `/v1/experience` with the v1 envelope.
@@ -546,10 +716,10 @@ class CortexConnector(ABC):
                 "explicitly. The actor must match the token's `sub` claim."
             )
 
+        scope = self._resolve_scope(episode)
+        key = self._idem_key(episode, scope)
         blob_id: str | None = None
         if episode.blob is not None and episode.blob.data:
-            scope = self._resolve_scope(episode)
-            key = episode.idempotency_key or self._fallback_idem(episode, scope)
             if await self.experience_exists(key, scope):
                 logger.info(
                     "attachment already ingested (idempotency_key=%s); "
@@ -560,6 +730,7 @@ class CortexConnector(ABC):
             blob_id = await self.upload_blob(
                 episode.blob.data, episode.blob.mime_type
             )
+            self._uploaded_blobs[key] = blob_id
 
         envelope = self._envelope(episode, blob_id=blob_id)
         headers = self._headers()
@@ -584,4 +755,12 @@ class CortexConnector(ABC):
                 )
             resp.raise_for_status()
             body = resp.json()
-        return body.get("event_id") or body.get("id") or episode.id
+        event_id = body.get("event_id") or body.get("id")
+        if event_id:
+            # Remembered so a connector that reconciles source deletions can
+            # name this exact event later. The server's idempotency records
+            # expire after 24h, so a status probe cannot resolve the id of
+            # something ingested days ago — which is exactly when a source
+            # deletion shows up.
+            self._ingested_events[key] = event_id
+        return event_id or episode.id

{cortexdb_connectors-0.2.10 → cortexdb_connectors-0.2.12}/cortexdb_connectors/cli.py

@@ -80,7 +80,12 @@ CONNECTORS: dict[str, dict[str, Any]] = {
             ("jira_email", "JIRA_EMAIL"),
             ("jira_api_token", "JIRA_API_TOKEN"),
         ],
-        "options": [("project_keys", "JIRA_PROJECT_KEYS", "list")],
+        "options": [
+            ("project_keys", "JIRA_PROJECT_KEYS", "list"),
+            ("reconcile_deletions", "JIRA_RECONCILE_DELETIONS", "bool"),
+            ("reconcile_interval_seconds", "JIRA_RECONCILE_INTERVAL_SECONDS", "int"),
+            ("ledger_path", "JIRA_LEDGER_PATH", "str"),
+        ],
     },
     "freshdesk": {
         "import": "cortexdb_connectors.freshdesk:FreshdeskConnector",
@@ -260,6 +265,7 @@ def _load_cortex_creds(args: argparse.Namespace) -> dict[str, str]:
 
 _OPTION_PARSERS: dict[str, Callable[[str], Any]] = {
     "str": lambda v: v,
+    "int": int,
     "list": lambda v: [x.strip() for x in v.split(",") if x.strip()],
     "list-int": lambda v: [int(x.strip()) for x in v.split(",") if x.strip()],
     "bool": lambda v: v.lower() in ("1", "true", "yes", "on"),
@@ -405,13 +411,52 @@ def _cmd_sync(args: argparse.Namespace) -> None:
     # would be silently skipped on the next incremental run. Hold the cursor
     # just before the earliest failure so it (and everything after) is
     # re-attempted; on a clean run advance to the latest item we ingested.
+    #
+    # T15 (dead-letter cap): "hold forever" turns one PERMANENTLY-failing
+    # item into a head-of-line block — the cursor can never pass it, and
+    # every later item is refetched every cycle for eternity. Track how many
+    # consecutive cycles the cursor has been pinned at the same failure; at
+    # _DEAD_LETTER_AFTER, advance past it with a LOUD error naming what was
+    # abandoned, so the sync heals and the operator has a precise artifact
+    # to chase.
+    _DEAD_LETTER_AFTER = 5
+    state = state_store.get(args.connector, args.scope)
     if result.earliest_failure is not None:
-        new_cursor = result.earliest_failure - timedelta(seconds=1)
+        pinned_at = result.earliest_failure.isoformat()
+        streak = int(state.extra.get("failure_streak", 0)) if (
+            state.extra.get("failure_pinned_at") == pinned_at
+        ) else 0
+        streak += 1
+        if streak >= _DEAD_LETTER_AFTER:
+            logger.error(
+                "DEAD-LETTER: item(s) at %s failed %d consecutive cycles — "
+                "advancing the cursor past them. They will NOT be retried; "
+                "see the per-item errors above for what was abandoned.",
+                pinned_at,
+                streak,
+            )
+            new_cursor = (
+                result.latest_success
+                if result.latest_success is not None
+                and result.latest_success > result.earliest_failure
+                else result.earliest_failure + timedelta(seconds=1)
+            )
+            state.extra.pop("failure_pinned_at", None)
+            state.extra.pop("failure_streak", None)
+        else:
+            new_cursor = result.earliest_failure - timedelta(seconds=1)
+            state.extra["failure_pinned_at"] = pinned_at
+            state.extra["failure_streak"] = streak
     elif result.latest_success is not None:
         new_cursor = result.latest_success
+        state.extra.pop("failure_pinned_at", None)
+        state.extra.pop("failure_streak", None)
     else:
         new_cursor = None  # empty window — nothing to miss; default to now()
-    state_store.update_last_synced(args.connector, args.scope, timestamp=new_cursor)
+        state.extra.pop("failure_pinned_at", None)
+        state.extra.pop("failure_streak", None)
+    state.last_synced_at = new_cursor or datetime.now(timezone.utc)
+    state_store.save(state)
 
     logger.info(
         "sync done   fetched=%d ingested=%d skipped=%d errors=%d duration=%.1fs",
@@ -477,6 +522,12 @@ def _cmd_watch(args: argparse.Namespace) -> None:
         except SystemExit as exc:
             # Don't exit the watch loop on per-cycle errors.
             logger.error("cycle failed: %s", exc)
+        except Exception:  # noqa: BLE001
+            # T6: an uncaught exception from fetch_events (e.g. a transient
+            # 429/5xx from the source API surviving the connector's retries)
+            # used to TERMINATE the watch daemon. A watcher's whole job is to
+            # outlive transient failures — log and take the next cycle.
+            logger.exception("cycle failed with unexpected error; continuing")
         logger.info("sleeping %ds before next cycle", args.interval)
         time.sleep(args.interval)