cortexdb-connectors 0.2.8__tar.gz → 0.2.10__tar.gz

{cortexdb_connectors-0.2.8 → cortexdb_connectors-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.8
+Version: 0.2.10
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0
@@ -156,7 +156,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.8 → cortexdb_connectors-0.2.10}/README.md

@@ -78,7 +78,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.8 → cortexdb_connectors-0.2.10}/cortexdb_connectors/base.py

@@ -102,7 +102,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.5"
+    connector_version: str = "0.2.10"
 
 
 @dataclass(frozen=True)
@@ -126,6 +126,25 @@ class Visibility:
     allowed_principals: tuple[str, ...] = ()
 
 
+@dataclass
+class Attachment:
+    """A binary attachment destined for `POST /v1/blobs` + a `blob_ref`
+    experience, so CortexDB's multimodal pipeline (Tika/vision/ASR content
+    processors) extracts and indexes it.
+
+    `data` carries the raw bytes (downloaded by the connector's
+    `fetch_events`); when `data` is None the attachment is metadata-only
+    (e.g. oversized skip) and only its filename line survives in text.
+    """
+
+    filename: str
+    mime_type: str
+    size_bytes: int = 0
+    data: bytes | None = None
+    source_url: str | None = None
+    attachment_id: str = ""
+
+
 @dataclass
 class RawEvent:
     source: str
@@ -158,6 +177,11 @@ class Episode:
     parent_id: str | None = None
     idempotency_key: str | None = None
     ttl_seconds: int | None = None
+    # When set, this episode is a BINARY attachment: `ingest_episode` uploads
+    # `blob.data` to `POST /v1/blobs` and sends the experience with
+    # `content.kind="blob_ref"` (the multimodal path) instead of text. The
+    # `content` field then serves only as the no-bytes fallback text.
+    blob: Attachment | None = None
 
 
 @dataclass
@@ -325,10 +349,26 @@ class CortexConnector(ABC):
 
         for key, value in replacements.items():
             tpl = tpl.replace("{" + key + "}", value or "")
+        # C6d: an unresolved placeholder used to pass through literally,
+        # producing a silently malformed scope (`.../issue:{entity.issue}`)
+        # that the server happily stores as a real path. Fail loudly instead —
+        # the sync loop records the error per event and holds the cursor.
+        leftover = re.findall(r"\{[^{}]+\}", tpl)
+        if leftover:
+            raise ValueError(
+                f"scope template resolved to {tpl!r} with unresolved "
+                f"placeholder(s) {', '.join(leftover)} — this episode does "
+                "not carry the referenced entity/field."
+            )
         return tpl
 
-    def _envelope(self, episode: Episode) -> dict[str, Any]:
-        """Build the v1 ExperienceItem JSON for this episode."""
+    def _envelope(self, episode: Episode, blob_id: str | None = None) -> dict[str, Any]:
+        """Build the v1 ExperienceItem JSON for this episode.
+
+        With `blob_id` (an id returned by `upload_blob`) the content becomes
+        `kind="blob_ref"` — the server fetches the stored bytes and runs the
+        per-modality content processor on them. Otherwise plain text.
+        """
         scope = self._resolve_scope(episode)
         modality = (
             episode.metadata.get("modality")
@@ -343,11 +383,19 @@ class CortexConnector(ABC):
         for ent in episode.entities:
             labels.append(f"{ent.entity_type}:{ent.entity_id}")
 
-        content: dict[str, Any] = {
-            "kind": "message",
-            "role": "user" if modality == "conversation" else "system",
-            "text": self._guard_content(episode.content, episode),
-        }
+        if blob_id is not None:
+            content: dict[str, Any] = {"kind": "blob_ref", "blob_id": blob_id}
+            # Filename/MIME ride labels — blob_ref content carries no text,
+            # and labels are the queryable metadata surface.
+            if episode.blob is not None:
+                labels.append(f"filename={episode.blob.filename}")
+                labels.append(f"mime={episode.blob.mime_type}")
+        else:
+            content = {
+                "kind": "message",
+                "role": "user" if modality == "conversation" else "system",
+                "text": self._guard_content(episode.content, episode),
+            }
         # The v1 write envelope is STRICT (deny_unknown_fields) as of v0.8.0:
         # thread_id / meta / preceded_by are NOT envelope context fields, and
         # sending them returns 422 INVALID_BODY (which stalls the sync cursor
@@ -422,10 +470,67 @@ class CortexConnector(ABC):
         )
         return capped
 
+    def _headers(self, content_type: str = "application/json") -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.cortex_api_key}",
+            "X-Cortex-Actor": self.actor,
+            "Content-Type": content_type,
+            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
+        }
+
+    async def upload_blob(self, data: bytes, mime_type: str) -> str:
+        """POST raw bytes to `/v1/blobs`; returns the server `blob_id`."""
+        import httpx
+
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/blobs",
+                content=data,
+                headers=self._headers(mime_type or "application/octet-stream"),
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "blob upload REJECTED: status=%s bytes=%d mime=%s body=%s",
+                    resp.status_code, len(data), mime_type, resp.text[:300],
+                )
+            resp.raise_for_status()
+            return resp.json()["blob_id"]
+
+    async def experience_exists(self, idempotency_key: str, scope: str) -> bool:
+        """Probe `GET /v1/experience/status` for a prior write with this key.
+
+        Used before attachment blob uploads so a re-sync doesn't store a
+        fresh (orphaned) blob copy for an experience that will just replay.
+        Best-effort: any probe failure returns False (upload proceeds — the
+        experience write itself still dedups via its idempotency key).
+        """
+        import httpx
+
+        try:
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.get(
+                    f"{self.cortex_url}/v1/experience/status",
+                    params={"idempotency_key": idempotency_key, "scope": scope},
+                    headers=self._headers(),
+                )
+                if resp.status_code != 200:
+                    return False
+                return bool(resp.json().get("found"))
+        except Exception:  # noqa: BLE001
+            return False
+
     async def ingest_episode(self, episode: Episode) -> str:
         """POST `/v1/experience` with the v1 envelope.
 
-        Returns the server-side event_id on success.
+        Blob-carrying episodes (`episode.blob` set with bytes) first upload
+        the bytes to `/v1/blobs` and go in as `content.kind="blob_ref"` so the
+        server's content processors extract them. A status probe on the
+        episode's idempotency key skips the upload when the experience already
+        exists (re-sync), avoiding orphaned duplicate blobs. Bytes-less blob
+        episodes (oversized skips) fall back to their text content.
+
+        Returns the server-side event_id on success ("exists" when the
+        status probe short-circuited a replay).
         """
         import httpx
 
@@ -441,13 +546,23 @@ class CortexConnector(ABC):
                 "explicitly. The actor must match the token's `sub` claim."
             )
 
-        envelope = self._envelope(episode)
-        headers = {
-            "Authorization": f"Bearer {self.cortex_api_key}",
-            "X-Cortex-Actor": self.actor,
-            "Content-Type": "application/json",
-            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
-        }
+        blob_id: str | None = None
+        if episode.blob is not None and episode.blob.data:
+            scope = self._resolve_scope(episode)
+            key = episode.idempotency_key or self._fallback_idem(episode, scope)
+            if await self.experience_exists(key, scope):
+                logger.info(
+                    "attachment already ingested (idempotency_key=%s); "
+                    "skipping blob upload",
+                    key,
+                )
+                return "exists"
+            blob_id = await self.upload_blob(
+                episode.blob.data, episode.blob.mime_type
+            )
+
+        envelope = self._envelope(episode, blob_id=blob_id)
+        headers = self._headers()
 
         async with httpx.AsyncClient(timeout=30.0) as client:
             resp = await client.post(

{cortexdb_connectors-0.2.8 → cortexdb_connectors-0.2.10}/cortexdb_connectors/cli.py

@@ -432,16 +432,41 @@ def _cmd_sync(args: argparse.Namespace) -> None:
 
 def _cmd_serve(args: argparse.Namespace) -> None:
     """Run the webhook receiver for real-time create/update/delete."""
-    from cortexdb_connectors.webhooks import WebhookServer, build_handlers
+    # C3: this used to import a `build_handlers` that never existed —
+    # `cortexdb-sync serve` ImportError'd on startup, killing every
+    # connector's webhook mode. WebhookServer wires its own handlers from
+    # env secrets; the CLI only supplies CortexDB creds + bind address.
+    import uvicorn
+
+    from cortexdb_connectors.webhooks import WebhookServer
 
     creds = _load_cortex_creds(args)
-    handlers = build_handlers(creds)
-    if not handlers:
+    secret_envs = [
+        "SLACK_SIGNING_SECRET",
+        "GITHUB_WEBHOOK_SECRET",
+        "JIRA_WEBHOOK_SECRET",
+        "FRESHDESK_WEBHOOK_SECRET",
+        "TLDV_WEBHOOK_SECRET",
+    ]
+    if not any(os.environ.get(v) for v in secret_envs):
         logger.warning(
-            "no webhook handlers configured — set per-source signing secrets "
-            "(e.g. SLACK_SIGNING_SECRET) so the receiver can verify deliveries."
+            "no webhook signing secrets configured — every delivery will be "
+            "rejected as unsigned. Set one or more of: %s",
+            ", ".join(secret_envs),
         )
-    WebhookServer(handlers, host=args.host, port=args.port).serve_forever()
+    server = WebhookServer(
+        cortexdb_url=creds["api_url"] or None,
+        cortexdb_api_key=creds["api_key"] or None,
+        scope=creds["scope_template"] or "user:default",
+        actor=creds["actor"] or None,
+    )
+    logger.info(
+        "webhook receiver listening on %s:%d (routes: /webhooks/{slack,github,"
+        "jira,freshdesk,tldv}/events, /health)",
+        args.host,
+        args.port,
+    )
+    uvicorn.run(server.build_app(), host=args.host, port=args.port, log_level="info")
 
 
 def _cmd_watch(args: argparse.Namespace) -> None: