cortexdb-connectors 0.2.9__tar.gz → 0.2.10__tar.gz

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.9
+Version: 0.2.10
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0
@@ -156,7 +156,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/README.md

@@ -78,7 +78,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/cortexdb_connectors/base.py

@@ -102,7 +102,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.9"
+    connector_version: str = "0.2.10"
 
 
 @dataclass(frozen=True)
@@ -126,6 +126,25 @@ class Visibility:
     allowed_principals: tuple[str, ...] = ()
 
 
+@dataclass
+class Attachment:
+    """A binary attachment destined for `POST /v1/blobs` + a `blob_ref`
+    experience, so CortexDB's multimodal pipeline (Tika/vision/ASR content
+    processors) extracts and indexes it.
+
+    `data` carries the raw bytes (downloaded by the connector's
+    `fetch_events`); when `data` is None the attachment is metadata-only
+    (e.g. oversized skip) and only its filename line survives in text.
+    """
+
+    filename: str
+    mime_type: str
+    size_bytes: int = 0
+    data: bytes | None = None
+    source_url: str | None = None
+    attachment_id: str = ""
+
+
 @dataclass
 class RawEvent:
     source: str
@@ -158,6 +177,11 @@ class Episode:
     parent_id: str | None = None
     idempotency_key: str | None = None
     ttl_seconds: int | None = None
+    # When set, this episode is a BINARY attachment: `ingest_episode` uploads
+    # `blob.data` to `POST /v1/blobs` and sends the experience with
+    # `content.kind="blob_ref"` (the multimodal path) instead of text. The
+    # `content` field then serves only as the no-bytes fallback text.
+    blob: Attachment | None = None
 
 
 @dataclass
@@ -338,8 +362,13 @@ class CortexConnector(ABC):
             )
         return tpl
 
-    def _envelope(self, episode: Episode) -> dict[str, Any]:
-        """Build the v1 ExperienceItem JSON for this episode."""
+    def _envelope(self, episode: Episode, blob_id: str | None = None) -> dict[str, Any]:
+        """Build the v1 ExperienceItem JSON for this episode.
+
+        With `blob_id` (an id returned by `upload_blob`) the content becomes
+        `kind="blob_ref"` — the server fetches the stored bytes and runs the
+        per-modality content processor on them. Otherwise plain text.
+        """
         scope = self._resolve_scope(episode)
         modality = (
             episode.metadata.get("modality")
@@ -354,11 +383,19 @@ class CortexConnector(ABC):
         for ent in episode.entities:
             labels.append(f"{ent.entity_type}:{ent.entity_id}")
 
-        content: dict[str, Any] = {
-            "kind": "message",
-            "role": "user" if modality == "conversation" else "system",
-            "text": self._guard_content(episode.content, episode),
-        }
+        if blob_id is not None:
+            content: dict[str, Any] = {"kind": "blob_ref", "blob_id": blob_id}
+            # Filename/MIME ride labels — blob_ref content carries no text,
+            # and labels are the queryable metadata surface.
+            if episode.blob is not None:
+                labels.append(f"filename={episode.blob.filename}")
+                labels.append(f"mime={episode.blob.mime_type}")
+        else:
+            content = {
+                "kind": "message",
+                "role": "user" if modality == "conversation" else "system",
+                "text": self._guard_content(episode.content, episode),
+            }
         # The v1 write envelope is STRICT (deny_unknown_fields) as of v0.8.0:
         # thread_id / meta / preceded_by are NOT envelope context fields, and
         # sending them returns 422 INVALID_BODY (which stalls the sync cursor
@@ -433,10 +470,67 @@ class CortexConnector(ABC):
         )
         return capped
 
+    def _headers(self, content_type: str = "application/json") -> dict[str, str]:
+        return {
+            "Authorization": f"Bearer {self.cortex_api_key}",
+            "X-Cortex-Actor": self.actor,
+            "Content-Type": content_type,
+            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
+        }
+
+    async def upload_blob(self, data: bytes, mime_type: str) -> str:
+        """POST raw bytes to `/v1/blobs`; returns the server `blob_id`."""
+        import httpx
+
+        async with httpx.AsyncClient(timeout=120.0) as client:
+            resp = await client.post(
+                f"{self.cortex_url}/v1/blobs",
+                content=data,
+                headers=self._headers(mime_type or "application/octet-stream"),
+            )
+            if resp.status_code >= 400:
+                logger.error(
+                    "blob upload REJECTED: status=%s bytes=%d mime=%s body=%s",
+                    resp.status_code, len(data), mime_type, resp.text[:300],
+                )
+            resp.raise_for_status()
+            return resp.json()["blob_id"]
+
+    async def experience_exists(self, idempotency_key: str, scope: str) -> bool:
+        """Probe `GET /v1/experience/status` for a prior write with this key.
+
+        Used before attachment blob uploads so a re-sync doesn't store a
+        fresh (orphaned) blob copy for an experience that will just replay.
+        Best-effort: any probe failure returns False (upload proceeds — the
+        experience write itself still dedups via its idempotency key).
+        """
+        import httpx
+
+        try:
+            async with httpx.AsyncClient(timeout=15.0) as client:
+                resp = await client.get(
+                    f"{self.cortex_url}/v1/experience/status",
+                    params={"idempotency_key": idempotency_key, "scope": scope},
+                    headers=self._headers(),
+                )
+                if resp.status_code != 200:
+                    return False
+                return bool(resp.json().get("found"))
+        except Exception:  # noqa: BLE001
+            return False
+
     async def ingest_episode(self, episode: Episode) -> str:
         """POST `/v1/experience` with the v1 envelope.
 
-        Returns the server-side event_id on success.
+        Blob-carrying episodes (`episode.blob` set with bytes) first upload
+        the bytes to `/v1/blobs` and go in as `content.kind="blob_ref"` so the
+        server's content processors extract them. A status probe on the
+        episode's idempotency key skips the upload when the experience already
+        exists (re-sync), avoiding orphaned duplicate blobs. Bytes-less blob
+        episodes (oversized skips) fall back to their text content.
+
+        Returns the server-side event_id on success ("exists" when the
+        status probe short-circuited a replay).
         """
         import httpx
 
@@ -452,13 +546,23 @@ class CortexConnector(ABC):
                 "explicitly. The actor must match the token's `sub` claim."
             )
 
-        envelope = self._envelope(episode)
-        headers = {
-            "Authorization": f"Bearer {self.cortex_api_key}",
-            "X-Cortex-Actor": self.actor,
-            "Content-Type": "application/json",
-            "User-Agent": f"cortexdb-connectors/{Source(system='base').connector_version}",
-        }
+        blob_id: str | None = None
+        if episode.blob is not None and episode.blob.data:
+            scope = self._resolve_scope(episode)
+            key = episode.idempotency_key or self._fallback_idem(episode, scope)
+            if await self.experience_exists(key, scope):
+                logger.info(
+                    "attachment already ingested (idempotency_key=%s); "
+                    "skipping blob upload",
+                    key,
+                )
+                return "exists"
+            blob_id = await self.upload_blob(
+                episode.blob.data, episode.blob.mime_type
+            )
+
+        envelope = self._envelope(episode, blob_id=blob_id)
+        headers = self._headers()
 
         async with httpx.AsyncClient(timeout=30.0) as client:
             resp = await client.post(

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/cortexdb_connectors/jira/__init__.py

@@ -14,6 +14,7 @@ from typing import Any
 
 from cortexdb_connectors.base import (
     Actor,
+    Attachment,
     CortexConnector,
     EntityRef,
     Episode,
@@ -30,6 +31,11 @@ _SOURCE = Source(system="jira")
 
 _PAGE_SIZE = 50
 
+# Attachments above this size are listed in the issue text but their bytes are
+# NOT uploaded (matches the server's 10 MB content guard; huge videos etc.
+# would blow request limits). The skip is logged loudly, never silent.
+_MAX_ATTACHMENT_BYTES = 10 * 1024 * 1024
+
 
 class JiraConnector(CortexConnector):
     """Connector that syncs Jira issues into CortexDB.
@@ -131,15 +137,21 @@ class JiraConnector(CortexConnector):
             for issue in results:
                 fields = issue.fields
                 updated_str = getattr(fields, "updated", None) or ""
-                try:
-                    updated_at = datetime.fromisoformat(
-                        updated_str.replace("Z", "+00:00")
-                    )
-                except (ValueError, TypeError):
-                    updated_at = datetime.now(timezone.utc)
+                # Same +0000-offset normalization as attachments: on Python
+                # 3.10 the old parse silently fell back to now(), making the
+                # version-aware idempotency key unstable across syncs.
+                updated_at = self._parse_att_ts(
+                    updated_str, datetime.now(timezone.utc)
+                )
 
                 issue_data = self._serialize_issue(issue, sprint_field, points_field)
                 project_data = issue_data.get("project", {})
+                permissions = {
+                    "project_key": project_data.get("key", ""),
+                    "project_type": project_data.get(
+                        "projectTypeKey", "software"
+                    ),
+                }
 
                 raw_events.append(
                     RawEvent(
@@ -147,15 +159,70 @@ class JiraConnector(CortexConnector):
                         external_id=issue.key,
                         timestamp=updated_at,
                         data=issue_data,
-                        permissions={
-                            "project_key": project_data.get("key", ""),
-                            "project_type": project_data.get(
-                                "projectTypeKey", "software"
-                            ),
-                        },
+                        permissions=permissions,
                     )
                 )
 
+                # One RawEvent per attachment: bytes are downloaded here (the
+                # jira client session is already authenticated) and ride to
+                # `normalize` → `ingest_episode`, which uploads them to
+                # /v1/blobs and sends a blob_ref experience so the server's
+                # content processors extract them. Attachments created before
+                # `since` were carried by the sync window that saw them added
+                # (an attachment add bumps the issue's `updated`), so they are
+                # skipped without re-downloading.
+                for att in getattr(fields, "attachment", None) or []:
+                    att_created = self._parse_att_ts(
+                        getattr(att, "created", None), updated_at
+                    )
+                    if since is not None and att_created < since:
+                        continue
+                    att_size = int(getattr(att, "size", 0) or 0)
+                    att_name = getattr(att, "filename", None) or "attachment"
+                    att_mime = (
+                        getattr(att, "mimeType", None)
+                        or "application/octet-stream"
+                    )
+                    data_bytes: bytes | None = None
+                    if att_size > _MAX_ATTACHMENT_BYTES:
+                        logger.error(
+                            "jira attachment SKIPPED (too large): issue=%s "
+                            "file=%s size=%d limit=%d — filename recorded, "
+                            "bytes NOT ingested",
+                            issue.key, att_name, att_size, _MAX_ATTACHMENT_BYTES,
+                        )
+                    else:
+                        try:
+                            data_bytes = att.get()
+                        except Exception as exc:  # noqa: BLE001
+                            logger.error(
+                                "jira attachment download FAILED: issue=%s "
+                                "file=%s: %s — filename recorded, bytes NOT "
+                                "ingested",
+                                issue.key, att_name, exc,
+                            )
+                    raw_events.append(
+                        RawEvent(
+                            source="jira",
+                            external_id=f"{issue.key}:att:{att.id}",
+                            timestamp=att_created,
+                            data={
+                                "kind": "attachment",
+                                "issue_key": issue.key,
+                                "issue_summary": issue_data.get("summary", ""),
+                                "attachment_id": str(att.id),
+                                "filename": att_name,
+                                "mime_type": att_mime,
+                                "size": att_size,
+                                "author": str(
+                                    getattr(att, "author", "") or ""
+                                ),
+                                "bytes": data_bytes,
+                            },
+                            permissions=dict(permissions),
+                        )
+                    )
+
             # C5: current Jira Cloud (enhanced /search/jql) reports
             # `results.total` as the CURRENT PAGE's length (and maxResults as
             # 0), so a total-based check exits after page 1 every time and
@@ -168,8 +235,10 @@ class JiraConnector(CortexConnector):
         return raw_events
 
     def normalize(self, raw: RawEvent) -> Episode:
-        """Convert a Jira issue ``RawEvent`` into a CortexDB ``Episode``."""
+        """Convert a Jira issue or attachment ``RawEvent`` into an ``Episode``."""
         data = raw.data
+        if data.get("kind") == "attachment":
+            return self._normalize_attachment(raw)
         actor = self._extract_actor(data)
         entities = self._extract_entities(data)
 
@@ -183,6 +252,17 @@ class JiraConnector(CortexConnector):
         content = f"[{issue_key}] {summary}"
         if description:
             content += f"\n\n{description}"
+        # Cheap always-there fallback (matches Discord/Teams/Zendesk): the
+        # attachment NAMES are searchable on the issue itself even if a blob
+        # upload is skipped or fails. Bytes go via the attachment episodes.
+        atts = data.get("attachments") or []
+        if atts:
+            listing = ", ".join(
+                f"{a.get('filename', '?')} ({a.get('mimeType', '?')}, "
+                f"{a.get('size', 0)} bytes)"
+                for a in atts
+            )
+            content += f"\n\nAttachments: {listing}"
 
         return Episode(
             actor=actor,
@@ -228,8 +308,73 @@ class JiraConnector(CortexConnector):
             return Visibility(level=VisibilityLevel.RESTRICTED)
         return Visibility(level=VisibilityLevel.ORGANIZATION)
 
+    def _normalize_attachment(self, raw: RawEvent) -> Episode:
+        """Attachment RawEvent → blob-carrying Episode (`content.kind="blob_ref"`).
+
+        The idempotency key is the stable attachment id (attachments are
+        immutable in Jira), so re-syncs of an updated issue replay instead of
+        duplicating — and `ingest_episode`'s status probe skips the blob
+        re-upload entirely. Without bytes (oversize/download failure) the
+        episode degrades to its text line, so the filename is never lost.
+        """
+        data = raw.data
+        issue_key = data.get("issue_key", "?")
+        filename = data.get("filename", "attachment")
+        mime = data.get("mime_type", "application/octet-stream")
+        size = int(data.get("size", 0) or 0)
+        att_id = data.get("attachment_id", "")
+        author = data.get("author") or "unknown"
+
+        blob = Attachment(
+            filename=filename,
+            mime_type=mime,
+            size_bytes=size,
+            data=data.get("bytes"),
+            attachment_id=att_id,
+        )
+        return Episode(
+            actor=Actor(id=str(author), name=str(author)),
+            source=_SOURCE,
+            episode_type=EpisodeType.DOCUMENT,
+            occurred_at=raw.timestamp,
+            content=(
+                f"[Jira attachment] {filename} ({mime}, {size} bytes) "
+                f"on {issue_key}: {data.get('issue_summary', '')}"
+            ),
+            raw_payload={k: v for k, v in data.items() if k != "bytes"},
+            scope=self.scope,
+            namespace="jira",
+            entities=[EntityRef(entity_type="ticket", entity_id=issue_key)],
+            tags=["jira", "attachment"],
+            metadata={
+                "issue_key": issue_key,
+                "attachment_id": att_id,
+                "size_bytes": size,
+            },
+            thread_id=f"jira:{issue_key}",
+            idempotency_key=f"jira:att:{att_id}",
+            blob=blob,
+        )
+
     # -- helpers -------------------------------------------------------------
 
+    @staticmethod
+    def _parse_att_ts(value: Any, fallback: datetime) -> datetime:
+        """Parse a Jira attachment `created` timestamp; fallback = issue updated.
+
+        Jira emits `+0000`-style offsets (no colon), which Python 3.10's
+        `fromisoformat` rejects — normalize to `+00:00` so the parse works on
+        every supported interpreter instead of silently falling back.
+        """
+        if isinstance(value, str) and value:
+            v = value.replace("Z", "+00:00")
+            v = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", v)
+            try:
+                return datetime.fromisoformat(v)
+            except ValueError:
+                pass
+        return fallback
+
     @staticmethod
     def _custom_field_ids(client: Any) -> tuple[str | None, str | None]:
         """Resolve the instance's Sprint / Story-Points custom field ids.
@@ -336,6 +481,18 @@ class JiraConnector(CortexConnector):
             "story_points": (
                 getattr(fields, points_field, None) if points_field else None
             ),
+            # Metadata only (id/name/mime/size) — bytes ride their own
+            # attachment RawEvents. Feeds the issue text's "Attachments:" line.
+            "attachments": [
+                {
+                    "id": str(getattr(a, "id", "")),
+                    "filename": getattr(a, "filename", None) or "attachment",
+                    "mimeType": getattr(a, "mimeType", None)
+                    or "application/octet-stream",
+                    "size": int(getattr(a, "size", 0) or 0),
+                }
+                for a in (getattr(fields, "attachment", None) or [])
+            ],
         }
 
     @staticmethod

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/cortexdb_connectors/webhooks.py

@@ -22,6 +22,7 @@ import hmac
 import json
 import logging
 import os
+import re
 import time
 from datetime import datetime, timezone
 from typing import Any
@@ -625,10 +626,17 @@ class _HmacWebhookHandler:
 
 
 def _parse_iso(value: Any) -> datetime:
-    """Parse an ISO-8601 string (Z or offset) → aware datetime; fall back to now."""
+    """Parse an ISO-8601 string (Z or offset) → aware datetime; fall back to now.
+
+    Jira emits `+0000`-style offsets (no colon), which Python 3.10's
+    `fromisoformat` rejects — normalize to `+00:00` so poll and webhook paths
+    parse identically on every interpreter (C4 key parity depends on it).
+    """
     if isinstance(value, str) and value:
+        v = value.replace("Z", "+00:00")
+        v = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", v)
         try:
-            return datetime.fromisoformat(value.replace("Z", "+00:00"))
+            return datetime.fromisoformat(v)
         except ValueError:
             pass
     return datetime.now(timezone.utc)

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/cortexdb_connectors.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.9
+Version: 0.2.10
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0
@@ -156,7 +156,7 @@ All 16 connectors are **🟢 Free** to self-host and **🔒 Starter+** ($29/mo)
 slack            SLACK_BOT_TOKEN
 github           GITHUB_TOKEN
 gitlab           GITLAB_TOKEN
-jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN
+jira             JIRA_URL, JIRA_EMAIL, JIRA_API_TOKEN   (incl. attachments → /v1/blobs multimodal extraction, 0.2.10+)
 linear           LINEAR_API_KEY
 confluence       CONFLUENCE_URL, CONFLUENCE_EMAIL, CONFLUENCE_API_TOKEN
 notion           NOTION_TOKEN

{cortexdb_connectors-0.2.9 → cortexdb_connectors-0.2.10}/cortexdb_connectors.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ tests/test_base.py
 tests/test_connector_coverage.py
 tests/test_github.py
 tests/test_ingest_e2e.py
+tests/test_jira_attachments.py
 tests/test_jira_findings_0_2_9.py
 tests/test_slack.py
 tests/test_streaming_cursor.py