cortexdb-connectors 0.2.5__tar.gz → 0.2.7__tar.gz

{cortexdb_connectors-0.2.5 → cortexdb_connectors-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.5
+Version: 0.2.7
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.5 → cortexdb_connectors-0.2.7}/cortexdb_connectors/__init__.py

@@ -1,40 +1,40 @@
-"""cortexdb_connectors — data connector SDK for the CortexDB v1 API.
-
-Each connector subclass implements `fetch_events`, `normalize`, and
-`map_permissions`; the base class POSTs every normalised Episode to
-`/v1/experience` with the required Authorization + X-Cortex-Actor
-headers. Use the `cortexdb-sync` CLI to run a connector once or in a
-poll loop; or `from cortexdb_connectors.slack import SlackConnector`
-to drive it programmatically.
-"""
-
-__version__ = "0.2.1"
-
-from cortexdb_connectors.base import (
-    Actor,
-    CortexConnector,
-    EntityRef,
-    Episode,
-    EpisodeType,
-    RawEvent,
-    Source,
-    SyncResult,
-    SyncState,
-    Visibility,
-    VisibilityLevel,
-)
-
-__all__ = [
-    "__version__",
-    "Actor",
-    "CortexConnector",
-    "EntityRef",
-    "Episode",
-    "EpisodeType",
-    "RawEvent",
-    "Source",
-    "SyncResult",
-    "SyncState",
-    "Visibility",
-    "VisibilityLevel",
-]
+"""cortexdb_connectors — data connector SDK for the CortexDB v1 API.
+
+Each connector subclass implements `fetch_events`, `normalize`, and
+`map_permissions`; the base class POSTs every normalised Episode to
+`/v1/experience` with the required Authorization + X-Cortex-Actor
+headers. Use the `cortexdb-sync` CLI to run a connector once or in a
+poll loop; or `from cortexdb_connectors.slack import SlackConnector`
+to drive it programmatically.
+"""
+
+__version__ = "0.2.7"
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    SyncResult,
+    SyncState,
+    Visibility,
+    VisibilityLevel,
+)
+
+__all__ = [
+    "__version__",
+    "Actor",
+    "CortexConnector",
+    "EntityRef",
+    "Episode",
+    "EpisodeType",
+    "RawEvent",
+    "Source",
+    "SyncResult",
+    "SyncState",
+    "Visibility",
+    "VisibilityLevel",
+]

{cortexdb_connectors-0.2.5 → cortexdb_connectors-0.2.7}/cortexdb_connectors/base.py

@@ -158,6 +158,13 @@ class Episode:
 class SyncResult:
     episodes_fetched: int = 0
     episodes_ingested: int = 0
+    # Items the connector's content policy deliberately dropped (`normalize`
+    # returned None: empty/attachment-only text, emoji-only, noise subtypes).
+    # Counted separately so operators can tell "filtered by design" from
+    # "failed"; skipped items ADVANCE the cursor like successes — a skip is
+    # a decision, not an error, and treating it as one would stall the sync
+    # at the first attachment-only message.
+    episodes_skipped: int = 0
     errors: list[str] = field(default_factory=list)
     duration_seconds: float = 0.0
     # Watermarks for cursor-safe syncing: the latest source timestamp we
@@ -237,7 +244,13 @@ class CortexConnector(ABC):
         ...
 
     @abstractmethod
-    def normalize(self, raw: RawEvent) -> Episode:
+    def normalize(self, raw: RawEvent) -> Episode | None:
+        """Convert a RawEvent into an Episode, or return None to SKIP it.
+
+        None means the item is dropped by content policy (nothing sent to
+        the server); the sync loop counts it in `episodes_skipped` and the
+        cursor advances past it.
+        """
         ...
 
     @abstractmethod
@@ -257,6 +270,13 @@ class CortexConnector(ABC):
         for raw in raw_events:
             try:
                 episode = self.normalize(raw)
+                if episode is None:
+                    # Content-policy skip: advance the cursor exactly like a
+                    # success so the next incremental run never re-fetches it.
+                    result.episodes_skipped += 1
+                    if result.latest_success is None or raw.timestamp > result.latest_success:
+                        result.latest_success = raw.timestamp
+                    continue
                 episode.visibility = self.map_permissions(raw)
                 await self.ingest_episode(episode)
                 result.episodes_ingested += 1
@@ -322,19 +342,28 @@ class CortexConnector(ABC):
             "role": "user" if modality == "conversation" else "system",
             "text": self._guard_content(episode.content, episode),
         }
+        # The v1 write envelope is STRICT (deny_unknown_fields) as of v0.8.0:
+        # thread_id / meta / preceded_by are NOT envelope context fields, and
+        # sending them returns 422 INVALID_BODY (which stalls the sync cursor
+        # forever). Everything routable rides `context.labels` as key=value
+        # strings — the queryable partition recall metadata filters target.
+        # `context.preceded_by` is a list of CortexDB event ids (evt_…), not
+        # source-side parent ids, so the connector's parent_id goes to a
+        # label instead.
+        labels = list(labels)
+        if episode.thread_id:
+            labels.append(f"thread={episode.thread_id}")
+        if episode.parent_id:
+            labels.append(f"parent={episode.parent_id}")
+        if episode.metadata:
+            for k, v in episode.metadata.items():
+                if k == "modality":
+                    continue
+                labels.append(f"{k}={v}")
         context: dict[str, Any] = {
             "observed_at": episode.occurred_at.isoformat(),
             "labels": labels,
         }
-        if episode.thread_id:
-            context["thread_id"] = episode.thread_id
-        if episode.parent_id:
-            context["preceded_by"] = [episode.parent_id]
-        if episode.metadata:
-            # Forward non-reserved metadata under a single key — the v1
-            # surface doesn't reject unknown context keys but we tuck
-            # everything under `meta` to keep the envelope tidy.
-            context["meta"] = {k: v for k, v in episode.metadata.items() if k != "modality"}
 
         envelope: dict[str, Any] = {
             "scope": scope,