cortexdb-connectors 0.2.6__tar.gz → 0.2.7__tar.gz

{cortexdb_connectors-0.2.6 → cortexdb_connectors-0.2.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.6
+Version: 0.2.7
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.6 → cortexdb_connectors-0.2.7}/cortexdb_connectors/__init__.py

@@ -8,7 +8,7 @@ poll loop; or `from cortexdb_connectors.slack import SlackConnector`
 to drive it programmatically.
 """
 
-__version__ = "0.2.6"
+__version__ = "0.2.7"
 
 from cortexdb_connectors.base import (
     Actor,

{cortexdb_connectors-0.2.6 → cortexdb_connectors-0.2.7}/cortexdb_connectors/base.py

@@ -158,6 +158,13 @@ class Episode:
 class SyncResult:
     episodes_fetched: int = 0
     episodes_ingested: int = 0
+    # Items the connector's content policy deliberately dropped (`normalize`
+    # returned None: empty/attachment-only text, emoji-only, noise subtypes).
+    # Counted separately so operators can tell "filtered by design" from
+    # "failed"; skipped items ADVANCE the cursor like successes — a skip is
+    # a decision, not an error, and treating it as one would stall the sync
+    # at the first attachment-only message.
+    episodes_skipped: int = 0
     errors: list[str] = field(default_factory=list)
     duration_seconds: float = 0.0
     # Watermarks for cursor-safe syncing: the latest source timestamp we
@@ -237,7 +244,13 @@ class CortexConnector(ABC):
         ...
 
     @abstractmethod
-    def normalize(self, raw: RawEvent) -> Episode:
+    def normalize(self, raw: RawEvent) -> Episode | None:
+        """Convert a RawEvent into an Episode, or return None to SKIP it.
+
+        None means the item is dropped by content policy (nothing sent to
+        the server); the sync loop counts it in `episodes_skipped` and the
+        cursor advances past it.
+        """
         ...
 
     @abstractmethod
@@ -257,6 +270,13 @@ class CortexConnector(ABC):
         for raw in raw_events:
             try:
                 episode = self.normalize(raw)
+                if episode is None:
+                    # Content-policy skip: advance the cursor exactly like a
+                    # success so the next incremental run never re-fetches it.
+                    result.episodes_skipped += 1
+                    if result.latest_success is None or raw.timestamp > result.latest_success:
+                        result.latest_success = raw.timestamp
+                    continue
                 episode.visibility = self.map_permissions(raw)
                 await self.ingest_episode(episode)
                 result.episodes_ingested += 1