cortexdb-connectors 0.2.16__tar.gz → 0.2.18__tar.gz

{cortexdb_connectors-0.2.16 → cortexdb_connectors-0.2.18}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.16
+Version: 0.2.18
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.16 → cortexdb_connectors-0.2.18}/cortexdb_connectors/__init__.py

@@ -8,7 +8,7 @@ poll loop; or `from cortexdb_connectors.slack import SlackConnector`
 to drive it programmatically.
 """
 
-__version__ = "0.2.16"
+__version__ = "0.2.18"
 
 from cortexdb_connectors.base import (
     Actor,

{cortexdb_connectors-0.2.16 → cortexdb_connectors-0.2.18}/cortexdb_connectors/base.py

@@ -114,7 +114,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.16"
+    connector_version: str = "0.2.18"
 
 
 @dataclass(frozen=True)
@@ -196,6 +196,20 @@ class Episode:
     blob: Attachment | None = None
 
 
+@dataclass(frozen=True)
+class SyncFailure:
+    """One item that failed to ingest this cycle — carries enough identity
+    (external_id + source timestamp) for the CLI's dead-letter cursor to
+    track a PER-ITEM retry budget and to name everything it abandons (R1;
+    the old single-watermark scheme under-reported the abandoned cohort
+    and could expire a late co-window failure on the earliest item's
+    budget)."""
+
+    external_id: str
+    timestamp: datetime
+    error: str
+
+
 @dataclass
 class SyncResult:
     episodes_fetched: int = 0
@@ -214,6 +228,10 @@ class SyncResult:
     # cursor before `earliest_failure` so a failed item is never skipped.
     latest_success: datetime | None = None
     earliest_failure: datetime | None = None
+    # Per-item failure records (R1): parallel to `errors`, but structured —
+    # the dead-letter cursor keys retry budgets on these instead of the
+    # single earliest_failure watermark.
+    failures: list[SyncFailure] = field(default_factory=list)
     # True when `fetch_events` itself failed (nothing was listed, so the
     # watermarks are meaningless): the caller must HOLD the cursor — the
     # empty-window default of "advance to now()" would silently skip
@@ -362,10 +380,18 @@ class CortexConnector(ABC):
                 # Include the exception TYPE: httpx timeout/transport errors
                 # stringify to "" and produced blank error lines that hid the
                 # real failure from operators.
-                result.errors.append(
+                error_line = (
                     f"Error processing event {raw.external_id}: "
                     f"{type(exc).__name__}: {exc}"
                 )
+                result.errors.append(error_line)
+                result.failures.append(
+                    SyncFailure(
+                        external_id=raw.external_id,
+                        timestamp=raw.timestamp,
+                        error=error_line,
+                    )
+                )
                 if result.earliest_failure is None or raw.timestamp < result.earliest_failure:
                     result.earliest_failure = raw.timestamp
 

{cortexdb_connectors-0.2.16 → cortexdb_connectors-0.2.18}/cortexdb_connectors/cli.py

@@ -53,7 +53,11 @@ CONNECTORS: dict[str, dict[str, Any]] = {
     "slack": {
         "import": "cortexdb_connectors.slack:SlackConnector",
         "secrets": [("slack_bot_token", "SLACK_BOT_TOKEN")],
-        "options": [("channels", "SLACK_CHANNELS", "list")],
+        "options": [
+            ("channels", "SLACK_CHANNELS", "list"),
+            ("ingest_files", "SLACK_INGEST_FILES", "bool"),
+            ("file_types", "SLACK_FILE_TYPES", "list"),
+        ],
     },
     "github": {
         "import": "cortexdb_connectors.github:GitHubConnector",
@@ -431,54 +435,73 @@ def _cmd_sync(args: argparse.Namespace) -> None:
 
     # Cursor safety: never advance past an item that FAILED to ingest — it
     # would be silently skipped on the next incremental run. Hold the cursor
-    # just before the earliest failure so it (and everything after) is
-    # re-attempted; on a clean run advance to the latest item we ingested.
+    # just before the earliest STILL-RETRYING failure so it (and everything
+    # after) is re-attempted; on a clean run advance to the latest item we
+    # ingested.
     #
     # T15 (dead-letter cap): "hold forever" turns one PERMANENTLY-failing
-    # item into a head-of-line block — the cursor can never pass it, and
-    # every later item is refetched every cycle for eternity. Track how many
-    # consecutive cycles the cursor has been pinned at the same failure; at
-    # _DEAD_LETTER_AFTER, advance past it with a LOUD error naming what was
-    # abandoned, so the sync heals and the operator has a precise artifact
-    # to chase.
+    # item into a head-of-line block. R1 refinement: the retry budget is
+    # tracked PER ITEM (keyed by external_id + timestamp), not per window —
+    # the old single-watermark streak abandoned the entire failure cohort
+    # on the earliest item's budget and named only one timestamp in the
+    # log. Now each item gets its own _DEAD_LETTER_AFTER cycles, the
+    # cursor advances only past exhausted items, and every abandoned item
+    # is named.
     _DEAD_LETTER_AFTER = 5
     state = state_store.get(args.connector, args.scope)
-    if result.earliest_failure is not None:
-        pinned_at = result.earliest_failure.isoformat()
-        streak = int(state.extra.get("failure_streak", 0)) if (
-            state.extra.get("failure_pinned_at") == pinned_at
-        ) else 0
-        streak += 1
-        if streak >= _DEAD_LETTER_AFTER:
+    # Legacy single-watermark keys from pre-0.2.17 states.
+    state.extra.pop("failure_pinned_at", None)
+    state.extra.pop("failure_streak", None)
+    if result.failures:
+        prior = state.extra.get("failure_streaks") or {}
+        streaks: dict[str, int] = {}
+        for f in result.failures:
+            key = f"{f.timestamp.isoformat()}|{f.external_id}"
+            streaks[key] = int(prior.get(key, 0)) + 1
+        exhausted = {k for k, v in streaks.items() if v >= _DEAD_LETTER_AFTER}
+        if exhausted:
             logger.error(
-                "DEAD-LETTER: item(s) at %s failed %d consecutive cycles — "
-                "advancing the cursor past them. They will NOT be retried; "
-                "see the per-item errors above for what was abandoned.",
-                pinned_at,
-                streak,
+                "DEAD-LETTER: abandoning %d item(s) after %d consecutive "
+                "failed cycles — they will NOT be retried: %s",
+                len(exhausted),
+                _DEAD_LETTER_AFTER,
+                ", ".join(sorted(k.split("|", 1)[1] + " @ " + k.split("|", 1)[0]
+                                 for k in exhausted)),
             )
+        surviving = [
+            f
+            for f in result.failures
+            if f"{f.timestamp.isoformat()}|{f.external_id}" not in exhausted
+        ]
+        if surviving:
+            new_cursor = min(f.timestamp for f in surviving) - timedelta(seconds=1)
+            state.extra["failure_streaks"] = {
+                k: v for k, v in streaks.items() if k not in exhausted
+            }
+        else:
+            # Every remaining failure just dead-lettered: advance past them.
+            max_abandoned = max(f.timestamp for f in result.failures)
             new_cursor = (
                 result.latest_success
                 if result.latest_success is not None
-                and result.latest_success > result.earliest_failure
-                else result.earliest_failure + timedelta(seconds=1)
+                and result.latest_success > max_abandoned
+                else max_abandoned + timedelta(seconds=1)
             )
-            state.extra.pop("failure_pinned_at", None)
-            state.extra.pop("failure_streak", None)
-        else:
-            new_cursor = result.earliest_failure - timedelta(seconds=1)
-            state.extra["failure_pinned_at"] = pinned_at
-            state.extra["failure_streak"] = streak
+            state.extra.pop("failure_streaks", None)
+        state.last_synced_at = new_cursor
+        state_store.save(state)
     elif result.latest_success is not None:
-        new_cursor = result.latest_success
-        state.extra.pop("failure_pinned_at", None)
-        state.extra.pop("failure_streak", None)
+        state.extra.pop("failure_streaks", None)
+        state.last_synced_at = result.latest_success
+        state_store.save(state)
     else:
-        new_cursor = None  # empty window — nothing to miss; default to now()
-        state.extra.pop("failure_pinned_at", None)
-        state.extra.pop("failure_streak", None)
-    state.last_synced_at = new_cursor or datetime.now(timezone.utc)
-    state_store.save(state)
+        # N4: an empty window used to stamp wall-clock now(), which loses a
+        # message posted between fetch and save (or any host-ahead-of-source
+        # clock skew). Nothing was fetched, so there is nothing to advance
+        # past — hold the previous cursor unchanged (None on a first run
+        # simply means the next run fetches from the start again).
+        state.extra.pop("failure_streaks", None)
+        state_store.save(state)
 
     logger.info(
         "sync done   fetched=%d ingested=%d skipped=%d errors=%d duration=%.1fs",