cortexdb-connectors 0.2.18__tar.gz → 0.2.19__tar.gz

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.18
+Version: 0.2.19
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors/__init__.py

@@ -8,7 +8,7 @@ poll loop; or `from cortexdb_connectors.slack import SlackConnector`
 to drive it programmatically.
 """
 
-__version__ = "0.2.18"
+__version__ = "0.2.19"
 
 from cortexdb_connectors.base import (
     Actor,

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors/base.py

@@ -114,7 +114,7 @@ class VisibilityLevel(str, Enum):
 @dataclass(frozen=True)
 class Source:
     system: str
-    connector_version: str = "0.2.18"
+    connector_version: str = "0.2.19"
 
 
 @dataclass(frozen=True)

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors/cli.py

@@ -124,6 +124,7 @@ CONNECTORS: dict[str, dict[str, Any]] = {
         "options": [
             ("database_ids", "NOTION_DATABASES", "list"),
             ("page_ids", "NOTION_PAGES", "list"),
+            ("ledger_path", "NOTION_LEDGER_PATH", "str"),
         ],
     },
     "pagerduty": {

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors/notion/__init__.py

@@ -10,8 +10,11 @@ from __future__ import annotations
 
 import asyncio
 import hashlib
+import json
 import logging
+import os
 from datetime import datetime, timedelta, timezone
+from pathlib import Path
 from typing import Any
 
 from cortexdb_connectors.base import (
@@ -36,6 +39,12 @@ _RATE_LIMIT_DELAY = 0.35
 # Maximum characters of page body to include in episode content.
 _BODY_EXCERPT_LENGTH = 2000
 
+# KAN-132/135: how long a page stays in the seen-pages ledger after it was
+# last observed. Archived pages are EXCLUDED from Notion search and
+# databases.query, so without the ledger their comments become unreachable
+# the moment the parent is archived — the ledger keeps known ids swept.
+_LEDGER_RETENTION_DAYS = 90
+
 
 class NotionConnector(CortexConnector):
     """Connector that syncs Notion pages, database entries, and comments
@@ -59,6 +68,14 @@ class NotionConnector(CortexConnector):
         CortexDB namespace for ingested episodes.
     backfill_days:
         Number of days of history to backfill on first sync.
+    ledger_path:
+        Where the seen-pages ledger lives (env ``NOTION_LEDGER_PATH``,
+        default ``~/.cortexdb/notion_pages_ledger.json``). Notion search
+        and ``databases.query`` EXCLUDE archived pages/rows, so a page
+        that gets archived falls out of discovery — the ledger remembers
+        every page seen in prior syncs and keeps sweeping its comments
+        (``comments.list`` works on archived parents, KAN-24). Entries
+        age out after ``_LEDGER_RETENTION_DAYS`` days unseen.
     """
 
     connector_name: str = "notion"
@@ -73,6 +90,7 @@ class NotionConnector(CortexConnector):
         scope: str = "user:default",
         namespace: str = "notion",
         backfill_days: int = 30,
+        ledger_path: str | None = None,
     ) -> None:
         super().__init__(cortex_url, cortex_api_key, scope)
         self.notion_token = notion_token
@@ -80,6 +98,11 @@ class NotionConnector(CortexConnector):
         self.page_ids = page_ids or []
         self.namespace = namespace
         self.backfill_days = backfill_days
+        self.ledger_path = Path(
+            ledger_path
+            or os.environ.get("NOTION_LEDGER_PATH")
+            or (Path.home() / ".cortexdb" / "notion_pages_ledger.json")
+        )
         self._user_cache: dict[str, dict[str, Any]] = {}
 
     # -- CortexConnector interface -------------------------------------------
@@ -136,6 +159,27 @@ class NotionConnector(CortexConnector):
                 page_ids_seen.add(pid)
                 sweep_ids.append(pid)
 
+        # KAN-132/135: pages seen in PRIOR syncs whose parent has since been
+        # archived produce no events this run (search and databases.query
+        # exclude trashed items) — without this sweep their comments are
+        # silently unreachable forever, even though comments.list still
+        # serves them. The ledger remembers every page ever seen under this
+        # scope and keeps them swept.
+        ledger = self._ledger_load()
+        known_pages = ledger.get(self._ledger_key(), {})
+        ledger_swept = 0
+        for pid in known_pages:
+            if pid not in page_ids_seen:
+                page_ids_seen.add(pid)
+                sweep_ids.append(pid)
+                ledger_swept += 1
+        if ledger_swept:
+            logger.info(
+                "notion: sweeping comments for %d ledger page(s) not "
+                "discovered this run (archived or unmodified)",
+                ledger_swept,
+            )
+
         failed_sweeps = 0
         for pid in sweep_ids:
             comment_events, failed = await self._fetch_comments(
@@ -152,8 +196,91 @@ class NotionConnector(CortexConnector):
                 failed_sweeps,
             )
 
+        # Record what this run saw: event pages refresh their last_seen;
+        # ledger-only pages keep their old stamp so they age out after
+        # _LEDGER_RETENTION_DAYS (pruned loudly, never silently).
+        now_iso = datetime.now(timezone.utc).isoformat()
+        for event in events:
+            pid = event.data.get("page_id", "")
+            if pid:
+                known_pages[pid] = now_iso
+        horizon = datetime.now(timezone.utc) - timedelta(
+            days=_LEDGER_RETENTION_DAYS
+        )
+        pruned = []
+        for pid, seen in list(known_pages.items()):
+            seen_at = self._parse_ledger_ts(seen)
+            if seen_at is not None and seen_at < horizon:
+                pruned.append(pid)
+                del known_pages[pid]
+        if pruned:
+            logger.warning(
+                "notion: %d page(s) aged out of the seen-pages ledger "
+                "(unseen for %dd): %s — comments on them are no longer "
+                "swept; re-add via NOTION_PAGE_IDS if still wanted",
+                len(pruned),
+                _LEDGER_RETENTION_DAYS,
+                ", ".join(pruned[:10]) + ("…" if len(pruned) > 10 else ""),
+            )
+        ledger[self._ledger_key()] = known_pages
+        self._ledger_save(ledger)
+
         return events
 
+    # -- seen-pages ledger (KAN-132/135) --------------------------------------
+
+    def _ledger_key(self) -> str:
+        return f"{self.scope_template or self.scope}"
+
+    @staticmethod
+    def _parse_ledger_ts(value: Any) -> datetime | None:
+        if not isinstance(value, str) or not value:
+            return None
+        try:
+            parsed = datetime.fromisoformat(value)
+        except ValueError:
+            return None
+        return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
+
+    def _ledger_load(self) -> dict[str, Any]:
+        """Load the ledger; a broken file degrades LOUDLY to empty.
+
+        Unlike Jira's deletion ledger (where an empty read would forget
+        deletions), an empty comments ledger only shrinks sweep coverage
+        for this cycle — not worth failing the whole sync over, but never
+        silent either.
+        """
+        try:
+            with open(self.ledger_path, "r", encoding="utf-8") as fh:
+                data = json.load(fh)
+            return data if isinstance(data, dict) else {}
+        except FileNotFoundError:
+            return {}
+        except (OSError, json.JSONDecodeError) as exc:
+            logger.error(
+                "notion: seen-pages ledger at %s is unreadable (%s) — "
+                "comments on archived pages will NOT be swept this cycle; "
+                "delete or restore the file to rebuild it",
+                self.ledger_path,
+                exc,
+            )
+            return {}
+
+    def _ledger_save(self, data: dict[str, Any]) -> None:
+        try:
+            self.ledger_path.parent.mkdir(parents=True, exist_ok=True)
+            tmp = self.ledger_path.with_suffix(".tmp")
+            with open(tmp, "w", encoding="utf-8") as fh:
+                json.dump(data, fh, indent=2, default=str)
+            os.replace(tmp, self.ledger_path)
+        except OSError as exc:
+            logger.error(
+                "notion: could not save seen-pages ledger to %s (%s) — "
+                "archived-page comment coverage will regress next run",
+                self.ledger_path,
+                exc,
+            )
+
     def normalize(self, raw: RawEvent) -> Episode:
         """Convert a Notion ``RawEvent`` into a CortexDB ``Episode``."""
         data = raw.data
@@ -209,6 +336,20 @@ class NotionConnector(CortexConnector):
         if database_name:
             tags.append(database_name.lower().replace(" ", "_"))
 
+        metadata = {
+            "page_id": data.get("page_id", ""),
+            "database_id": data.get("database_id", ""),
+            "url": data.get("url", ""),
+            "properties": data.get("properties_raw", {}),
+            "last_edited_time": data.get("last_edited_time", ""),
+            "created_time": data.get("created_time", ""),
+            "archived": data.get("archived", False),
+        }
+        if data.get("body_unavailable"):
+            # KAN-134: the body could not be fetched (archived page) — a
+            # queryable marker so operators can find these gaps at recall.
+            metadata["body_unavailable"] = True
+
         return Episode(
             actor=actor,
             source=_SOURCE,
@@ -220,15 +361,7 @@ class NotionConnector(CortexConnector):
             namespace=self.namespace,
             entities=entities,
             tags=tags,
-            metadata={
-                "page_id": data.get("page_id", ""),
-                "database_id": data.get("database_id", ""),
-                "url": data.get("url", ""),
-                "properties": data.get("properties_raw", {}),
-                "last_edited_time": data.get("last_edited_time", ""),
-                "created_time": data.get("created_time", ""),
-                "archived": data.get("archived", False),
-            },
+            metadata=metadata,
             thread_id=f"notion:{data.get('page_id', '')}",
             parent_id=(
                 f"notion:{data['parent_id']}" if data.get("parent_id") else None
@@ -355,9 +488,24 @@ class NotionConnector(CortexConnector):
         if edited_at < cutoff:
             return []
 
-        # Fetch page content blocks.
+        # Fetch page content blocks. KAN-134: Notion's blocks API errors on
+        # archived pages — the body of a page archived before it was ever
+        # synced is UNREACHABLE via the API. That loss must be loud and
+        # queryable, never a silent title-only episode.
         body_text = await self._fetch_block_children_text(client, page_id)
+        body_unavailable = not body_text and bool(page.get("archived"))
+        if body_unavailable:
+            logger.warning(
+                "notion: page %s is archived and its body blocks are not "
+                "retrievable via the API — ingesting title/properties only, "
+                "marked body_unavailable. Content archived before the first "
+                "sync of this page cannot be recovered unless the page is "
+                "restored from trash.",
+                page_id,
+            )
         raw = self._page_to_raw_event(page, body_text=body_text)
+        if body_unavailable:
+            raw.data["body_unavailable"] = True
         return [raw]
 
     async def _search_recent_pages(
@@ -527,7 +675,25 @@ class NotionConnector(CortexConnector):
 
                 results = client.blocks.children.list(**params)
                 await asyncio.sleep(_RATE_LIMIT_DELAY)
-            except Exception:
+            except Exception as exc:
+                # KAN-134: this used to be a bare `break` — a blocks-API
+                # failure (archived page, revoked block capability, 5xx)
+                # silently truncated the page body to whatever was fetched
+                # so far, and the episode ingested as if that were the
+                # whole page. Log it; the caller marks archived pages.
+                status = (
+                    getattr(exc, "status", None)
+                    or getattr(exc, "code", None)
+                    or "?"
+                )
+                logger.warning(
+                    "notion: block children fetch failed for %s "
+                    "(status=%s): %s — page body truncated at %d chars",
+                    block_id,
+                    status,
+                    exc,
+                    sum(len(p) for p in parts),
+                )
                 break
 
             for block in results.get("results", []):

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.18
+Version: 0.2.19
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/cortexdb_connectors.egg-info/SOURCES.txt

@@ -43,6 +43,7 @@ tests/test_jira_attachments.py
 tests/test_jira_findings_0_2_11.py
 tests/test_jira_findings_0_2_9.py
 tests/test_notion_comments.py
+tests/test_notion_findings_0_2_19.py
 tests/test_product_findings_0_2_12.py
 tests/test_product_findings_0_2_14.py
 tests/test_slack.py

{cortexdb_connectors-0.2.18 → cortexdb_connectors-0.2.19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cortexdb-connectors"
-version = "0.2.18"
+version = "0.2.19"
 description = "Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API."
 readme = "README.md"
 requires-python = ">=3.10"

cortexdb_connectors-0.2.19/tests/test_notion_findings_0_2_19.py

@@ -0,0 +1,192 @@
+"""Pins for the Notion archive-flow campaign findings, fixed in 0.2.19.
+
+Campaign 20260806 (KAN-132/134/135): comment-then-archive and archive-page
+flows recalled nothing. Root causes, in the connector:
+
+- Notion search and ``databases.query`` EXCLUDE archived pages/rows, and
+  the comment sweep only covered pages seen THIS run or explicitly
+  configured — so a page seen in a prior sync that later got archived
+  silently lost all comment coverage forever (``comments.list`` would have
+  served them, KAN-24). Fixed with a persistent seen-pages ledger.
+- ``_fetch_block_children_text`` swallowed every blocks-API failure with a
+  bare ``break`` — an archived page's unreachable body (or any 403/5xx
+  truncation) ingested as a silently title-only/truncated episode. Now
+  logged loudly and, for archived pages, marked ``body_unavailable`` as a
+  queryable label.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+from cortexdb_connectors.notion import NotionConnector, _LEDGER_RETENTION_DAYS
+
+from .test_notion_comments import (
+    _RECENT,
+    _FakeNotion,
+    _comment,
+    _connector,
+    _fetch,
+    _comment_events,
+    _page,
+)
+
+
+# ---------------------------------------------------------------------------
+# Seen-pages ledger (KAN-132 / KAN-135)
+# ---------------------------------------------------------------------------
+
+
+def test_ledger_records_pages_seen_via_search(tmp_path):
+    ledger = tmp_path / "ledger.json"
+    c = _connector(ledger_path=str(ledger))
+    fake = _FakeNotion(
+        pages={"p1": _page("p1")}, search_results=[_page("p1")]
+    )
+    _fetch(c, fake)
+    data = json.loads(ledger.read_text())
+    assert "p1" in data.get("user:test", {})
+
+
+def test_archived_page_comments_swept_via_ledger(tmp_path):
+    """The KAN-132 flow: page seen in sync 1, archived before sync 2 —
+    sync 2 must still fetch its comments even though search no longer
+    returns the page and it is not configured."""
+    ledger = tmp_path / "ledger.json"
+
+    # Sync 1: page discovered via search; no comments yet.
+    c1 = _connector(ledger_path=str(ledger))
+    fake1 = _FakeNotion(pages={"p1": _page("p1")}, search_results=[_page("p1")])
+    _fetch(c1, fake1)
+
+    # Page archived; a comment lands while archived. Search excludes it now.
+    c2 = _connector(ledger_path=str(ledger))
+    fake2 = _FakeNotion(
+        pages={"p1": _page("p1", archived=True)},
+        search_results=[_page("p1", archived=True)],  # fake search drops archived
+        comments={"p1": [_comment("c1", text="orphan canary")]},
+    )
+    events = _fetch(c2, fake2)
+
+    comments = _comment_events(events)
+    assert [e.data["text"] for e in comments] == ["orphan canary"]
+    assert "p1" in fake2.comment_calls  # swept from the ledger, not discovery
+
+
+def test_ledger_page_with_active_comments_stays_fresh(tmp_path):
+    """A ledger-swept page whose comments still flow refreshes last_seen —
+    an archived-but-active discussion never ages out."""
+    ledger = tmp_path / "ledger.json"
+    stale = (
+        datetime.now(timezone.utc) - timedelta(days=_LEDGER_RETENTION_DAYS - 1)
+    ).isoformat()
+    ledger.write_text(json.dumps({"user:test": {"p1": stale}}))
+
+    c = _connector(ledger_path=str(ledger))
+    fake = _FakeNotion(comments={"p1": [_comment("c1")]})
+    _fetch(c, fake)
+
+    data = json.loads(ledger.read_text())
+    refreshed = datetime.fromisoformat(data["user:test"]["p1"])
+    assert refreshed > datetime.fromisoformat(stale)
+
+
+def test_ledger_prunes_long_unseen_pages_loudly(tmp_path, caplog):
+    ledger = tmp_path / "ledger.json"
+    ancient = (
+        datetime.now(timezone.utc) - timedelta(days=_LEDGER_RETENTION_DAYS + 10)
+    ).isoformat()
+    ledger.write_text(json.dumps({"user:test": {"dead-page": ancient}}))
+
+    c = _connector(ledger_path=str(ledger))
+    with caplog.at_level("WARNING"):
+        _fetch(c, _FakeNotion())
+
+    data = json.loads(ledger.read_text())
+    assert "dead-page" not in data.get("user:test", {})
+    assert "aged out" in caplog.text and "dead-page" in caplog.text
+
+
+def test_corrupt_ledger_degrades_loudly_not_fatally(tmp_path, caplog):
+    ledger = tmp_path / "ledger.json"
+    ledger.write_text("{not json")
+    c = _connector(ledger_path=str(ledger))
+    fake = _FakeNotion(pages={"p1": _page("p1")}, search_results=[_page("p1")])
+    with caplog.at_level("ERROR"):
+        events = _fetch(c, fake)
+    assert events  # the sync itself still ran
+    assert "unreadable" in caplog.text
+    # And the save afterwards rebuilt a valid ledger.
+    assert "p1" in json.loads(ledger.read_text()).get("user:test", {})
+
+
+def test_ledger_is_scope_keyed(tmp_path):
+    """Two connectors with different scopes don't sweep each other's pages."""
+    ledger = tmp_path / "ledger.json"
+    ledger.write_text(json.dumps({"user:other": {"p9": _RECENT}}))
+    c = _connector(ledger_path=str(ledger))  # scope user:test
+    fake = _FakeNotion()
+    _fetch(c, fake)
+    assert "p9" not in fake.comment_calls
+
+
+# ---------------------------------------------------------------------------
+# Loud body loss (KAN-134)
+# ---------------------------------------------------------------------------
+
+
+class _ArchivedBlocksError(Exception):
+    status = 400
+    code = "validation_error"
+
+
+class _BrokenBlocksNotion(_FakeNotion):
+    def _list_blocks(self, **params):
+        raise _ArchivedBlocksError("Can't get children of an archived block")
+
+
+def test_archived_page_body_marked_unavailable(tmp_path, caplog):
+    c = _connector(
+        page_ids=["p1"], ledger_path=str(tmp_path / "ledger.json")
+    )
+    fake = _BrokenBlocksNotion(pages={"p1": _page("p1", archived=True)})
+    with caplog.at_level("WARNING"):
+        events = _fetch(c, fake)
+
+    page_events = [e for e in events if e.data.get("_event_kind") != "comment"]
+    assert len(page_events) == 1
+    assert page_events[0].data.get("body_unavailable") is True
+    assert "archived" in caplog.text and "body_unavailable" in caplog.text
+
+    ep = c.normalize(page_events[0])
+    assert ep.metadata.get("body_unavailable") is True  # queryable label
+
+
+def test_blocks_failure_on_live_page_logs_truncation(tmp_path, caplog):
+    c = _connector(
+        page_ids=["p1"], ledger_path=str(tmp_path / "ledger.json")
+    )
+    fake = _BrokenBlocksNotion(pages={"p1": _page("p1", archived=False)})
+    with caplog.at_level("WARNING"):
+        events = _fetch(c, fake)
+
+    page_events = [e for e in events if e.data.get("_event_kind") != "comment"]
+    assert len(page_events) == 1
+    # Not archived → no body_unavailable marker, but the failure is loud.
+    assert page_events[0].data.get("body_unavailable") is None
+    assert "block children fetch failed" in caplog.text
+
+
+def test_live_page_with_healthy_blocks_has_no_marker(tmp_path):
+    c = _connector(
+        page_ids=["p1"], ledger_path=str(tmp_path / "ledger.json")
+    )
+    fake = _FakeNotion(pages={"p1": _page("p1")})
+    events = _fetch(c, fake)
+    page_events = [e for e in events if e.data.get("_event_kind") != "comment"]
+    assert page_events[0].data.get("body_unavailable") is None
+    ep = c.normalize(page_events[0])
+    assert "body_unavailable" not in ep.metadata