cortexdb-connectors 0.2.13__tar.gz → 0.2.14__tar.gz

{cortexdb_connectors-0.2.13 → cortexdb_connectors-0.2.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cortexdb-connectors
-Version: 0.2.13
+Version: 0.2.14
 Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
 Author-email: CortexDB Team <team@cortexdb.ai>
 License-Expression: Apache-2.0

{cortexdb_connectors-0.2.13 → cortexdb_connectors-0.2.14}/cortexdb_connectors/__init__.py

@@ -1,40 +1,40 @@
-"""cortexdb_connectors — data connector SDK for the CortexDB v1 API.
-
-Each connector subclass implements `fetch_events`, `normalize`, and
-`map_permissions`; the base class POSTs every normalised Episode to
-`/v1/experience` with the required Authorization + X-Cortex-Actor
-headers. Use the `cortexdb-sync` CLI to run a connector once or in a
-poll loop; or `from cortexdb_connectors.slack import SlackConnector`
-to drive it programmatically.
-"""
-
-__version__ = "0.2.8"
-
-from cortexdb_connectors.base import (
-    Actor,
-    CortexConnector,
-    EntityRef,
-    Episode,
-    EpisodeType,
-    RawEvent,
-    Source,
-    SyncResult,
-    SyncState,
-    Visibility,
-    VisibilityLevel,
-)
-
-__all__ = [
-    "__version__",
-    "Actor",
-    "CortexConnector",
-    "EntityRef",
-    "Episode",
-    "EpisodeType",
-    "RawEvent",
-    "Source",
-    "SyncResult",
-    "SyncState",
-    "Visibility",
-    "VisibilityLevel",
-]
+"""cortexdb_connectors — data connector SDK for the CortexDB v1 API.
+
+Each connector subclass implements `fetch_events`, `normalize`, and
+`map_permissions`; the base class POSTs every normalised Episode to
+`/v1/experience` with the required Authorization + X-Cortex-Actor
+headers. Use the `cortexdb-sync` CLI to run a connector once or in a
+poll loop; or `from cortexdb_connectors.slack import SlackConnector`
+to drive it programmatically.
+"""
+
+__version__ = "0.2.14"
+
+from cortexdb_connectors.base import (
+    Actor,
+    CortexConnector,
+    EntityRef,
+    Episode,
+    EpisodeType,
+    RawEvent,
+    Source,
+    SyncResult,
+    SyncState,
+    Visibility,
+    VisibilityLevel,
+)
+
+__all__ = [
+    "__version__",
+    "Actor",
+    "CortexConnector",
+    "EntityRef",
+    "Episode",
+    "EpisodeType",
+    "RawEvent",
+    "Source",
+    "SyncResult",
+    "SyncState",
+    "Visibility",
+    "VisibilityLevel",
+]

{cortexdb_connectors-0.2.13 → cortexdb_connectors-0.2.14}/cortexdb_connectors/confluence/__init__.py

@@ -43,6 +43,15 @@ _SOURCE = Source(system="confluence")
 
 _PAGE_LIMIT = 50
 
+# Expansions needed to build a full RawEvent from a content object. The CQL
+# search endpoint nests content under `content.`; the content REST API
+# returns the object directly.
+_CONTENT_EXPAND = (
+    "body.storage,version,history.lastUpdated,metadata.labels,"
+    "ancestors,container,restrictions.read.restrictions.user,"
+    "restrictions.read.restrictions.group"
+)
+
 _TAG_RE = re.compile(r"<[^>]+>")
 _WS_RE = re.compile(r"[ \t]+")
 
@@ -109,9 +118,16 @@ class ConfluenceConnector(CortexConnector):
     async def fetch_events(
         self, since: datetime | None = None
     ) -> list[RawEvent]:
-        """Fetch recently updated pages from configured Confluence spaces.
-
-        Uses the CQL ``lastModified`` filter to retrieve incremental updates.
+        """Fetch recently updated pages and comments from configured spaces.
+
+        Pages are enumerated primarily through the content REST API, which
+        reads the content store directly — a page created seconds before
+        the sync is visible immediately. The CQL search index is only a
+        supplement (it is the sole space-wide way to enumerate comments):
+        it is eventually consistent, so relying on it as the ONLY discovery
+        path meant a just-created page was invisible to its own sync run
+        (KAN-114..122). Duplicates across the two sources are deduped by
+        ``{id}:v{version}`` — the same shape as the idempotency key.
         """
         from atlassian import Confluence
 
@@ -126,94 +142,186 @@ class ConfluenceConnector(CortexConnector):
         raw_events: list[RawEvent] = []
 
         for space_key in spaces:
-            # Comments are content too (type filter includes them): a
-            # conversation planted on a page used to be silently dropped —
-            # the connector fetched pages only.
-            cql = f'space = "{space_key}" AND type IN ("page", "comment")'
-            if since:
-                # CQL date strings are interpreted in the API user's account
-                # timezone and carry no offset syntax; a UTC-formatted cursor
-                # can therefore sit up to a day in the "future" or "past" of
-                # what Confluence compares against. Widen the window by 24h —
-                # the overlap is absorbed by idempotency replay, a missed
-                # window is silent data loss.
-                cql_date = (since - timedelta(hours=24)).strftime("%Y-%m-%d %H:%M")
-                cql += f' AND lastModified >= "{cql_date}"'
-            cql += " ORDER BY lastModified DESC"
-
-            start = 0
-            while True:
-                try:
-                    results = client.cql(
-                        cql,
-                        start=start,
-                        limit=_PAGE_LIMIT,
-                        expand="content.body.storage,content.version,"
-                        "content.history.lastUpdated,"
-                        "content.metadata.labels,"
-                        "content.ancestors,"
-                        "content.container,"
-                        "content.restrictions.read.restrictions.user,"
-                        "content.restrictions.read.restrictions.group",
-                    )
-                except Exception:
-                    logger.exception(
-                        "CQL query failed for space %s", space_key
-                    )
-                    break
+            seen: set[str] = set()
+            # Primary: content REST API (read-your-writes).
+            raw_events.extend(
+                self._fetch_space_pages_rest(client, space_key, since, seen)
+            )
+            # Supplement: CQL search (comments + anything REST missed).
+            raw_events.extend(
+                self._fetch_space_cql(client, space_key, since, seen)
+            )
 
-                search_results = results.get("results", [])
-                if not search_results:
-                    break
+        return raw_events
+
+    def _fetch_space_pages_rest(
+        self,
+        client: Any,
+        space_key: str,
+        since: datetime | None,
+        seen: set[str],
+    ) -> list[RawEvent]:
+        """Enumerate a space's pages via ``/rest/api/content``.
 
-                for result in search_results:
-                    content_obj = result.get("content", result)
-                    is_comment = (content_obj.get("type") == "comment")
-                    if is_comment:
-                        item_data = self._extract_comment_data(
-                            content_obj, space_key
-                        )
-                        if not item_data.get("comment_id"):
-                            continue
-                        external_id = f"comment:{item_data['comment_id']}"
-                    else:
-                        item_data = self._extract_page_data(
-                            content_obj, space_key
-                        )
-                        # The search `excerpt` lives on the RESULT row, not
-                        # the content object (the old read was dead code).
-                        if not item_data.get("excerpt"):
-                            item_data["excerpt"] = result.get("excerpt", "")
-                        external_id = item_data.get("page_id", "")
-                    updated_str = item_data.get("last_modified", "")
-                    try:
-                        updated_at = datetime.fromisoformat(
-                            updated_str.replace("Z", "+00:00")
-                        )
-                    except (ValueError, TypeError):
-                        updated_at = datetime.now(timezone.utc)
-
-                    restrictions = self._extract_restrictions(content_obj)
-
-                    raw_events.append(
-                        RawEvent(
-                            source="confluence",
-                            external_id=external_id,
-                            timestamp=updated_at,
-                            data=item_data,
-                            permissions={
-                                "space_key": space_key,
-                                "restrictions": restrictions,
-                            },
-                        )
-                    )
-
-                total = results.get("totalSize", 0)
-                start += _PAGE_LIMIT
-                if start >= total:
+        Ordered newest-first by ``history.lastUpdated`` so incremental runs
+        can stop paging at the (24h-widened, same as CQL) cursor window.
+        """
+        events: list[RawEvent] = []
+        cutoff = (since - timedelta(hours=24)) if since else None
+        start = 0
+        while True:
+            try:
+                results = client.get(
+                    "rest/api/content",
+                    params={
+                        "spaceKey": space_key,
+                        "type": "page",
+                        "orderby": "-history.lastUpdated",
+                        "expand": _CONTENT_EXPAND,
+                        "start": start,
+                        "limit": _PAGE_LIMIT,
+                    },
+                )
+            except Exception:
+                logger.exception(
+                    "content listing failed for space %s", space_key
+                )
+                break
+
+            items = results.get("results", []) if isinstance(results, dict) else []
+            if not items:
+                break
+
+            reached_cutoff = False
+            for content_obj in items:
+                event = self._content_to_event(content_obj, space_key)
+                if event is None:
+                    continue
+                if cutoff is not None and event.timestamp < cutoff:
+                    reached_cutoff = True
                     break
+                key = f"{event.external_id}:v{event.data.get('version', 1)}"
+                if key in seen:
+                    continue
+                seen.add(key)
+                events.append(event)
 
-        return raw_events
+            if reached_cutoff or len(items) < _PAGE_LIMIT:
+                break
+            start += _PAGE_LIMIT
+
+        return events
+
+    def _fetch_space_cql(
+        self,
+        client: Any,
+        space_key: str,
+        since: datetime | None,
+        seen: set[str],
+    ) -> list[RawEvent]:
+        """Enumerate a space's content via CQL search (supplement)."""
+        events: list[RawEvent] = []
+        # Comments are content too (type filter includes them): a
+        # conversation planted on a page used to be silently dropped —
+        # the connector fetched pages only.
+        cql = f'space = "{space_key}" AND type IN ("page", "comment")'
+        if since:
+            # CQL date strings are interpreted in the API user's account
+            # timezone and carry no offset syntax; a UTC-formatted cursor
+            # can therefore sit up to a day in the "future" or "past" of
+            # what Confluence compares against. Widen the window by 24h —
+            # the overlap is absorbed by idempotency replay, a missed
+            # window is silent data loss.
+            cql_date = (since - timedelta(hours=24)).strftime("%Y-%m-%d %H:%M")
+            cql += f' AND lastModified >= "{cql_date}"'
+        cql += " ORDER BY lastModified DESC"
+
+        start = 0
+        while True:
+            try:
+                results = client.cql(
+                    cql,
+                    start=start,
+                    limit=_PAGE_LIMIT,
+                    expand=",".join(
+                        f"content.{part}" for part in _CONTENT_EXPAND.split(",")
+                    ),
+                )
+            except Exception:
+                logger.exception(
+                    "CQL query failed for space %s", space_key
+                )
+                break
+
+            search_results = results.get("results", [])
+            if not search_results:
+                break
+
+            for result in search_results:
+                content_obj = result.get("content", result)
+                event = self._content_to_event(
+                    content_obj,
+                    space_key,
+                    # The search `excerpt` lives on the RESULT row, not
+                    # the content object (the old read was dead code).
+                    fallback_excerpt=result.get("excerpt", ""),
+                )
+                if event is None:
+                    continue
+                key = f"{event.external_id}:v{event.data.get('version', 1)}"
+                if key in seen:
+                    continue
+                seen.add(key)
+                events.append(event)
+
+            total = results.get("totalSize", 0)
+            start += _PAGE_LIMIT
+            if start >= total:
+                break
+
+        return events
+
+    def _content_to_event(
+        self,
+        content_obj: dict[str, Any],
+        space_key: str,
+        fallback_excerpt: str = "",
+    ) -> RawEvent | None:
+        """Build a RawEvent from a content object (page or comment)."""
+        is_comment = (content_obj.get("type") == "comment")
+        if is_comment:
+            item_data = self._extract_comment_data(content_obj, space_key)
+            if not item_data.get("comment_id"):
+                return None
+            external_id = f"comment:{item_data['comment_id']}"
+        else:
+            item_data = self._extract_page_data(content_obj, space_key)
+            if not item_data.get("excerpt"):
+                item_data["excerpt"] = fallback_excerpt
+            external_id = item_data.get("page_id", "")
+
+        updated_str = item_data.get("last_modified", "")
+        try:
+            updated_at = datetime.fromisoformat(
+                updated_str.replace("Z", "+00:00")
+            )
+        except (ValueError, TypeError):
+            updated_at = datetime.now(timezone.utc)
+        if updated_at.tzinfo is None:
+            updated_at = updated_at.replace(tzinfo=timezone.utc)
+
+        restrictions = self._extract_restrictions(content_obj)
+
+        return RawEvent(
+            source="confluence",
+            external_id=external_id,
+            timestamp=updated_at,
+            data=item_data,
+            permissions={
+                "space_key": space_key,
+                "restrictions": restrictions,
+            },
+        )
 
     def normalize(self, raw: RawEvent) -> Episode | None:
         """Convert a Confluence ``RawEvent`` (page or comment) into an

{cortexdb_connectors-0.2.13 → cortexdb_connectors-0.2.14}/cortexdb_connectors/discord/__init__.py

@@ -146,8 +146,15 @@ class DiscordConnector(CortexConnector):
 
         return events
 
-    def normalize(self, raw: RawEvent) -> Episode:
-        """Convert a Discord ``RawEvent`` into a CortexDB ``Episode``."""
+    def normalize(self, raw: RawEvent) -> Episode | None:
+        """Convert a Discord ``RawEvent`` into a CortexDB ``Episode``.
+
+        Returns None (skip) for messages the content policy excludes:
+        text-less system messages (thread-created announcements, member
+        joins, pins) and anything else with no renderable content — the
+        server rejects empty-text envelopes (422 INVALID_ENVELOPE,
+        permanent), so they must be dropped client-side.
+        """
         data = raw.data
         event_kind = data.get("_event_kind", "message")
 
@@ -189,9 +196,82 @@ class DiscordConnector(CortexConnector):
 
     # -- message normalization -----------------------------------------------
 
-    def _normalize_message(self, raw: RawEvent) -> Episode:
-        """Normalize a Discord message into an Episode."""
+    @staticmethod
+    def _render_message_content(data: dict[str, Any]) -> str | None:
+        """Discord content policy; None means skip the message.
+
+        The server rejects empty-text envelopes (422 INVALID_ENVELOPE,
+        permanent — KAN-97..111/123..127), so a text-less message must
+        either render its non-text payload or be skipped client-side.
+        Rendered: message text, attachment filenames, embeds (title, or
+        description/URL for untitled embeds), stickers, poll question +
+        options. Skipped: only when nothing renderable remains — which
+        covers thread-created / member-join / pin system messages, whose
+        ``content`` is always empty.
+        """
+        segments: list[str] = []
+
+        text = (data.get("content") or "").strip()
+        if text:
+            segments.append(text)
+
+        # Attachment filenames.
+        attachments = data.get("attachments", [])
+        if attachments:
+            att_desc = ", ".join(
+                a.get("filename", "attachment") for a in attachments
+            )
+            segments.append(f"[Attachments: {att_desc}]")
+
+        # Embeds: titled embeds render their title (previous behaviour);
+        # untitled embeds fall back to description or URL instead of
+        # vanishing (untitled-embed-only messages were sent empty).
+        embed_descs = [
+            desc
+            for e in data.get("embeds", [])
+            if (
+                desc := str(
+                    e.get("title") or e.get("description") or e.get("url") or ""
+                ).strip()
+            )
+        ]
+        if embed_descs:
+            segments.append(f"[Embeds: {', '.join(embed_descs)}]")
+
+        # Stickers (sticker-only messages carry no ``content``).
+        for sticker in data.get("sticker_items") or data.get("stickers") or []:
+            name = (sticker.get("name") or "").strip()
+            segments.append(f"[sticker: {name}]" if name else "[sticker]")
+
+        # Polls: question + options (poll-only messages carry no ``content``).
+        poll = data.get("poll") or {}
+        question = ((poll.get("question") or {}).get("text") or "").strip()
+        options = [
+            opt
+            for a in poll.get("answers", [])
+            if (opt := ((a.get("poll_media") or {}).get("text") or "").strip())
+        ]
+        if question or options:
+            poll_desc = question
+            if options:
+                joined = ", ".join(options)
+                poll_desc = (
+                    f"{question} — options: {joined}"
+                    if question
+                    else f"options: {joined}"
+                )
+            segments.append(f"[poll: {poll_desc}]")
+
+        if not segments:
+            return None
+        return "\n".join(segments)
+
+    def _normalize_message(self, raw: RawEvent) -> Episode | None:
+        """Normalize a Discord message into an Episode (None = skip)."""
         data = raw.data
+        content = self._render_message_content(data)
+        if content is None:
+            return None
         author = data.get("author", {})
         actor = Actor(
             id=author.get("id", "unknown"),
@@ -209,24 +289,8 @@ class DiscordConnector(CortexConnector):
         guild_id = data.get("guild_id", "")
         guild_name = data.get("_guild_name", guild_id)
 
-        content = data.get("content", "")
-
-        # Append attachment descriptions.
         attachments = data.get("attachments", [])
-        if attachments:
-            att_desc = ", ".join(
-                a.get("filename", "attachment") for a in attachments
-            )
-            content += f"\n[Attachments: {att_desc}]"
-
-        # Append embed titles.
         embeds = data.get("embeds", [])
-        if embeds:
-            embed_desc = ", ".join(
-                e.get("title", "embed") for e in embeds if e.get("title")
-            )
-            if embed_desc:
-                content += f"\n[Embeds: {embed_desc}]"
 
         entities = self._extract_message_entities(data)
 
@@ -278,8 +342,12 @@ class DiscordConnector(CortexConnector):
             idempotency_key=f"discord:{data.get('id', raw.external_id)}",
         )
 
-    def _normalize_forum_post(self, raw: RawEvent) -> Episode:
-        """Normalize a Discord forum post (thread starter) into an Episode."""
+    def _normalize_forum_post(self, raw: RawEvent) -> Episode | None:
+        """Normalize a Discord forum post (thread starter) into an Episode.
+
+        Returns None (skip) when neither the thread name nor the starter
+        message renders any content.
+        """
         data = raw.data
         author = data.get("author", {})
         actor = Actor(
@@ -294,9 +362,12 @@ class DiscordConnector(CortexConnector):
         )
 
         thread_name = data.get("_thread_name", "")
-        content = data.get("content", "")
+        body = self._render_message_content(data)
+        if body is None and not thread_name:
+            return None
+        content = body or ""
         if thread_name:
-            content = f"[{thread_name}]\n\n{content}"
+            content = f"[{thread_name}]\n\n{content}" if content else f"[{thread_name}]"
 
         entities = self._extract_message_entities(data)
 

{cortexdb_connectors-0.2.13 → cortexdb_connectors-0.2.14}/cortexdb_connectors/google_workspace/__init__.py

@@ -752,8 +752,13 @@ class GoogleWorkspaceConnector(CortexConnector):
 
         return events
 
-    def normalize(self, raw: RawEvent) -> Episode:
-        """Route normalization to the appropriate sub-connector handler."""
+    def normalize(self, raw: RawEvent) -> Episode | None:
+        """Route normalization to the appropriate sub-connector handler.
+
+        Returns None (skip) for items with no renderable content — the
+        server rejects empty-text envelopes (422 INVALID_ENVELOPE,
+        permanent), so they must be dropped client-side.
+        """
         sub_source = raw.data.get("_sub_source", "")
         if sub_source == "gmail":
             return self._normalize_gmail(raw)
@@ -776,8 +781,8 @@ class GoogleWorkspaceConnector(CortexConnector):
 
     # -- Gmail normalization -------------------------------------------------
 
-    def _normalize_gmail(self, raw: RawEvent) -> Episode:
-        """Normalize a Gmail message into a CortexDB Episode."""
+    def _normalize_gmail(self, raw: RawEvent) -> Episode | None:
+        """Normalize a Gmail message into a CortexDB Episode (None = skip)."""
         data = raw.data
         actor = Actor(
             id=data.get("from_email", "unknown"),
@@ -785,11 +790,18 @@ class GoogleWorkspaceConnector(CortexConnector):
             email=data.get("from_email"),
         )
 
-        subject = data.get("subject", "(no subject)")
+        subject = (data.get("subject") or "").strip()
         # Full body when the fetch carried one (format="full"); the ~200-char
         # snippet is only the no-body fallback — it silently lost every mail
         # body otherwise.
-        body = data.get("body", "") or data.get("snippet", "")
+        body = (data.get("body", "") or data.get("snippet", "")).strip()
+        if not subject and not body:
+            # Nothing renderable (empty Subject, no body, no snippet): skip
+            # client-side — the server rejects empty-text envelopes with a
+            # permanent 422 INVALID_ENVELOPE.
+            return None
+        if not subject:
+            subject = "(no subject)"
         content = f"Subject: {subject}\n\n{body}" if body else subject
 
         thread_id = data.get("thread_id")