cortexdb-connectors 0.2.17__tar.gz → 0.2.18__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/PKG-INFO +1 -1
  2. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/__init__.py +1 -1
  3. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/base.py +1 -1
  4. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/cli.py +5 -1
  5. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/slack/__init__.py +317 -13
  6. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/webhooks.py +133 -16
  7. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/PKG-INFO +1 -1
  8. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/SOURCES.txt +1 -0
  9. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/pyproject.toml +1 -1
  10. cortexdb_connectors-0.2.18/tests/test_slack_file_ingestion.py +599 -0
  11. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/README.md +0 -0
  12. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/confluence/__init__.py +0 -0
  13. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/discord/__init__.py +0 -0
  14. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/freshdesk/__init__.py +0 -0
  15. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/github/__init__.py +0 -0
  16. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/gitlab/__init__.py +0 -0
  17. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/google_workspace/__init__.py +0 -0
  18. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/hubspot/__init__.py +0 -0
  19. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/__init__.py +0 -0
  20. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/api.py +0 -0
  21. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/detectors.py +0 -0
  22. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/engine.py +0 -0
  23. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/intercom/__init__.py +0 -0
  24. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/jira/__init__.py +0 -0
  25. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/linear/__init__.py +0 -0
  26. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/notion/__init__.py +0 -0
  27. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/pagerduty/__init__.py +0 -0
  28. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/salesforce/__init__.py +0 -0
  29. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/servicenow/__init__.py +0 -0
  30. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/state.py +0 -0
  31. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/teams/__init__.py +0 -0
  32. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/tldv/__init__.py +0 -0
  33. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/worker.py +0 -0
  34. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/zendesk/__init__.py +0 -0
  35. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/dependency_links.txt +0 -0
  36. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/entry_points.txt +0 -0
  37. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/requires.txt +0 -0
  38. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/top_level.txt +0 -0
  39. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/setup.cfg +0 -0
  40. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_base.py +0 -0
  41. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_connector_coverage.py +0 -0
  42. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_discord_normalize.py +0 -0
  43. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_github.py +0 -0
  44. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_ingest_e2e.py +0 -0
  45. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_attachments.py +0 -0
  46. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_findings_0_2_11.py +0 -0
  47. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_findings_0_2_9.py +0 -0
  48. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_notion_comments.py +0 -0
  49. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_product_findings_0_2_12.py +0 -0
  50. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_product_findings_0_2_14.py +0 -0
  51. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack.py +0 -0
  52. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack_findings_0_2_16.py +0 -0
  53. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack_findings_0_2_17.py +0 -0
  54. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_streaming_cursor.py +0 -0
  55. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_tldv_findings_0_2_12.py +0 -0
  56. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_tldv_findings_0_2_15.py +0 -0
  57. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_webhooks.py +0 -0
  58. {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_webhooks_replay.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: cortexdb-connectors
3
- Version: 0.2.17
3
+ Version: 0.2.18
4
4
  Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
5
5
  Author-email: CortexDB Team <team@cortexdb.ai>
6
6
  License-Expression: Apache-2.0
@@ -8,7 +8,7 @@ poll loop; or `from cortexdb_connectors.slack import SlackConnector`
8
8
  to drive it programmatically.
9
9
  """
10
10
 
11
- __version__ = "0.2.17"
11
+ __version__ = "0.2.18"
12
12
 
13
13
  from cortexdb_connectors.base import (
14
14
  Actor,
@@ -114,7 +114,7 @@ class VisibilityLevel(str, Enum):
114
114
  @dataclass(frozen=True)
115
115
  class Source:
116
116
  system: str
117
- connector_version: str = "0.2.17"
117
+ connector_version: str = "0.2.18"
118
118
 
119
119
 
120
120
  @dataclass(frozen=True)
@@ -53,7 +53,11 @@ CONNECTORS: dict[str, dict[str, Any]] = {
53
53
  "slack": {
54
54
  "import": "cortexdb_connectors.slack:SlackConnector",
55
55
  "secrets": [("slack_bot_token", "SLACK_BOT_TOKEN")],
56
- "options": [("channels", "SLACK_CHANNELS", "list")],
56
+ "options": [
57
+ ("channels", "SLACK_CHANNELS", "list"),
58
+ ("ingest_files", "SLACK_INGEST_FILES", "bool"),
59
+ ("file_types", "SLACK_FILE_TYPES", "list"),
60
+ ],
57
61
  },
58
62
  "github": {
59
63
  "import": "cortexdb_connectors.github:GitHubConnector",
@@ -1,9 +1,18 @@
1
1
  """
2
2
  Slack connector for CortexDB.
3
3
 
4
- Ingests messages (including threaded replies) from Slack channels into
5
- CortexDB as Episodes. Uses the ``slack_sdk`` WebClient to interact with
6
- the Slack API and respects Tier-3 rate limits (~50 requests/minute).
4
+ Ingests messages (including threaded replies) and shared files from Slack
5
+ channels into CortexDB as Episodes. Uses the ``slack_sdk`` WebClient to
6
+ interact with the Slack API and respects Tier-3 rate limits
7
+ (~50 requests/minute).
8
+
9
+ File shares ride the multimodal path: each ``msg["files"]`` entry becomes
10
+ its own blob-carrying Episode (``Episode.blob`` → ``POST /v1/blobs`` →
11
+ ``content.kind="blob_ref"``) so the server's content processors
12
+ (Tika/vision/ASR) extract and index the bytes — same as Jira attachments.
13
+ Requires the ``files:read`` bot-token scope; without it Slack answers file
14
+ downloads with an HTML login page, which is detected and logged loudly
15
+ (metadata-only ingest, never silent).
7
16
  """
8
17
 
9
18
  from __future__ import annotations
@@ -12,12 +21,14 @@ import asyncio
12
21
  import hashlib
13
22
  import html
14
23
  import logging
24
+ import os
15
25
  import re
16
26
  from datetime import datetime, timedelta, timezone
17
27
  from typing import Any
18
28
 
19
29
  from cortexdb_connectors.base import (
20
30
  Actor,
31
+ Attachment,
21
32
  ConnectorFetchError,
22
33
  CortexConnector,
23
34
  EntityRef,
@@ -56,6 +67,11 @@ _HAS_CONTENT_RE = re.compile(r"[^\W_]", re.UNICODE)
56
67
  # S9: bounded scan for old threads with new replies (see fetch_events).
57
68
  _THREAD_SCAN_PAGE_LIMIT = 5
58
69
 
70
+ # F1: the server's `/v1/blobs` body cap — anything larger 413s. Files over
71
+ # this ingest metadata-only (the filename line survives as text) instead of
72
+ # crashing the sync or being silently dropped.
73
+ _MAX_FILE_BYTES = 32 * 1024 * 1024
74
+
59
75
 
60
76
  def content_version(content: str) -> str:
61
77
  """Short stable digest of the ingested content — the mutable version
@@ -79,8 +95,20 @@ def message_idempotency_key(channel_id: str, ts: str, content: str) -> str:
79
95
  return f"slack:{channel_id}:{ts}:{content_version(content)}"
80
96
 
81
97
 
98
+ def file_idempotency_key(channel_id: str, file_id: str, version: str) -> str:
99
+ """Versioned file key: ``slack:file:{channel}:{file_id}:{version}``.
100
+
101
+ Keyed on the Slack file id (stable across re-fetches of the same share)
102
+ plus an update marker, consistent with the versioned-key scheme (S1b):
103
+ a re-listed unchanged file replays cleanly server-side, while an edited
104
+ editable file (post/canvas) lands as a new version.
105
+ """
106
+ return f"slack:file:{channel_id}:{file_id}:{version}"
107
+
108
+
82
109
  class SlackConnector(CortexConnector):
83
- """Connector that syncs Slack channel messages into CortexDB.
110
+ """Connector that syncs Slack channel messages and shared files into
111
+ CortexDB.
84
112
 
85
113
  Parameters
86
114
  ----------
@@ -89,12 +117,25 @@ class SlackConnector(CortexConnector):
89
117
  cortex_api_key:
90
118
  Bearer token for CortexDB authentication.
91
119
  slack_bot_token:
92
- Slack Bot User OAuth Token (``xoxb-...``).
120
+ Slack Bot User OAuth Token (``xoxb-...``). Needs ``files:read``
121
+ for file-content ingestion (without it, files ingest
122
+ metadata-only and the sync logs the missing scope loudly).
93
123
  channels:
94
124
  Explicit list of channel IDs to sync. If empty, the connector
95
125
  will auto-discover all channels the bot has been added to.
96
126
  scope:
97
127
  CortexDB scope path (hierarchical string, e.g. ``"user:default"``).
128
+ ingest_files:
129
+ Master switch for file-content ingestion (default on; env
130
+ ``SLACK_INGEST_FILES=0`` to disable). Off = pre-0.2.18 behavior,
131
+ as an explicit operator opt-out.
132
+ file_types:
133
+ Cost control: when non-empty, only files matching an entry ingest
134
+ their BYTES (everything else ingests metadata-only, so filenames
135
+ stay searchable at zero media-processing cost). Entries match the
136
+ full mimetype (``application/pdf``), the mime major type
137
+ (``image``, ``audio``), or the file extension (``pdf``). Env:
138
+ ``SLACK_FILE_TYPES`` (comma-separated). Empty = all types.
98
139
  """
99
140
 
100
141
  connector_name: str = "slack"
@@ -121,10 +162,23 @@ class SlackConnector(CortexConnector):
121
162
  slack_bot_token: str,
122
163
  channels: list[str] | None = None,
123
164
  scope: str = "user:default",
165
+ ingest_files: bool | None = None,
166
+ file_types: list[str] | None = None,
124
167
  ) -> None:
125
168
  super().__init__(cortex_url, cortex_api_key, scope)
126
169
  self.slack_bot_token = slack_bot_token
127
170
  self.channels = channels or []
171
+ # F1 cost controls; None = read the env so the webhook path (which
172
+ # builds this connector directly, not via the CLI) honors the same
173
+ # configuration.
174
+ if ingest_files is None:
175
+ env = os.environ.get("SLACK_INGEST_FILES", "").strip().lower()
176
+ ingest_files = env not in ("0", "false", "no", "off") if env else True
177
+ self.ingest_files = ingest_files
178
+ if file_types is None:
179
+ env_types = os.environ.get("SLACK_FILE_TYPES", "")
180
+ file_types = [t for t in (p.strip() for p in env_types.split(",")) if t]
181
+ self.file_types = list(file_types)
128
182
  self._user_cache: dict[str, dict[str, Any]] = {}
129
183
  self._user_cache_populated = False
130
184
  # R3: one conversations.info (+ conversations.members) per channel
@@ -132,6 +186,13 @@ class SlackConnector(CortexConnector):
132
186
  self._channel_info_cache: dict[str, dict[str, Any]] = {}
133
187
  # R5: per-channel failures recorded for the cycle (channel_id → error).
134
188
  self.last_channel_errors: dict[str, str] = {}
189
+ # F1: file RawEvents emitted this run — a thread_broadcast reply is
190
+ # listed by BOTH conversations.history and conversations.replies, and
191
+ # without dedup its files would be downloaded and uploaded twice.
192
+ self._emitted_file_ids: set[str] = set()
193
+ # F1: warn about a missing files:read scope once per run, not once
194
+ # per file.
195
+ self._files_scope_warned = False
135
196
 
136
197
  # -- CortexConnector interface -------------------------------------------
137
198
 
@@ -184,6 +245,8 @@ class SlackConnector(CortexConnector):
184
245
  run_started = datetime.now(timezone.utc)
185
246
  self._channel_info_cache.clear() # roster/privacy may change between runs
186
247
  self.last_channel_errors = {}
248
+ self._emitted_file_ids.clear()
249
+ self._files_scope_warned = False
187
250
 
188
251
  for channel_id in channels:
189
252
  try:
@@ -290,6 +353,9 @@ class SlackConnector(CortexConnector):
290
353
  permissions=dict(permissions),
291
354
  )
292
355
  events.append(raw)
356
+ events.extend(
357
+ self._file_events(msg, channel_id, dict(permissions))
358
+ )
293
359
 
294
360
  # Fetch thread replies if this message has a thread.
295
361
  if msg.get("reply_count", 0) > 0:
@@ -327,14 +393,21 @@ class SlackConnector(CortexConnector):
327
393
  return events
328
394
 
329
395
  def normalize(self, raw: RawEvent) -> Episode | None:
330
- """Convert a Slack message ``RawEvent`` into a CortexDB ``Episode``.
396
+ """Convert a Slack ``RawEvent`` into a CortexDB ``Episode``.
397
+
398
+ File RawEvents (``kind="file"``, emitted per ``msg["files"]`` entry
399
+ by ``fetch_events``) become blob-carrying Episodes on the multimodal
400
+ path. A file-only MESSAGE still returns None here — its content
401
+ rides the separate file episodes, so nothing is lost.
331
402
 
332
403
  Returns None (skip) for messages the first-cut content policy
333
- excludes: join/leave noise, attachment/file-only messages with no
334
- text, and emoji-only messages. Emoji reactions never become
335
- content (they ride ``raw_payload`` only).
404
+ excludes: join/leave noise, messages with no text, and emoji-only
405
+ messages. Emoji reactions never become content (they ride
406
+ ``raw_payload`` only).
336
407
  """
337
408
  data = raw.data
409
+ if data.get("kind") == "file":
410
+ return self._normalize_file(raw)
338
411
  text = self._render_text(data)
339
412
  if text is None:
340
413
  return None
@@ -456,6 +529,233 @@ class SlackConnector(CortexConnector):
456
529
  )
457
530
  return Visibility(level=VisibilityLevel.ORGANIZATION)
458
531
 
532
+ # -- file ingestion (F1) -------------------------------------------------
533
+
534
+ def _file_events(
535
+ self,
536
+ msg: dict[str, Any],
537
+ channel_id: str,
538
+ permissions: dict[str, Any],
539
+ ) -> list[RawEvent]:
540
+ """One ``kind="file"`` RawEvent per file shared on *msg*.
541
+
542
+ Bytes are downloaded here, mirroring Jira: fetch downloads,
543
+ ``normalize`` wraps them in ``Episode.blob``, ``ingest_episode``
544
+ uploads to ``/v1/blobs``. Oversize files, external shares with no
545
+ downloadable URL, and failed downloads degrade to metadata-only
546
+ events — loudly logged, never silently dropped, never sync-fatal.
547
+ """
548
+ if not self.ingest_files:
549
+ return []
550
+ events: list[RawEvent] = []
551
+ ts = str(msg.get("ts", ""))
552
+ for f in msg.get("files") or []:
553
+ file_id = f.get("id")
554
+ if not file_id:
555
+ continue
556
+ if f.get("mode") == "tombstone" or f.get("file_access") not in (
557
+ None,
558
+ "visible",
559
+ ):
560
+ # Deleted or access-revoked at the source — nothing to fetch.
561
+ continue
562
+ external_id = f"{channel_id}:{ts}:file:{file_id}"
563
+ if external_id in self._emitted_file_ids:
564
+ continue
565
+ self._emitted_file_ids.add(external_id)
566
+
567
+ name = f.get("name") or f.get("title") or file_id
568
+ mime = f.get("mimetype") or "application/octet-stream"
569
+ size = int(f.get("size") or 0)
570
+ url = f.get("url_private_download") or f.get("url_private")
571
+ data_bytes: bytes | None = None
572
+ if not url:
573
+ logger.warning(
574
+ "slack file has no downloadable URL (external share?): "
575
+ "channel=%s file=%s — filename recorded, bytes NOT "
576
+ "ingested",
577
+ channel_id,
578
+ name,
579
+ )
580
+ elif self.file_types and not self._file_type_allowed(mime, name):
581
+ logger.info(
582
+ "slack file filtered by file-type policy "
583
+ "(SLACK_FILE_TYPES=%s): channel=%s file=%s mime=%s — "
584
+ "metadata-only",
585
+ ",".join(self.file_types),
586
+ channel_id,
587
+ name,
588
+ mime,
589
+ )
590
+ elif size > _MAX_FILE_BYTES:
591
+ logger.error(
592
+ "slack file SKIPPED (too large): channel=%s file=%s "
593
+ "size=%d limit=%d — filename recorded, bytes NOT ingested",
594
+ channel_id,
595
+ name,
596
+ size,
597
+ _MAX_FILE_BYTES,
598
+ )
599
+ else:
600
+ data_bytes = self._download_file(url, name, mime)
601
+
602
+ events.append(
603
+ RawEvent(
604
+ source="slack",
605
+ external_id=external_id,
606
+ timestamp=datetime.fromtimestamp(
607
+ float(ts or "0"), tz=timezone.utc
608
+ ),
609
+ data={
610
+ "kind": "file",
611
+ "file_id": file_id,
612
+ "name": name,
613
+ "title": f.get("title") or "",
614
+ "mimetype": mime,
615
+ "size": size,
616
+ "user": f.get("user") or msg.get("user") or "",
617
+ "version": str(
618
+ f.get("updated")
619
+ or f.get("timestamp")
620
+ or f.get("created")
621
+ or ""
622
+ ),
623
+ "permalink": f.get("permalink") or "",
624
+ "message_ts": ts,
625
+ "thread_ts": msg.get("thread_ts") or "",
626
+ "_channel_id": channel_id,
627
+ "bytes": data_bytes,
628
+ },
629
+ permissions=permissions,
630
+ )
631
+ )
632
+ return events
633
+
634
+ def _file_type_allowed(self, mime: str, name: str) -> bool:
635
+ """Does *mime*/*name* match the ``file_types`` allowlist?
636
+
637
+ An entry matches the full mimetype (``application/pdf``), the mime
638
+ major type (``image``), or the file extension (``pdf``).
639
+ """
640
+ allowed = {t.lower().lstrip(".") for t in self.file_types}
641
+ mime = mime.lower()
642
+ ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
643
+ return (
644
+ mime in allowed
645
+ or mime.split("/", 1)[0] in allowed
646
+ or (bool(ext) and ext in allowed)
647
+ )
648
+
649
+ def _download_file(self, url: str, name: str, mime: str) -> bytes | None:
650
+ """GET a Slack file's bytes with the bot token; None on any failure.
651
+
652
+ A token without the ``files:read`` scope gets a 200 HTML login page
653
+ instead of the bytes — detected via content-type (unless the file
654
+ itself is HTML) so an auth page is never ingested as content.
655
+ """
656
+ import httpx
657
+
658
+ try:
659
+ resp = httpx.get(
660
+ url,
661
+ headers={"Authorization": f"Bearer {self.slack_bot_token}"},
662
+ follow_redirects=True,
663
+ timeout=120.0,
664
+ )
665
+ resp.raise_for_status()
666
+ except Exception as exc: # noqa: BLE001
667
+ logger.error(
668
+ "slack file download FAILED: file=%s: %s — filename recorded, "
669
+ "bytes NOT ingested",
670
+ name,
671
+ exc,
672
+ )
673
+ return None
674
+ ctype = (resp.headers.get("content-type") or "").lower()
675
+ if "text/html" in ctype and "html" not in mime.lower():
676
+ if not self._files_scope_warned:
677
+ self._files_scope_warned = True
678
+ logger.error(
679
+ "slack file download returned an HTML page instead of "
680
+ "bytes — the bot token is likely missing the files:read "
681
+ "scope. File content will NOT be ingested until it is "
682
+ "granted (file=%s).",
683
+ name,
684
+ )
685
+ return None
686
+ return resp.content
687
+
688
+ def _normalize_file(self, raw: RawEvent) -> Episode:
689
+ """File RawEvent → blob-carrying Episode (``content.kind="blob_ref"``).
690
+
691
+ Without bytes (oversize / no-URL / download failure) the episode
692
+ degrades to its text line so the filename is never lost — same
693
+ contract as Jira attachments.
694
+ """
695
+ data = raw.data
696
+ channel_id = data.get("_channel_id", "")
697
+ file_id = data.get("file_id", "")
698
+ name = data.get("name", "file")
699
+ mime = data.get("mimetype", "application/octet-stream")
700
+ size = int(data.get("size", 0) or 0)
701
+ user_id = data.get("user") or "unknown"
702
+ sharer = self._display_name(user_id)
703
+ title = data.get("title") or ""
704
+ message_ts = data.get("message_ts", "")
705
+ thread_ts = data.get("thread_ts") or None
706
+
707
+ content = (
708
+ f"[Slack file] {name} ({mime}, {size} bytes) shared by {sharer} "
709
+ f"in {data.get('_channel_name', channel_id)}"
710
+ )
711
+ if title and title != name:
712
+ content += f": {title}"
713
+
714
+ entities = [
715
+ EntityRef(
716
+ entity_type="channel",
717
+ entity_id=channel_id,
718
+ display_name=data.get("_channel_name", channel_id),
719
+ )
720
+ ]
721
+ if data.get("user"):
722
+ entities.append(
723
+ EntityRef(
724
+ entity_type="user", entity_id=user_id, display_name=sharer
725
+ )
726
+ )
727
+
728
+ return Episode(
729
+ actor=self._make_actor(user_id),
730
+ source=_SOURCE,
731
+ episode_type=EpisodeType.DOCUMENT,
732
+ occurred_at=raw.timestamp,
733
+ content=content,
734
+ raw_payload={k: v for k, v in data.items() if k != "bytes"},
735
+ scope=self.scope,
736
+ namespace="slack",
737
+ entities=entities,
738
+ tags=["slack", "file"],
739
+ metadata={
740
+ "file_id": file_id,
741
+ "size_bytes": size,
742
+ },
743
+ thread_id=f"{channel_id}:{thread_ts}" if thread_ts else None,
744
+ # The carrying message, so recall can hop file → conversation.
745
+ parent_id=f"{channel_id}:{message_ts}" if message_ts else None,
746
+ idempotency_key=file_idempotency_key(
747
+ channel_id, file_id, str(data.get("version", ""))
748
+ ),
749
+ blob=Attachment(
750
+ filename=name,
751
+ mime_type=mime,
752
+ size_bytes=size,
753
+ data=data.get("bytes"),
754
+ source_url=data.get("permalink") or None,
755
+ attachment_id=file_id,
756
+ ),
757
+ )
758
+
459
759
  # -- helpers -------------------------------------------------------------
460
760
 
461
761
  def _discover_channels(self, client: Any) -> list[str]:
@@ -590,6 +890,10 @@ class SlackConnector(CortexConnector):
590
890
  # Skip the parent message itself (already captured).
591
891
  if msg.get("ts") == thread_ts and not msg.get("parent_user_id"):
592
892
  continue
893
+ reply_permissions = {
894
+ "is_private": channel_info.get("is_private", False),
895
+ "channel_members": channel_info.get("members", []),
896
+ }
593
897
  raw = RawEvent(
594
898
  source="slack",
595
899
  external_id=f"{channel_id}:{msg.get('ts', '')}",
@@ -597,12 +901,12 @@ class SlackConnector(CortexConnector):
597
901
  float(msg.get("ts", "0")), tz=timezone.utc
598
902
  ),
599
903
  data={**msg, "_channel_id": channel_id},
600
- permissions={
601
- "is_private": channel_info.get("is_private", False),
602
- "channel_members": channel_info.get("members", []),
603
- },
904
+ permissions=reply_permissions,
604
905
  )
605
906
  events.append(raw)
907
+ events.extend(
908
+ self._file_events(msg, channel_id, dict(reply_permissions))
909
+ )
606
910
 
607
911
  next_cursor = resp.get("response_metadata", {}).get("next_cursor")
608
912
  if not next_cursor:
@@ -119,6 +119,8 @@ class SlackWebhookHandler:
119
119
  - Respond to Slack URL verification challenges.
120
120
  - Verify request authenticity using the Slack signing secret.
121
121
  - Normalize incoming event_callback events into CortexDB Episodes.
122
+ - Ingest shared files through the multimodal blob path (F1), reusing
123
+ the poll connector's pipeline for key/content parity.
122
124
  - Forward episodes to the CortexDB ingestion API.
123
125
  """
124
126
 
@@ -231,14 +233,19 @@ class SlackWebhookHandler:
231
233
  return JSONResponse({"ok": True})
232
234
 
233
235
  try:
234
- episode = self.normalize(payload, event)
236
+ episode: Episode | None = self.normalize(payload, event)
237
+ files, file_msg = self._slack_files(event, event_type)
235
238
  # The server rejects empty-text envelopes (422, permanent) --
236
- # attachment/file-only messages must be acked-and-skipped, not
237
- # forwarded. Slack retries non-2xx deliveries, so failing here
238
- # would also cause redelivery storms.
239
- if not episode.content.strip():
240
- logger.debug("skipping text-less Slack event %s", event_type)
241
- return JSONResponse({"ok": True, "skipped": "empty-text"})
239
+ # the text episode of a text-less message must be skipped, not
240
+ # forwarded (Slack retries non-2xx deliveries, so failing here
241
+ # would also cause redelivery storms). F1: a file-only message
242
+ # is no longer a full skip — its content rides the file
243
+ # episodes ingested below.
244
+ if episode is not None and not episode.content.strip():
245
+ if not files:
246
+ logger.debug("skipping text-less Slack event %s", event_type)
247
+ return JSONResponse({"ok": True, "skipped": "empty-text"})
248
+ episode = None
242
249
  except Exception:
243
250
  # normalize is deterministic — a failure here would fail the
244
251
  # redelivery identically, so retrying can't help. Ack and log
@@ -250,35 +257,86 @@ class SlackWebhookHandler:
250
257
  )
251
258
  return JSONResponse({"ok": True, "skipped": "normalize-failed"})
252
259
 
253
- # T3: ack now, ingest in the background.
254
- task = asyncio.create_task(self._ingest_background(episode, event_type))
260
+ # T3: ack now, ingest (text episode + any shared files) in the
261
+ # background — file downloads especially must stay off the 3 s ack
262
+ # path.
263
+ task = asyncio.create_task(
264
+ self._ingest_background(
265
+ episode,
266
+ event_type,
267
+ file_msg=file_msg,
268
+ channel_id=event.get("channel", ""),
269
+ )
270
+ )
255
271
  self._pending_ingests.add(task)
256
272
  task.add_done_callback(self._pending_ingests.discard)
257
273
  return JSONResponse({"ok": True, "queued": True})
258
274
 
275
+ @staticmethod
276
+ def _slack_files(
277
+ event: dict[str, Any], event_type: str
278
+ ) -> tuple[list[dict[str, Any]], dict[str, Any]]:
279
+ """F1: the files on a message-family event + a container-message
280
+ dict shaped for ``SlackConnector._file_events`` (poll parity).
281
+
282
+ For ``message_changed`` the files ride the nested ``event.message``
283
+ (same place the edited text lives, S1a).
284
+ """
285
+ if event_type not in ("message", "message_changed"):
286
+ return [], {}
287
+ msg = event.get("message", {}) if event_type == "message_changed" else event
288
+ files = [f for f in (msg.get("files") or []) if isinstance(f, dict)]
289
+ if not files:
290
+ return [], {}
291
+ container = {
292
+ "ts": msg.get("ts") or event.get("ts") or event.get("event_ts", "0"),
293
+ "user": msg.get("user") or event.get("user", ""),
294
+ "thread_ts": msg.get("thread_ts") or event.get("thread_ts"),
295
+ "files": files,
296
+ }
297
+ return files, container
298
+
259
299
  # Delays between background ingest attempts (T3). Bounded: after the
260
300
  # last attempt the failure is logged loudly — the delivery was already
261
301
  # acked, so the poll connector is the recovery path.
262
302
  _INGEST_RETRY_DELAYS = (1.0, 5.0, 25.0)
263
303
  _VIS_CACHE_TTL_SECS = 900.0
264
304
 
265
- async def _ingest_background(self, episode: Episode, event_type: str) -> None:
305
+ async def _ingest_background(
306
+ self,
307
+ episode: Episode | None,
308
+ event_type: str,
309
+ file_msg: dict[str, Any] | None = None,
310
+ channel_id: str = "",
311
+ ) -> None:
312
+ if episode is not None and episode.entities:
313
+ channel_id = channel_id or episode.entities[0].entity_id
266
314
  # N3: resolve channel privacy → visibility OFF the ack path (it may
267
315
  # call the Slack API). Failures keep the episode's default.
316
+ vis = None
268
317
  try:
269
318
  vis = await asyncio.to_thread(
270
- self._resolve_channel_visibility,
271
- episode.entities[0].entity_id if episode.entities else "",
319
+ self._resolve_channel_visibility, channel_id
272
320
  )
273
- if vis is not None:
274
- episode.visibility = vis
275
321
  except Exception: # noqa: BLE001
276
322
  logger.exception("slack webhook: visibility resolution failed")
277
323
 
324
+ if episode is not None:
325
+ if vis is not None:
326
+ episode.visibility = vis
327
+ await self._ingest_with_retries(
328
+ lambda: self._ingest(episode), event_type, episode.idempotency_key
329
+ )
330
+ if file_msg:
331
+ await self._ingest_files_background(file_msg, channel_id, vis, event_type)
332
+
333
+ async def _ingest_with_retries(
334
+ self, attempt_fn: Any, event_type: str, idempotency_key: str | None
335
+ ) -> None:
278
336
  last_error: BaseException | None = None
279
337
  for attempt in range(len(self._INGEST_RETRY_DELAYS) + 1):
280
338
  try:
281
- await self._ingest(episode)
339
+ await attempt_fn()
282
340
  return
283
341
  except httpx.HTTPStatusError as exc:
284
342
  if exc.response is not None and exc.response.status_code == 409:
@@ -299,10 +357,69 @@ class SlackWebhookHandler:
299
357
  "running",
300
358
  len(self._INGEST_RETRY_DELAYS) + 1,
301
359
  event_type,
302
- episode.idempotency_key,
360
+ idempotency_key,
303
361
  last_error,
304
362
  )
305
363
 
364
+ async def _ingest_files_background(
365
+ self,
366
+ file_msg: dict[str, Any],
367
+ channel_id: str,
368
+ vis: Any,
369
+ event_type: str,
370
+ ) -> None:
371
+ """F1: download + ingest a delivery's shared files off the ack path.
372
+
373
+ Reuses the poll connector's file pipeline (download →
374
+ ``Episode.blob`` → ``/v1/blobs`` → ``blob_ref``) with a fresh
375
+ per-delivery ``SlackConnector``, so webhook and poll produce
376
+ identical episodes under identical versioned keys — whichever path
377
+ lands first, the other replays. Needs ``SLACK_BOT_TOKEN`` for the
378
+ downloads; without it the files are left to the poll connector,
379
+ loudly.
380
+ """
381
+ token = os.environ.get("SLACK_BOT_TOKEN", "")
382
+ if not token:
383
+ if not getattr(self, "_files_token_warned", False):
384
+ self._files_token_warned = True
385
+ logger.error(
386
+ "slack webhook: file(s) shared but SLACK_BOT_TOKEN is not "
387
+ "set — file content cannot be downloaded on the webhook "
388
+ "path; the poll connector will pick them up if running"
389
+ )
390
+ return
391
+ from cortexdb_connectors.slack import SlackConnector
392
+
393
+ conn = SlackConnector(
394
+ self.cortexdb_url, self.cortexdb_api_key, token, scope=self.scope
395
+ )
396
+ conn.actor = self.actor
397
+ conn.scope_template = self.scope or self.actor
398
+ try:
399
+ raws = await asyncio.to_thread(
400
+ conn._file_events, file_msg, channel_id, {}
401
+ )
402
+ except Exception: # noqa: BLE001
403
+ logger.exception("slack webhook: file download failed")
404
+ return
405
+ for raw in raws:
406
+ try:
407
+ ep = conn.normalize(raw)
408
+ except Exception: # noqa: BLE001
409
+ logger.exception(
410
+ "slack webhook: file normalize failed for %s", raw.external_id
411
+ )
412
+ continue
413
+ if ep is None:
414
+ continue
415
+ if vis is not None:
416
+ ep.visibility = vis
417
+ await self._ingest_with_retries(
418
+ lambda ep=ep: conn.ingest_episode(ep),
419
+ f"{event_type}:file",
420
+ ep.idempotency_key,
421
+ )
422
+
306
423
  def _resolve_channel_visibility(self, channel_id: str):
307
424
  """N3: derive visibility from real channel privacy (poll parity).
308
425