cortexdb-connectors 0.2.17__tar.gz → 0.2.18__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/PKG-INFO +1 -1
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/__init__.py +1 -1
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/base.py +1 -1
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/cli.py +5 -1
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/slack/__init__.py +317 -13
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/webhooks.py +133 -16
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/PKG-INFO +1 -1
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/SOURCES.txt +1 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/pyproject.toml +1 -1
- cortexdb_connectors-0.2.18/tests/test_slack_file_ingestion.py +599 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/README.md +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/confluence/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/discord/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/freshdesk/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/github/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/gitlab/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/google_workspace/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/hubspot/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/api.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/detectors.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/insights/engine.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/intercom/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/jira/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/linear/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/notion/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/pagerduty/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/salesforce/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/servicenow/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/state.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/teams/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/tldv/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/worker.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/zendesk/__init__.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/dependency_links.txt +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/entry_points.txt +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/requires.txt +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors.egg-info/top_level.txt +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/setup.cfg +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_base.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_connector_coverage.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_discord_normalize.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_github.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_ingest_e2e.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_attachments.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_findings_0_2_11.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_jira_findings_0_2_9.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_notion_comments.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_product_findings_0_2_12.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_product_findings_0_2_14.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack_findings_0_2_16.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_slack_findings_0_2_17.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_streaming_cursor.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_tldv_findings_0_2_12.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_tldv_findings_0_2_15.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_webhooks.py +0 -0
- {cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/tests/test_webhooks_replay.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: cortexdb-connectors
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.18
|
|
4
4
|
Summary: Data connectors for CortexDB — ingest from Slack, GitHub, GitLab, Jira, Linear, Confluence, Notion, PagerDuty, Discord, Teams, Google Workspace, Salesforce, HubSpot, Zendesk, Intercom, and ServiceNow into the v1 memory API.
|
|
5
5
|
Author-email: CortexDB Team <team@cortexdb.ai>
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -53,7 +53,11 @@ CONNECTORS: dict[str, dict[str, Any]] = {
|
|
|
53
53
|
"slack": {
|
|
54
54
|
"import": "cortexdb_connectors.slack:SlackConnector",
|
|
55
55
|
"secrets": [("slack_bot_token", "SLACK_BOT_TOKEN")],
|
|
56
|
-
"options": [
|
|
56
|
+
"options": [
|
|
57
|
+
("channels", "SLACK_CHANNELS", "list"),
|
|
58
|
+
("ingest_files", "SLACK_INGEST_FILES", "bool"),
|
|
59
|
+
("file_types", "SLACK_FILE_TYPES", "list"),
|
|
60
|
+
],
|
|
57
61
|
},
|
|
58
62
|
"github": {
|
|
59
63
|
"import": "cortexdb_connectors.github:GitHubConnector",
|
{cortexdb_connectors-0.2.17 → cortexdb_connectors-0.2.18}/cortexdb_connectors/slack/__init__.py
RENAMED
|
@@ -1,9 +1,18 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Slack connector for CortexDB.
|
|
3
3
|
|
|
4
|
-
Ingests messages (including threaded replies) from Slack
|
|
5
|
-
CortexDB as Episodes. Uses the ``slack_sdk`` WebClient to
|
|
6
|
-
the Slack API and respects Tier-3 rate limits
|
|
4
|
+
Ingests messages (including threaded replies) and shared files from Slack
|
|
5
|
+
channels into CortexDB as Episodes. Uses the ``slack_sdk`` WebClient to
|
|
6
|
+
interact with the Slack API and respects Tier-3 rate limits
|
|
7
|
+
(~50 requests/minute).
|
|
8
|
+
|
|
9
|
+
File shares ride the multimodal path: each ``msg["files"]`` entry becomes
|
|
10
|
+
its own blob-carrying Episode (``Episode.blob`` → ``POST /v1/blobs`` →
|
|
11
|
+
``content.kind="blob_ref"``) so the server's content processors
|
|
12
|
+
(Tika/vision/ASR) extract and index the bytes — same as Jira attachments.
|
|
13
|
+
Requires the ``files:read`` bot-token scope; without it Slack answers file
|
|
14
|
+
downloads with an HTML login page, which is detected and logged loudly
|
|
15
|
+
(metadata-only ingest, never silent).
|
|
7
16
|
"""
|
|
8
17
|
|
|
9
18
|
from __future__ import annotations
|
|
@@ -12,12 +21,14 @@ import asyncio
|
|
|
12
21
|
import hashlib
|
|
13
22
|
import html
|
|
14
23
|
import logging
|
|
24
|
+
import os
|
|
15
25
|
import re
|
|
16
26
|
from datetime import datetime, timedelta, timezone
|
|
17
27
|
from typing import Any
|
|
18
28
|
|
|
19
29
|
from cortexdb_connectors.base import (
|
|
20
30
|
Actor,
|
|
31
|
+
Attachment,
|
|
21
32
|
ConnectorFetchError,
|
|
22
33
|
CortexConnector,
|
|
23
34
|
EntityRef,
|
|
@@ -56,6 +67,11 @@ _HAS_CONTENT_RE = re.compile(r"[^\W_]", re.UNICODE)
|
|
|
56
67
|
# S9: bounded scan for old threads with new replies (see fetch_events).
|
|
57
68
|
_THREAD_SCAN_PAGE_LIMIT = 5
|
|
58
69
|
|
|
70
|
+
# F1: the server's `/v1/blobs` body cap — anything larger 413s. Files over
|
|
71
|
+
# this ingest metadata-only (the filename line survives as text) instead of
|
|
72
|
+
# crashing the sync or being silently dropped.
|
|
73
|
+
_MAX_FILE_BYTES = 32 * 1024 * 1024
|
|
74
|
+
|
|
59
75
|
|
|
60
76
|
def content_version(content: str) -> str:
|
|
61
77
|
"""Short stable digest of the ingested content — the mutable version
|
|
@@ -79,8 +95,20 @@ def message_idempotency_key(channel_id: str, ts: str, content: str) -> str:
|
|
|
79
95
|
return f"slack:{channel_id}:{ts}:{content_version(content)}"
|
|
80
96
|
|
|
81
97
|
|
|
98
|
+
def file_idempotency_key(channel_id: str, file_id: str, version: str) -> str:
|
|
99
|
+
"""Versioned file key: ``slack:file:{channel}:{file_id}:{version}``.
|
|
100
|
+
|
|
101
|
+
Keyed on the Slack file id (stable across re-fetches of the same share)
|
|
102
|
+
plus an update marker, consistent with the versioned-key scheme (S1b):
|
|
103
|
+
a re-listed unchanged file replays cleanly server-side, while an edited
|
|
104
|
+
editable file (post/canvas) lands as a new version.
|
|
105
|
+
"""
|
|
106
|
+
return f"slack:file:{channel_id}:{file_id}:{version}"
|
|
107
|
+
|
|
108
|
+
|
|
82
109
|
class SlackConnector(CortexConnector):
|
|
83
|
-
"""Connector that syncs Slack channel messages into
|
|
110
|
+
"""Connector that syncs Slack channel messages and shared files into
|
|
111
|
+
CortexDB.
|
|
84
112
|
|
|
85
113
|
Parameters
|
|
86
114
|
----------
|
|
@@ -89,12 +117,25 @@ class SlackConnector(CortexConnector):
|
|
|
89
117
|
cortex_api_key:
|
|
90
118
|
Bearer token for CortexDB authentication.
|
|
91
119
|
slack_bot_token:
|
|
92
|
-
Slack Bot User OAuth Token (``xoxb-...``).
|
|
120
|
+
Slack Bot User OAuth Token (``xoxb-...``). Needs ``files:read``
|
|
121
|
+
for file-content ingestion (without it, files ingest
|
|
122
|
+
metadata-only and the sync logs the missing scope loudly).
|
|
93
123
|
channels:
|
|
94
124
|
Explicit list of channel IDs to sync. If empty, the connector
|
|
95
125
|
will auto-discover all channels the bot has been added to.
|
|
96
126
|
scope:
|
|
97
127
|
CortexDB scope path (hierarchical string, e.g. ``"user:default"``).
|
|
128
|
+
ingest_files:
|
|
129
|
+
Master switch for file-content ingestion (default on; env
|
|
130
|
+
``SLACK_INGEST_FILES=0`` to disable). Off = pre-0.2.18 behavior,
|
|
131
|
+
as an explicit operator opt-out.
|
|
132
|
+
file_types:
|
|
133
|
+
Cost control: when non-empty, only files matching an entry ingest
|
|
134
|
+
their BYTES (everything else ingests metadata-only, so filenames
|
|
135
|
+
stay searchable at zero media-processing cost). Entries match the
|
|
136
|
+
full mimetype (``application/pdf``), the mime major type
|
|
137
|
+
(``image``, ``audio``), or the file extension (``pdf``). Env:
|
|
138
|
+
``SLACK_FILE_TYPES`` (comma-separated). Empty = all types.
|
|
98
139
|
"""
|
|
99
140
|
|
|
100
141
|
connector_name: str = "slack"
|
|
@@ -121,10 +162,23 @@ class SlackConnector(CortexConnector):
|
|
|
121
162
|
slack_bot_token: str,
|
|
122
163
|
channels: list[str] | None = None,
|
|
123
164
|
scope: str = "user:default",
|
|
165
|
+
ingest_files: bool | None = None,
|
|
166
|
+
file_types: list[str] | None = None,
|
|
124
167
|
) -> None:
|
|
125
168
|
super().__init__(cortex_url, cortex_api_key, scope)
|
|
126
169
|
self.slack_bot_token = slack_bot_token
|
|
127
170
|
self.channels = channels or []
|
|
171
|
+
# F1 cost controls; None = read the env so the webhook path (which
|
|
172
|
+
# builds this connector directly, not via the CLI) honors the same
|
|
173
|
+
# configuration.
|
|
174
|
+
if ingest_files is None:
|
|
175
|
+
env = os.environ.get("SLACK_INGEST_FILES", "").strip().lower()
|
|
176
|
+
ingest_files = env not in ("0", "false", "no", "off") if env else True
|
|
177
|
+
self.ingest_files = ingest_files
|
|
178
|
+
if file_types is None:
|
|
179
|
+
env_types = os.environ.get("SLACK_FILE_TYPES", "")
|
|
180
|
+
file_types = [t for t in (p.strip() for p in env_types.split(",")) if t]
|
|
181
|
+
self.file_types = list(file_types)
|
|
128
182
|
self._user_cache: dict[str, dict[str, Any]] = {}
|
|
129
183
|
self._user_cache_populated = False
|
|
130
184
|
# R3: one conversations.info (+ conversations.members) per channel
|
|
@@ -132,6 +186,13 @@ class SlackConnector(CortexConnector):
|
|
|
132
186
|
self._channel_info_cache: dict[str, dict[str, Any]] = {}
|
|
133
187
|
# R5: per-channel failures recorded for the cycle (channel_id → error).
|
|
134
188
|
self.last_channel_errors: dict[str, str] = {}
|
|
189
|
+
# F1: file RawEvents emitted this run — a thread_broadcast reply is
|
|
190
|
+
# listed by BOTH conversations.history and conversations.replies, and
|
|
191
|
+
# without dedup its files would be downloaded and uploaded twice.
|
|
192
|
+
self._emitted_file_ids: set[str] = set()
|
|
193
|
+
# F1: warn about a missing files:read scope once per run, not once
|
|
194
|
+
# per file.
|
|
195
|
+
self._files_scope_warned = False
|
|
135
196
|
|
|
136
197
|
# -- CortexConnector interface -------------------------------------------
|
|
137
198
|
|
|
@@ -184,6 +245,8 @@ class SlackConnector(CortexConnector):
|
|
|
184
245
|
run_started = datetime.now(timezone.utc)
|
|
185
246
|
self._channel_info_cache.clear() # roster/privacy may change between runs
|
|
186
247
|
self.last_channel_errors = {}
|
|
248
|
+
self._emitted_file_ids.clear()
|
|
249
|
+
self._files_scope_warned = False
|
|
187
250
|
|
|
188
251
|
for channel_id in channels:
|
|
189
252
|
try:
|
|
@@ -290,6 +353,9 @@ class SlackConnector(CortexConnector):
|
|
|
290
353
|
permissions=dict(permissions),
|
|
291
354
|
)
|
|
292
355
|
events.append(raw)
|
|
356
|
+
events.extend(
|
|
357
|
+
self._file_events(msg, channel_id, dict(permissions))
|
|
358
|
+
)
|
|
293
359
|
|
|
294
360
|
# Fetch thread replies if this message has a thread.
|
|
295
361
|
if msg.get("reply_count", 0) > 0:
|
|
@@ -327,14 +393,21 @@ class SlackConnector(CortexConnector):
|
|
|
327
393
|
return events
|
|
328
394
|
|
|
329
395
|
def normalize(self, raw: RawEvent) -> Episode | None:
|
|
330
|
-
"""Convert a Slack
|
|
396
|
+
"""Convert a Slack ``RawEvent`` into a CortexDB ``Episode``.
|
|
397
|
+
|
|
398
|
+
File RawEvents (``kind="file"``, emitted per ``msg["files"]`` entry
|
|
399
|
+
by ``fetch_events``) become blob-carrying Episodes on the multimodal
|
|
400
|
+
path. A file-only MESSAGE still returns None here — its content
|
|
401
|
+
rides the separate file episodes, so nothing is lost.
|
|
331
402
|
|
|
332
403
|
Returns None (skip) for messages the first-cut content policy
|
|
333
|
-
excludes: join/leave noise,
|
|
334
|
-
|
|
335
|
-
|
|
404
|
+
excludes: join/leave noise, messages with no text, and emoji-only
|
|
405
|
+
messages. Emoji reactions never become content (they ride
|
|
406
|
+
``raw_payload`` only).
|
|
336
407
|
"""
|
|
337
408
|
data = raw.data
|
|
409
|
+
if data.get("kind") == "file":
|
|
410
|
+
return self._normalize_file(raw)
|
|
338
411
|
text = self._render_text(data)
|
|
339
412
|
if text is None:
|
|
340
413
|
return None
|
|
@@ -456,6 +529,233 @@ class SlackConnector(CortexConnector):
|
|
|
456
529
|
)
|
|
457
530
|
return Visibility(level=VisibilityLevel.ORGANIZATION)
|
|
458
531
|
|
|
532
|
+
# -- file ingestion (F1) -------------------------------------------------
|
|
533
|
+
|
|
534
|
+
def _file_events(
|
|
535
|
+
self,
|
|
536
|
+
msg: dict[str, Any],
|
|
537
|
+
channel_id: str,
|
|
538
|
+
permissions: dict[str, Any],
|
|
539
|
+
) -> list[RawEvent]:
|
|
540
|
+
"""One ``kind="file"`` RawEvent per file shared on *msg*.
|
|
541
|
+
|
|
542
|
+
Bytes are downloaded here, mirroring Jira: fetch downloads,
|
|
543
|
+
``normalize`` wraps them in ``Episode.blob``, ``ingest_episode``
|
|
544
|
+
uploads to ``/v1/blobs``. Oversize files, external shares with no
|
|
545
|
+
downloadable URL, and failed downloads degrade to metadata-only
|
|
546
|
+
events — loudly logged, never silently dropped, never sync-fatal.
|
|
547
|
+
"""
|
|
548
|
+
if not self.ingest_files:
|
|
549
|
+
return []
|
|
550
|
+
events: list[RawEvent] = []
|
|
551
|
+
ts = str(msg.get("ts", ""))
|
|
552
|
+
for f in msg.get("files") or []:
|
|
553
|
+
file_id = f.get("id")
|
|
554
|
+
if not file_id:
|
|
555
|
+
continue
|
|
556
|
+
if f.get("mode") == "tombstone" or f.get("file_access") not in (
|
|
557
|
+
None,
|
|
558
|
+
"visible",
|
|
559
|
+
):
|
|
560
|
+
# Deleted or access-revoked at the source — nothing to fetch.
|
|
561
|
+
continue
|
|
562
|
+
external_id = f"{channel_id}:{ts}:file:{file_id}"
|
|
563
|
+
if external_id in self._emitted_file_ids:
|
|
564
|
+
continue
|
|
565
|
+
self._emitted_file_ids.add(external_id)
|
|
566
|
+
|
|
567
|
+
name = f.get("name") or f.get("title") or file_id
|
|
568
|
+
mime = f.get("mimetype") or "application/octet-stream"
|
|
569
|
+
size = int(f.get("size") or 0)
|
|
570
|
+
url = f.get("url_private_download") or f.get("url_private")
|
|
571
|
+
data_bytes: bytes | None = None
|
|
572
|
+
if not url:
|
|
573
|
+
logger.warning(
|
|
574
|
+
"slack file has no downloadable URL (external share?): "
|
|
575
|
+
"channel=%s file=%s — filename recorded, bytes NOT "
|
|
576
|
+
"ingested",
|
|
577
|
+
channel_id,
|
|
578
|
+
name,
|
|
579
|
+
)
|
|
580
|
+
elif self.file_types and not self._file_type_allowed(mime, name):
|
|
581
|
+
logger.info(
|
|
582
|
+
"slack file filtered by file-type policy "
|
|
583
|
+
"(SLACK_FILE_TYPES=%s): channel=%s file=%s mime=%s — "
|
|
584
|
+
"metadata-only",
|
|
585
|
+
",".join(self.file_types),
|
|
586
|
+
channel_id,
|
|
587
|
+
name,
|
|
588
|
+
mime,
|
|
589
|
+
)
|
|
590
|
+
elif size > _MAX_FILE_BYTES:
|
|
591
|
+
logger.error(
|
|
592
|
+
"slack file SKIPPED (too large): channel=%s file=%s "
|
|
593
|
+
"size=%d limit=%d — filename recorded, bytes NOT ingested",
|
|
594
|
+
channel_id,
|
|
595
|
+
name,
|
|
596
|
+
size,
|
|
597
|
+
_MAX_FILE_BYTES,
|
|
598
|
+
)
|
|
599
|
+
else:
|
|
600
|
+
data_bytes = self._download_file(url, name, mime)
|
|
601
|
+
|
|
602
|
+
events.append(
|
|
603
|
+
RawEvent(
|
|
604
|
+
source="slack",
|
|
605
|
+
external_id=external_id,
|
|
606
|
+
timestamp=datetime.fromtimestamp(
|
|
607
|
+
float(ts or "0"), tz=timezone.utc
|
|
608
|
+
),
|
|
609
|
+
data={
|
|
610
|
+
"kind": "file",
|
|
611
|
+
"file_id": file_id,
|
|
612
|
+
"name": name,
|
|
613
|
+
"title": f.get("title") or "",
|
|
614
|
+
"mimetype": mime,
|
|
615
|
+
"size": size,
|
|
616
|
+
"user": f.get("user") or msg.get("user") or "",
|
|
617
|
+
"version": str(
|
|
618
|
+
f.get("updated")
|
|
619
|
+
or f.get("timestamp")
|
|
620
|
+
or f.get("created")
|
|
621
|
+
or ""
|
|
622
|
+
),
|
|
623
|
+
"permalink": f.get("permalink") or "",
|
|
624
|
+
"message_ts": ts,
|
|
625
|
+
"thread_ts": msg.get("thread_ts") or "",
|
|
626
|
+
"_channel_id": channel_id,
|
|
627
|
+
"bytes": data_bytes,
|
|
628
|
+
},
|
|
629
|
+
permissions=permissions,
|
|
630
|
+
)
|
|
631
|
+
)
|
|
632
|
+
return events
|
|
633
|
+
|
|
634
|
+
def _file_type_allowed(self, mime: str, name: str) -> bool:
|
|
635
|
+
"""Does *mime*/*name* match the ``file_types`` allowlist?
|
|
636
|
+
|
|
637
|
+
An entry matches the full mimetype (``application/pdf``), the mime
|
|
638
|
+
major type (``image``), or the file extension (``pdf``).
|
|
639
|
+
"""
|
|
640
|
+
allowed = {t.lower().lstrip(".") for t in self.file_types}
|
|
641
|
+
mime = mime.lower()
|
|
642
|
+
ext = name.rsplit(".", 1)[-1].lower() if "." in name else ""
|
|
643
|
+
return (
|
|
644
|
+
mime in allowed
|
|
645
|
+
or mime.split("/", 1)[0] in allowed
|
|
646
|
+
or (bool(ext) and ext in allowed)
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
def _download_file(self, url: str, name: str, mime: str) -> bytes | None:
|
|
650
|
+
"""GET a Slack file's bytes with the bot token; None on any failure.
|
|
651
|
+
|
|
652
|
+
A token without the ``files:read`` scope gets a 200 HTML login page
|
|
653
|
+
instead of the bytes — detected via content-type (unless the file
|
|
654
|
+
itself is HTML) so an auth page is never ingested as content.
|
|
655
|
+
"""
|
|
656
|
+
import httpx
|
|
657
|
+
|
|
658
|
+
try:
|
|
659
|
+
resp = httpx.get(
|
|
660
|
+
url,
|
|
661
|
+
headers={"Authorization": f"Bearer {self.slack_bot_token}"},
|
|
662
|
+
follow_redirects=True,
|
|
663
|
+
timeout=120.0,
|
|
664
|
+
)
|
|
665
|
+
resp.raise_for_status()
|
|
666
|
+
except Exception as exc: # noqa: BLE001
|
|
667
|
+
logger.error(
|
|
668
|
+
"slack file download FAILED: file=%s: %s — filename recorded, "
|
|
669
|
+
"bytes NOT ingested",
|
|
670
|
+
name,
|
|
671
|
+
exc,
|
|
672
|
+
)
|
|
673
|
+
return None
|
|
674
|
+
ctype = (resp.headers.get("content-type") or "").lower()
|
|
675
|
+
if "text/html" in ctype and "html" not in mime.lower():
|
|
676
|
+
if not self._files_scope_warned:
|
|
677
|
+
self._files_scope_warned = True
|
|
678
|
+
logger.error(
|
|
679
|
+
"slack file download returned an HTML page instead of "
|
|
680
|
+
"bytes — the bot token is likely missing the files:read "
|
|
681
|
+
"scope. File content will NOT be ingested until it is "
|
|
682
|
+
"granted (file=%s).",
|
|
683
|
+
name,
|
|
684
|
+
)
|
|
685
|
+
return None
|
|
686
|
+
return resp.content
|
|
687
|
+
|
|
688
|
+
def _normalize_file(self, raw: RawEvent) -> Episode:
|
|
689
|
+
"""File RawEvent → blob-carrying Episode (``content.kind="blob_ref"``).
|
|
690
|
+
|
|
691
|
+
Without bytes (oversize / no-URL / download failure) the episode
|
|
692
|
+
degrades to its text line so the filename is never lost — same
|
|
693
|
+
contract as Jira attachments.
|
|
694
|
+
"""
|
|
695
|
+
data = raw.data
|
|
696
|
+
channel_id = data.get("_channel_id", "")
|
|
697
|
+
file_id = data.get("file_id", "")
|
|
698
|
+
name = data.get("name", "file")
|
|
699
|
+
mime = data.get("mimetype", "application/octet-stream")
|
|
700
|
+
size = int(data.get("size", 0) or 0)
|
|
701
|
+
user_id = data.get("user") or "unknown"
|
|
702
|
+
sharer = self._display_name(user_id)
|
|
703
|
+
title = data.get("title") or ""
|
|
704
|
+
message_ts = data.get("message_ts", "")
|
|
705
|
+
thread_ts = data.get("thread_ts") or None
|
|
706
|
+
|
|
707
|
+
content = (
|
|
708
|
+
f"[Slack file] {name} ({mime}, {size} bytes) shared by {sharer} "
|
|
709
|
+
f"in {data.get('_channel_name', channel_id)}"
|
|
710
|
+
)
|
|
711
|
+
if title and title != name:
|
|
712
|
+
content += f": {title}"
|
|
713
|
+
|
|
714
|
+
entities = [
|
|
715
|
+
EntityRef(
|
|
716
|
+
entity_type="channel",
|
|
717
|
+
entity_id=channel_id,
|
|
718
|
+
display_name=data.get("_channel_name", channel_id),
|
|
719
|
+
)
|
|
720
|
+
]
|
|
721
|
+
if data.get("user"):
|
|
722
|
+
entities.append(
|
|
723
|
+
EntityRef(
|
|
724
|
+
entity_type="user", entity_id=user_id, display_name=sharer
|
|
725
|
+
)
|
|
726
|
+
)
|
|
727
|
+
|
|
728
|
+
return Episode(
|
|
729
|
+
actor=self._make_actor(user_id),
|
|
730
|
+
source=_SOURCE,
|
|
731
|
+
episode_type=EpisodeType.DOCUMENT,
|
|
732
|
+
occurred_at=raw.timestamp,
|
|
733
|
+
content=content,
|
|
734
|
+
raw_payload={k: v for k, v in data.items() if k != "bytes"},
|
|
735
|
+
scope=self.scope,
|
|
736
|
+
namespace="slack",
|
|
737
|
+
entities=entities,
|
|
738
|
+
tags=["slack", "file"],
|
|
739
|
+
metadata={
|
|
740
|
+
"file_id": file_id,
|
|
741
|
+
"size_bytes": size,
|
|
742
|
+
},
|
|
743
|
+
thread_id=f"{channel_id}:{thread_ts}" if thread_ts else None,
|
|
744
|
+
# The carrying message, so recall can hop file → conversation.
|
|
745
|
+
parent_id=f"{channel_id}:{message_ts}" if message_ts else None,
|
|
746
|
+
idempotency_key=file_idempotency_key(
|
|
747
|
+
channel_id, file_id, str(data.get("version", ""))
|
|
748
|
+
),
|
|
749
|
+
blob=Attachment(
|
|
750
|
+
filename=name,
|
|
751
|
+
mime_type=mime,
|
|
752
|
+
size_bytes=size,
|
|
753
|
+
data=data.get("bytes"),
|
|
754
|
+
source_url=data.get("permalink") or None,
|
|
755
|
+
attachment_id=file_id,
|
|
756
|
+
),
|
|
757
|
+
)
|
|
758
|
+
|
|
459
759
|
# -- helpers -------------------------------------------------------------
|
|
460
760
|
|
|
461
761
|
def _discover_channels(self, client: Any) -> list[str]:
|
|
@@ -590,6 +890,10 @@ class SlackConnector(CortexConnector):
|
|
|
590
890
|
# Skip the parent message itself (already captured).
|
|
591
891
|
if msg.get("ts") == thread_ts and not msg.get("parent_user_id"):
|
|
592
892
|
continue
|
|
893
|
+
reply_permissions = {
|
|
894
|
+
"is_private": channel_info.get("is_private", False),
|
|
895
|
+
"channel_members": channel_info.get("members", []),
|
|
896
|
+
}
|
|
593
897
|
raw = RawEvent(
|
|
594
898
|
source="slack",
|
|
595
899
|
external_id=f"{channel_id}:{msg.get('ts', '')}",
|
|
@@ -597,12 +901,12 @@ class SlackConnector(CortexConnector):
|
|
|
597
901
|
float(msg.get("ts", "0")), tz=timezone.utc
|
|
598
902
|
),
|
|
599
903
|
data={**msg, "_channel_id": channel_id},
|
|
600
|
-
permissions=
|
|
601
|
-
"is_private": channel_info.get("is_private", False),
|
|
602
|
-
"channel_members": channel_info.get("members", []),
|
|
603
|
-
},
|
|
904
|
+
permissions=reply_permissions,
|
|
604
905
|
)
|
|
605
906
|
events.append(raw)
|
|
907
|
+
events.extend(
|
|
908
|
+
self._file_events(msg, channel_id, dict(reply_permissions))
|
|
909
|
+
)
|
|
606
910
|
|
|
607
911
|
next_cursor = resp.get("response_metadata", {}).get("next_cursor")
|
|
608
912
|
if not next_cursor:
|
|
@@ -119,6 +119,8 @@ class SlackWebhookHandler:
|
|
|
119
119
|
- Respond to Slack URL verification challenges.
|
|
120
120
|
- Verify request authenticity using the Slack signing secret.
|
|
121
121
|
- Normalize incoming event_callback events into CortexDB Episodes.
|
|
122
|
+
- Ingest shared files through the multimodal blob path (F1), reusing
|
|
123
|
+
the poll connector's pipeline for key/content parity.
|
|
122
124
|
- Forward episodes to the CortexDB ingestion API.
|
|
123
125
|
"""
|
|
124
126
|
|
|
@@ -231,14 +233,19 @@ class SlackWebhookHandler:
|
|
|
231
233
|
return JSONResponse({"ok": True})
|
|
232
234
|
|
|
233
235
|
try:
|
|
234
|
-
episode = self.normalize(payload, event)
|
|
236
|
+
episode: Episode | None = self.normalize(payload, event)
|
|
237
|
+
files, file_msg = self._slack_files(event, event_type)
|
|
235
238
|
# The server rejects empty-text envelopes (422, permanent) --
|
|
236
|
-
#
|
|
237
|
-
# forwarded
|
|
238
|
-
# would also cause redelivery storms.
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
239
|
+
# the text episode of a text-less message must be skipped, not
|
|
240
|
+
# forwarded (Slack retries non-2xx deliveries, so failing here
|
|
241
|
+
# would also cause redelivery storms). F1: a file-only message
|
|
242
|
+
# is no longer a full skip — its content rides the file
|
|
243
|
+
# episodes ingested below.
|
|
244
|
+
if episode is not None and not episode.content.strip():
|
|
245
|
+
if not files:
|
|
246
|
+
logger.debug("skipping text-less Slack event %s", event_type)
|
|
247
|
+
return JSONResponse({"ok": True, "skipped": "empty-text"})
|
|
248
|
+
episode = None
|
|
242
249
|
except Exception:
|
|
243
250
|
# normalize is deterministic — a failure here would fail the
|
|
244
251
|
# redelivery identically, so retrying can't help. Ack and log
|
|
@@ -250,35 +257,86 @@ class SlackWebhookHandler:
|
|
|
250
257
|
)
|
|
251
258
|
return JSONResponse({"ok": True, "skipped": "normalize-failed"})
|
|
252
259
|
|
|
253
|
-
# T3: ack now, ingest in the
|
|
254
|
-
|
|
260
|
+
# T3: ack now, ingest (text episode + any shared files) in the
|
|
261
|
+
# background — file downloads especially must stay off the 3 s ack
|
|
262
|
+
# path.
|
|
263
|
+
task = asyncio.create_task(
|
|
264
|
+
self._ingest_background(
|
|
265
|
+
episode,
|
|
266
|
+
event_type,
|
|
267
|
+
file_msg=file_msg,
|
|
268
|
+
channel_id=event.get("channel", ""),
|
|
269
|
+
)
|
|
270
|
+
)
|
|
255
271
|
self._pending_ingests.add(task)
|
|
256
272
|
task.add_done_callback(self._pending_ingests.discard)
|
|
257
273
|
return JSONResponse({"ok": True, "queued": True})
|
|
258
274
|
|
|
275
|
+
@staticmethod
|
|
276
|
+
def _slack_files(
|
|
277
|
+
event: dict[str, Any], event_type: str
|
|
278
|
+
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
|
279
|
+
"""F1: the files on a message-family event + a container-message
|
|
280
|
+
dict shaped for ``SlackConnector._file_events`` (poll parity).
|
|
281
|
+
|
|
282
|
+
For ``message_changed`` the files ride the nested ``event.message``
|
|
283
|
+
(same place the edited text lives, S1a).
|
|
284
|
+
"""
|
|
285
|
+
if event_type not in ("message", "message_changed"):
|
|
286
|
+
return [], {}
|
|
287
|
+
msg = event.get("message", {}) if event_type == "message_changed" else event
|
|
288
|
+
files = [f for f in (msg.get("files") or []) if isinstance(f, dict)]
|
|
289
|
+
if not files:
|
|
290
|
+
return [], {}
|
|
291
|
+
container = {
|
|
292
|
+
"ts": msg.get("ts") or event.get("ts") or event.get("event_ts", "0"),
|
|
293
|
+
"user": msg.get("user") or event.get("user", ""),
|
|
294
|
+
"thread_ts": msg.get("thread_ts") or event.get("thread_ts"),
|
|
295
|
+
"files": files,
|
|
296
|
+
}
|
|
297
|
+
return files, container
|
|
298
|
+
|
|
259
299
|
# Delays between background ingest attempts (T3). Bounded: after the
|
|
260
300
|
# last attempt the failure is logged loudly — the delivery was already
|
|
261
301
|
# acked, so the poll connector is the recovery path.
|
|
262
302
|
_INGEST_RETRY_DELAYS = (1.0, 5.0, 25.0)
|
|
263
303
|
_VIS_CACHE_TTL_SECS = 900.0
|
|
264
304
|
|
|
265
|
-
async def _ingest_background(
|
|
305
|
+
async def _ingest_background(
|
|
306
|
+
self,
|
|
307
|
+
episode: Episode | None,
|
|
308
|
+
event_type: str,
|
|
309
|
+
file_msg: dict[str, Any] | None = None,
|
|
310
|
+
channel_id: str = "",
|
|
311
|
+
) -> None:
|
|
312
|
+
if episode is not None and episode.entities:
|
|
313
|
+
channel_id = channel_id or episode.entities[0].entity_id
|
|
266
314
|
# N3: resolve channel privacy → visibility OFF the ack path (it may
|
|
267
315
|
# call the Slack API). Failures keep the episode's default.
|
|
316
|
+
vis = None
|
|
268
317
|
try:
|
|
269
318
|
vis = await asyncio.to_thread(
|
|
270
|
-
self._resolve_channel_visibility,
|
|
271
|
-
episode.entities[0].entity_id if episode.entities else "",
|
|
319
|
+
self._resolve_channel_visibility, channel_id
|
|
272
320
|
)
|
|
273
|
-
if vis is not None:
|
|
274
|
-
episode.visibility = vis
|
|
275
321
|
except Exception: # noqa: BLE001
|
|
276
322
|
logger.exception("slack webhook: visibility resolution failed")
|
|
277
323
|
|
|
324
|
+
if episode is not None:
|
|
325
|
+
if vis is not None:
|
|
326
|
+
episode.visibility = vis
|
|
327
|
+
await self._ingest_with_retries(
|
|
328
|
+
lambda: self._ingest(episode), event_type, episode.idempotency_key
|
|
329
|
+
)
|
|
330
|
+
if file_msg:
|
|
331
|
+
await self._ingest_files_background(file_msg, channel_id, vis, event_type)
|
|
332
|
+
|
|
333
|
+
async def _ingest_with_retries(
|
|
334
|
+
self, attempt_fn: Any, event_type: str, idempotency_key: str | None
|
|
335
|
+
) -> None:
|
|
278
336
|
last_error: BaseException | None = None
|
|
279
337
|
for attempt in range(len(self._INGEST_RETRY_DELAYS) + 1):
|
|
280
338
|
try:
|
|
281
|
-
await
|
|
339
|
+
await attempt_fn()
|
|
282
340
|
return
|
|
283
341
|
except httpx.HTTPStatusError as exc:
|
|
284
342
|
if exc.response is not None and exc.response.status_code == 409:
|
|
@@ -299,10 +357,69 @@ class SlackWebhookHandler:
|
|
|
299
357
|
"running",
|
|
300
358
|
len(self._INGEST_RETRY_DELAYS) + 1,
|
|
301
359
|
event_type,
|
|
302
|
-
|
|
360
|
+
idempotency_key,
|
|
303
361
|
last_error,
|
|
304
362
|
)
|
|
305
363
|
|
|
364
|
+
async def _ingest_files_background(
|
|
365
|
+
self,
|
|
366
|
+
file_msg: dict[str, Any],
|
|
367
|
+
channel_id: str,
|
|
368
|
+
vis: Any,
|
|
369
|
+
event_type: str,
|
|
370
|
+
) -> None:
|
|
371
|
+
"""F1: download + ingest a delivery's shared files off the ack path.
|
|
372
|
+
|
|
373
|
+
Reuses the poll connector's file pipeline (download →
|
|
374
|
+
``Episode.blob`` → ``/v1/blobs`` → ``blob_ref``) with a fresh
|
|
375
|
+
per-delivery ``SlackConnector``, so webhook and poll produce
|
|
376
|
+
identical episodes under identical versioned keys — whichever path
|
|
377
|
+
lands first, the other replays. Needs ``SLACK_BOT_TOKEN`` for the
|
|
378
|
+
downloads; without it the files are left to the poll connector,
|
|
379
|
+
loudly.
|
|
380
|
+
"""
|
|
381
|
+
token = os.environ.get("SLACK_BOT_TOKEN", "")
|
|
382
|
+
if not token:
|
|
383
|
+
if not getattr(self, "_files_token_warned", False):
|
|
384
|
+
self._files_token_warned = True
|
|
385
|
+
logger.error(
|
|
386
|
+
"slack webhook: file(s) shared but SLACK_BOT_TOKEN is not "
|
|
387
|
+
"set — file content cannot be downloaded on the webhook "
|
|
388
|
+
"path; the poll connector will pick them up if running"
|
|
389
|
+
)
|
|
390
|
+
return
|
|
391
|
+
from cortexdb_connectors.slack import SlackConnector
|
|
392
|
+
|
|
393
|
+
conn = SlackConnector(
|
|
394
|
+
self.cortexdb_url, self.cortexdb_api_key, token, scope=self.scope
|
|
395
|
+
)
|
|
396
|
+
conn.actor = self.actor
|
|
397
|
+
conn.scope_template = self.scope or self.actor
|
|
398
|
+
try:
|
|
399
|
+
raws = await asyncio.to_thread(
|
|
400
|
+
conn._file_events, file_msg, channel_id, {}
|
|
401
|
+
)
|
|
402
|
+
except Exception: # noqa: BLE001
|
|
403
|
+
logger.exception("slack webhook: file download failed")
|
|
404
|
+
return
|
|
405
|
+
for raw in raws:
|
|
406
|
+
try:
|
|
407
|
+
ep = conn.normalize(raw)
|
|
408
|
+
except Exception: # noqa: BLE001
|
|
409
|
+
logger.exception(
|
|
410
|
+
"slack webhook: file normalize failed for %s", raw.external_id
|
|
411
|
+
)
|
|
412
|
+
continue
|
|
413
|
+
if ep is None:
|
|
414
|
+
continue
|
|
415
|
+
if vis is not None:
|
|
416
|
+
ep.visibility = vis
|
|
417
|
+
await self._ingest_with_retries(
|
|
418
|
+
lambda ep=ep: conn.ingest_episode(ep),
|
|
419
|
+
f"{event_type}:file",
|
|
420
|
+
ep.idempotency_key,
|
|
421
|
+
)
|
|
422
|
+
|
|
306
423
|
def _resolve_channel_visibility(self, channel_id: str):
|
|
307
424
|
"""N3: derive visibility from real channel privacy (poll parity).
|
|
308
425
|
|