cellcog 2.8.0__tar.gz → 2.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {cellcog-2.8.0 → cellcog-2.8.2}/PKG-INFO +1 -1
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/__init__.py +1 -1
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/docs/api_reference.md +2 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/files.py +159 -37
- {cellcog-2.8.0 → cellcog-2.8.2}/pyproject.toml +1 -1
- cellcog-2.8.2/tests/test_bare_path_scan.py +66 -0
- cellcog-2.8.2/tests/test_storage_fallback.py +186 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/.gitignore +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/LICENSE +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/README.md +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/auth.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/chat.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/client.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/config.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/daemon/__init__.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/daemon/delivery.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/daemon/main.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/daemon/state.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/docs/__init__.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/docs/support.md +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/exceptions.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/message_formatter.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/message_processor.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/state_paths.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/telemetry.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/cellcog/version_detection.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/examples/basic_usage.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/examples/file_handling.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/examples/streaming_usage.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/examples/sync_usage.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/examples/workflow_usage.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/tests/test_gs_url_handling.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/tests/test_support_and_credits.py +0 -0
- {cellcog-2.8.0 → cellcog-2.8.2}/tests/test_telemetry.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: cellcog
|
|
3
|
-
Version: 2.8.
|
|
3
|
+
Version: 2.8.2
|
|
4
4
|
Summary: Python SDK for CellCog - Any-to-Any AI for agents. Fire-and-forget pattern with WebSocket notifications.
|
|
5
5
|
Project-URL: Homepage, https://cellcog.ai
|
|
6
6
|
Project-URL: Documentation, https://cellcog.ai/developer/docs
|
|
@@ -77,6 +77,8 @@ Prompts may embed files for the agent with `<SHOW_FILE>/abs/path</SHOW_FILE>` ta
|
|
|
77
77
|
|
|
78
78
|
- **`hc://` paths do NOT resolve in SDK-created chats.** hc:// resolution is scoped to the chat that owns the Desktop (Co-work) session; a chat created via `create_chat()` has no such session, even when your own orchestrating chat does. The working pattern: stage the files into a workspace the new chat can reach first (e.g. FileTransfer them into your container), then reference the CONTAINER paths in the prompt — the platform uploads and renders them for the sub-agent. This also keeps prompts small (a 6-file design brief shrank 143K to 10K chars vs embedding sources inline).
|
|
79
79
|
- **Prompt size cap: 50,000 characters** (`create_chat` returns 422 `string_too_long` beyond it). Embedding source files inline is the usual way to hit it — use the staging pattern above instead.
|
|
80
|
+
- **Restricted networks (SDK 2.8.1+):** files normally go straight to CellCog's file storage. When your environment allows `cellcog.ai` but blocks that storage (Claude's code-execution sandbox does), the SDK sends and fetches the same files through `cellcog.ai` instead, so `cellcog.ai` is the only domain you need to allow. Uploads through `cellcog.ai` are limited to 100 MB per file.
|
|
81
|
+
- **A file that cannot be uploaded stops the send (SDK 2.8.1+):** `create_chat()` and `send_message()` raise `FileUploadError` naming each file and the reason, and nothing is sent. A result file that cannot be downloaded keeps its path in the message, followed by a `[cellcog: this file was not downloaded: <reason>]` line.
|
|
80
82
|
|
|
81
83
|
### send_message()
|
|
82
84
|
|
|
@@ -18,6 +18,34 @@ from .state_paths import state_path
|
|
|
18
18
|
from .telemetry import host_of, report_error
|
|
19
19
|
|
|
20
20
|
|
|
21
|
+
# CEL-3867 (Nov 2026): some environments allow cellcog.ai but refuse storage.googleapis.com (Claude's code-execution
|
|
22
|
+
# sandbox: its proxy answers 403 to the CONNECT). Uploads and downloads then go through cellcog.ai instead.
|
|
23
|
+
PROXY_UPLOAD_MAX_BYTES = 100 * 1024 * 1024 # the server's cap for bytes sent through cellcog.ai
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _is_storage_block(exc: BaseException) -> bool:
|
|
27
|
+
"""A connection that never reached storage (proxy refusal, DNS, refused connect), as opposed to an HTTP answer."""
|
|
28
|
+
return isinstance(exc, requests.exceptions.ConnectionError)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _short_reason(exc: BaseException) -> str:
|
|
32
|
+
"""One readable reason, never a signed URL."""
|
|
33
|
+
resp = getattr(exc, "response", None)
|
|
34
|
+
code = getattr(resp, "status_code", None)
|
|
35
|
+
if isinstance(code, int):
|
|
36
|
+
detail = ""
|
|
37
|
+
try:
|
|
38
|
+
detail = str((resp.json() or {}).get("detail") or "")[:200]
|
|
39
|
+
except Exception: # noqa: BLE001
|
|
40
|
+
pass
|
|
41
|
+
return f"HTTP {code}" + (f": {detail}" if detail else "")
|
|
42
|
+
if isinstance(exc, requests.exceptions.ProxyError):
|
|
43
|
+
return "the network proxy refused the connection"
|
|
44
|
+
if isinstance(exc, requests.exceptions.Timeout):
|
|
45
|
+
return "the connection timed out"
|
|
46
|
+
return type(exc).__name__
|
|
47
|
+
|
|
48
|
+
|
|
21
49
|
class FileProcessor:
|
|
22
50
|
"""
|
|
23
51
|
Handles file upload/download and path translation for CellCog SDK.
|
|
@@ -39,6 +67,8 @@ class FileProcessor:
|
|
|
39
67
|
def __init__(self, config: Config):
|
|
40
68
|
self.config = config
|
|
41
69
|
self.default_download_dir = state_path("chats")
|
|
70
|
+
# Set after the first blocked storage connection: the rest of this run goes through cellcog.ai directly.
|
|
71
|
+
self._storage_blocked = False
|
|
42
72
|
|
|
43
73
|
def transform_outgoing(self, message: str) -> tuple[str, list]:
|
|
44
74
|
"""
|
|
@@ -63,6 +93,7 @@ class FileProcessor:
|
|
|
63
93
|
"""
|
|
64
94
|
uploaded = []
|
|
65
95
|
missing_files = []
|
|
96
|
+
failed_uploads = []
|
|
66
97
|
|
|
67
98
|
def replace_show_file(match):
|
|
68
99
|
attrs = match.group(1)
|
|
@@ -86,8 +117,9 @@ class FileProcessor:
|
|
|
86
117
|
|
|
87
118
|
# Add external_local_path to preserve original path for history restoration
|
|
88
119
|
return f'<SHOW_FILE external_local_path="{file_path}">{blob_name}</SHOW_FILE>'
|
|
89
|
-
except FileUploadError:
|
|
90
|
-
#
|
|
120
|
+
except FileUploadError as e:
|
|
121
|
+
# Never send a dead local path (CEL-3867): collect it and fail loudly below.
|
|
122
|
+
failed_uploads.append((file_path, str(e)))
|
|
91
123
|
return match.group(0)
|
|
92
124
|
except (OSError, ValueError):
|
|
93
125
|
# os.path.exists() can raise on severely malformed paths
|
|
@@ -169,11 +201,18 @@ class FileProcessor:
|
|
|
169
201
|
|
|
170
202
|
# Match paths starting with / or ~/ that contain at least one / separator
|
|
171
203
|
# and are not already inside XML-like tags
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
)
|
|
204
|
+
# Scan only the text OUTSIDE existing file tags (Nov 6 2026): inside a tag, a Windows path such as
|
|
205
|
+
# external_local_path="C:/Users/me/a.webp" contains "/Users/me/a.webp", which the bare-path pattern matched
|
|
206
|
+
# and wrapped in a second tag INSIDE the attribute, so the chat rendered broken tag text.
|
|
207
|
+
_bare_path_re = re.compile(r"(?<![<\"=])(?:/[\w.+\-][\w.+\-/]*|~/[\w.+\-][\w.+\-/]*)")
|
|
208
|
+
_tag_span_re = re.compile(r"<(SHOW_FILE|GENERATE_FILE)\b[^>]*>.*?</\1>", re.DOTALL)
|
|
209
|
+
_parts, _pos = [], 0
|
|
210
|
+
for _m in _tag_span_re.finditer(transformed):
|
|
211
|
+
_parts.append(_bare_path_re.sub(_auto_upload_bare_path, transformed[_pos:_m.start()]))
|
|
212
|
+
_parts.append(_m.group(0))
|
|
213
|
+
_pos = _m.end()
|
|
214
|
+
_parts.append(_bare_path_re.sub(_auto_upload_bare_path, transformed[_pos:]))
|
|
215
|
+
transformed = "".join(_parts)
|
|
177
216
|
|
|
178
217
|
# Fail early if any SHOW_FILE-referenced local files are missing
|
|
179
218
|
if missing_files:
|
|
@@ -189,6 +228,15 @@ class FileProcessor:
|
|
|
189
228
|
f"access is available."
|
|
190
229
|
)
|
|
191
230
|
|
|
231
|
+
# Fail loudly if any SHOW_FILE file exists but could not be uploaded (CEL-3867)
|
|
232
|
+
if failed_uploads:
|
|
233
|
+
files_list = "\n".join(f" - {path}: {why}" for path, why in failed_uploads)
|
|
234
|
+
raise FileUploadError(
|
|
235
|
+
f"Cannot send this message: {len(failed_uploads)} file(s) failed to upload to CellCog:\n"
|
|
236
|
+
f"{files_list}\n\n"
|
|
237
|
+
f"Nothing was sent to CellCog. Fix the cause and send again."
|
|
238
|
+
)
|
|
239
|
+
|
|
192
240
|
return transformed, uploaded
|
|
193
241
|
|
|
194
242
|
def transform_incoming_history(
|
|
@@ -248,11 +296,10 @@ class FileProcessor:
|
|
|
248
296
|
if is_cellcog_message and should_download and blob_name in blob_name_to_url:
|
|
249
297
|
url_data = blob_name_to_url[blob_name]
|
|
250
298
|
try:
|
|
251
|
-
self._download_file(url_data["url"], local_path)
|
|
252
|
-
except FileDownloadError:
|
|
253
|
-
#
|
|
254
|
-
|
|
255
|
-
pass
|
|
299
|
+
self._download_file(url_data["url"], local_path, chat_id=chat_id, ref=blob_name)
|
|
300
|
+
except FileDownloadError as e:
|
|
301
|
+
# Loud, never silent (CEL-3867): say the file was not saved, and why.
|
|
302
|
+
return f"<SHOW_FILE>{local_path}</SHOW_FILE>\n[cellcog: this file was not downloaded: {e}]"
|
|
256
303
|
|
|
257
304
|
# Return with local path (remove blob_name and attributes)
|
|
258
305
|
return f"<SHOW_FILE>{local_path}</SHOW_FILE>"
|
|
@@ -355,20 +402,28 @@ class FileProcessor:
|
|
|
355
402
|
mime=mime_type, size=file_size)
|
|
356
403
|
raise FileUploadError(f"Failed to get upload URL: {e}")
|
|
357
404
|
|
|
358
|
-
# Step 2: Upload to signed URL
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
405
|
+
# Step 2: Upload the bytes. Direct to the signed storage URL first; a network that refuses storage falls back to
|
|
406
|
+
# cellcog.ai, and the rest of this run skips the direct attempt (CEL-3867).
|
|
407
|
+
direct_ok = False
|
|
408
|
+
if not self._storage_blocked:
|
|
409
|
+
try:
|
|
410
|
+
with open(local_path, "rb") as f:
|
|
411
|
+
put_resp = requests.put(
|
|
412
|
+
data["upload_url"],
|
|
413
|
+
data=f,
|
|
414
|
+
headers={"Content-Type": mime_type},
|
|
415
|
+
timeout=300, # 5 min timeout for large files
|
|
416
|
+
)
|
|
417
|
+
put_resp.raise_for_status()
|
|
418
|
+
direct_ok = True
|
|
419
|
+
except requests.RequestException as e:
|
|
420
|
+
report_error(self.config, "upload_put", e, host=host_of(data.get("upload_url")), path=local_path,
|
|
421
|
+
mime=mime_type, size=file_size)
|
|
422
|
+
if not _is_storage_block(e):
|
|
423
|
+
raise FileUploadError(f"Failed to upload {path.name}: {_short_reason(e)}")
|
|
424
|
+
self._storage_blocked = True
|
|
425
|
+
if not direct_ok:
|
|
426
|
+
self._upload_via_cellcog(local_path, data["file_id"], mime_type, file_size)
|
|
372
427
|
|
|
373
428
|
# Step 3: Confirm upload
|
|
374
429
|
try:
|
|
@@ -385,35 +440,102 @@ class FileProcessor:
|
|
|
385
440
|
|
|
386
441
|
return data["blob_name"]
|
|
387
442
|
|
|
388
|
-
def
|
|
443
|
+
def _upload_via_cellcog(self, local_path: str, file_id: str, mime_type: str, file_size: int) -> None:
|
|
444
|
+
"""Send a pending upload's bytes through cellcog.ai (storage.googleapis.com unreachable from here)."""
|
|
445
|
+
name = Path(local_path).name
|
|
446
|
+
if file_size > PROXY_UPLOAD_MAX_BYTES:
|
|
447
|
+
raise FileUploadError(
|
|
448
|
+
f"Cannot upload {name} ({file_size / (1024 * 1024):.1f} MB): this environment blocks CellCog's file "
|
|
449
|
+
f"storage, and uploads through cellcog.ai are limited to {PROXY_UPLOAD_MAX_BYTES // (1024 * 1024)} MB. "
|
|
450
|
+
f"Send a smaller file."
|
|
451
|
+
)
|
|
452
|
+
api = self.config.api_base_url
|
|
453
|
+
try:
|
|
454
|
+
with open(local_path, "rb") as f:
|
|
455
|
+
resp = requests.put(
|
|
456
|
+
f"{api}/files/upload-bytes/{file_id}",
|
|
457
|
+
data=f,
|
|
458
|
+
headers={**self.config.get_request_headers(), "Content-Type": mime_type},
|
|
459
|
+
timeout=300,
|
|
460
|
+
)
|
|
461
|
+
resp.raise_for_status()
|
|
462
|
+
except requests.RequestException as e:
|
|
463
|
+
report_error(self.config, "upload_fallback", e, host=host_of(api), path=local_path, mime=mime_type,
|
|
464
|
+
size=file_size)
|
|
465
|
+
raise FileUploadError(f"Failed to upload {name} through cellcog.ai: {_short_reason(e)}")
|
|
466
|
+
|
|
467
|
+
def _download_file(self, url: str, local_path: str, chat_id: Optional[str] = None,
|
|
468
|
+
ref: Optional[str] = None) -> None:
|
|
389
469
|
"""
|
|
390
470
|
Download file from URL to local path.
|
|
391
471
|
|
|
392
472
|
Args:
|
|
393
473
|
url: Signed URL to download from
|
|
394
474
|
local_path: Local path to save file
|
|
475
|
+
chat_id: The chat the file belongs to (enables the cellcog.ai fallback)
|
|
476
|
+
ref: The SHOW_FILE reference as it appears in the chat history (enables the cellcog.ai fallback)
|
|
395
477
|
|
|
396
478
|
Raises:
|
|
397
479
|
FileDownloadError: If download fails
|
|
398
480
|
"""
|
|
399
481
|
try:
|
|
400
|
-
# Create parent directories
|
|
401
482
|
Path(local_path).parent.mkdir(parents=True, exist_ok=True)
|
|
483
|
+
except OSError as e:
|
|
484
|
+
report_error(self.config, "download_save", e, host=host_of(url), path=local_path)
|
|
485
|
+
raise FileDownloadError(f"could not create the folder for it ({type(e).__name__})")
|
|
402
486
|
|
|
403
|
-
|
|
404
|
-
|
|
487
|
+
can_fall_back = bool(chat_id and ref)
|
|
488
|
+
if not self._storage_blocked:
|
|
489
|
+
try:
|
|
490
|
+
resp = requests.get(url, stream=True, timeout=300)
|
|
491
|
+
resp.raise_for_status()
|
|
492
|
+
self._save_stream(resp, local_path, url)
|
|
493
|
+
return
|
|
494
|
+
except requests.RequestException as e:
|
|
495
|
+
report_error(self.config, "download", e, host=host_of(url), path=local_path)
|
|
496
|
+
if not (_is_storage_block(e) and can_fall_back):
|
|
497
|
+
raise FileDownloadError(f"download failed ({_short_reason(e)})")
|
|
498
|
+
self._storage_blocked = True
|
|
499
|
+
if not can_fall_back:
|
|
500
|
+
raise FileDownloadError("download failed (CellCog's file storage is not reachable from this environment)")
|
|
501
|
+
|
|
502
|
+
# CEL-3867: the same file through cellcog.ai, same access rules as the chat history.
|
|
503
|
+
api = self.config.api_base_url
|
|
504
|
+
try:
|
|
505
|
+
resp = requests.get(
|
|
506
|
+
f"{api}/cellcog/chat/{chat_id}/file",
|
|
507
|
+
params={"ref": ref},
|
|
508
|
+
headers=self.config.get_request_headers(),
|
|
509
|
+
stream=True,
|
|
510
|
+
timeout=300,
|
|
511
|
+
)
|
|
405
512
|
resp.raise_for_status()
|
|
513
|
+
self._save_stream(resp, local_path, api)
|
|
514
|
+
except requests.RequestException as e:
|
|
515
|
+
report_error(self.config, "download_fallback", e, host=host_of(api), path=local_path, chat_id=chat_id)
|
|
516
|
+
raise FileDownloadError(f"download through cellcog.ai failed ({_short_reason(e)})")
|
|
406
517
|
|
|
407
|
-
|
|
518
|
+
def _save_stream(self, resp, local_path: str, source_url: str) -> None:
|
|
519
|
+
"""Write a streamed response to local_path via a .part file, so a broken download never leaves a half file."""
|
|
520
|
+
part = f"{local_path}.part"
|
|
521
|
+
try:
|
|
522
|
+
with open(part, "wb") as f:
|
|
408
523
|
for chunk in resp.iter_content(chunk_size=8192):
|
|
409
524
|
f.write(chunk)
|
|
410
|
-
|
|
411
|
-
except requests.RequestException as e:
|
|
412
|
-
report_error(self.config, "download", e, host=host_of(url), path=local_path)
|
|
413
|
-
raise FileDownloadError(f"Failed to download file: {e}")
|
|
525
|
+
os.replace(part, local_path)
|
|
414
526
|
except IOError as e:
|
|
415
|
-
report_error(self.config, "download_save", e, host=host_of(
|
|
416
|
-
|
|
527
|
+
report_error(self.config, "download_save", e, host=host_of(source_url), path=local_path)
|
|
528
|
+
try:
|
|
529
|
+
os.remove(part)
|
|
530
|
+
except OSError:
|
|
531
|
+
pass
|
|
532
|
+
raise FileDownloadError(f"could not save it ({type(e).__name__})")
|
|
533
|
+
except requests.RequestException:
|
|
534
|
+
try:
|
|
535
|
+
os.remove(part)
|
|
536
|
+
except OSError:
|
|
537
|
+
pass
|
|
538
|
+
raise
|
|
417
539
|
|
|
418
540
|
def _get_mime_type(self, path: Path) -> str:
|
|
419
541
|
"""Get MIME type for a file."""
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "cellcog"
|
|
7
|
-
version = "2.8.
|
|
7
|
+
version = "2.8.2"
|
|
8
8
|
description = "Python SDK for CellCog - Any-to-Any AI for agents. Fire-and-forget pattern with WebSocket notifications."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = {text = "MIT"}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Standalone test: the bare-path auto-upload scans only text OUTSIDE existing file tags (Nov 6 2026).
|
|
2
|
+
|
|
3
|
+
A Windows path kept in a tag attribute (external_local_path="C:/Users/me/a.webp") contains "/Users/me/a.webp".
|
|
4
|
+
The old scan matched it and nested a second tag inside the attribute, so the chat rendered broken tag text.
|
|
5
|
+
Runs on any OS: os.path.exists/isfile and the upload are stubbed, no network.
|
|
6
|
+
|
|
7
|
+
Run: python3 tests/test_bare_path_scan.py
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
import sys
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
from unittest import mock
|
|
14
|
+
|
|
15
|
+
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
16
|
+
|
|
17
|
+
import cellcog.files as files_mod # noqa: E402
|
|
18
|
+
from cellcog.config import Config # noqa: E402
|
|
19
|
+
from cellcog.files import FileProcessor # noqa: E402
|
|
20
|
+
|
|
21
|
+
FT = "SHOW_" + "FILE"
|
|
22
|
+
failed = 0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check(label, ok, detail=""):
|
|
26
|
+
global failed
|
|
27
|
+
if ok:
|
|
28
|
+
print(f"[OK ] {label}")
|
|
29
|
+
else:
|
|
30
|
+
failed += 1
|
|
31
|
+
print(f"[FAIL] {label}\n {detail}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def fake_exists(p):
|
|
35
|
+
return "Users/me/" in str(p)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
fp = FileProcessor(Config())
|
|
39
|
+
counter = {"n": 0}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def fake_upload(path):
|
|
43
|
+
counter["n"] += 1
|
|
44
|
+
return f"blob{counter['n']}/{Path(str(path).replace(chr(92), '/')).name}"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
with mock.patch.object(files_mod.os.path, "exists", fake_exists), \
|
|
48
|
+
mock.patch.object(files_mod.os.path, "isfile", fake_exists), \
|
|
49
|
+
mock.patch.object(fp, "_upload_file", fake_upload):
|
|
50
|
+
msg = f"Hero: <{FT}>C:/Users/me/a.webp</{FT}>\nAlso see /Users/me/b.png please."
|
|
51
|
+
out, uploaded = fp.transform_outgoing(msg)
|
|
52
|
+
|
|
53
|
+
tags = re.findall(rf"<{FT}[^>]*>.*?</{FT}>", out, re.S)
|
|
54
|
+
check("exactly two file tags (the tagged one + the bare one)", len(tags) == 2, out)
|
|
55
|
+
check("no tag nested inside another tag's attribute", f'"C:<{FT}' not in out and out.count(f"<{FT}") == 2, out)
|
|
56
|
+
check("the Windows path stays intact in the attribute", 'external_local_path="C:/Users/me/a.webp"' in out, out)
|
|
57
|
+
check("the bare path in prose is still auto-uploaded", 'external_local_path="/Users/me/b.png"' in out, out)
|
|
58
|
+
check("two uploads, not three", len(uploaded) == 2, uploaded)
|
|
59
|
+
|
|
60
|
+
# FAIL control: the old whole-string scan DOES match inside the tag attribute (the reported bug).
|
|
61
|
+
old_re = re.compile(r"(?<![<\"=])(?:/[\w.+\-][\w.+\-/]*|~/[\w.+\-][\w.+\-/]*)")
|
|
62
|
+
first_tag = tags[0] if tags else ""
|
|
63
|
+
check("CONTROL: the old scan matches a path inside the tag", bool(old_re.search(first_tag)), first_tag)
|
|
64
|
+
|
|
65
|
+
print("\n" + ("ALL PASS" if failed == 0 else f"{failed} FAILURE(S)"))
|
|
66
|
+
sys.exit(1 if failed else 0)
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
"""Tests for the storage fallback (CEL-3867): when storage.googleapis.com is unreachable, uploads and downloads go
|
|
2
|
+
through cellcog.ai, and a file that still fails is reported loudly. No network: requests calls are replaced.
|
|
3
|
+
|
|
4
|
+
Run: python3 tests/test_storage_fallback.py (or pytest)
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import os
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
|
|
11
|
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
|
|
12
|
+
os.environ["CELLCOG_DISABLE_ERROR_REPORTS"] = "1"
|
|
13
|
+
|
|
14
|
+
import requests # noqa: E402
|
|
15
|
+
|
|
16
|
+
from cellcog import files as F # noqa: E402
|
|
17
|
+
from cellcog.exceptions import FileDownloadError, FileUploadError # noqa: E402
|
|
18
|
+
|
|
19
|
+
API = "https://cellcog.ai/api"
|
|
20
|
+
STORAGE = "https://storage.googleapis.com/cellcog_files/blob?X-Goog-Signature=abc"
|
|
21
|
+
CALLS = []
|
|
22
|
+
PASS = FAIL = 0
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def check(name, cond, note=""):
|
|
26
|
+
global PASS, FAIL
|
|
27
|
+
if cond:
|
|
28
|
+
PASS += 1
|
|
29
|
+
print(f" ✅ {name}")
|
|
30
|
+
else:
|
|
31
|
+
FAIL += 1
|
|
32
|
+
print(f" ❌ {name} {note}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class _Cfg:
|
|
36
|
+
api_key = "test"
|
|
37
|
+
api_base_url = API
|
|
38
|
+
|
|
39
|
+
def get_request_headers(self):
|
|
40
|
+
return {"X-API-Key": "test", "X-CellCog-Python-SDK-Version": "2.8.1"}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class _Resp:
|
|
44
|
+
def __init__(self, status=200, payload=None, body=b""):
|
|
45
|
+
self.status_code, self._payload, self._body = status, payload or {}, body
|
|
46
|
+
|
|
47
|
+
def raise_for_status(self):
|
|
48
|
+
if self.status_code >= 400:
|
|
49
|
+
raise requests.exceptions.HTTPError(f"{self.status_code}", response=self)
|
|
50
|
+
|
|
51
|
+
def json(self):
|
|
52
|
+
return self._payload
|
|
53
|
+
|
|
54
|
+
def iter_content(self, chunk_size=8192):
|
|
55
|
+
yield self._body
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def proxy_block(*a, **k):
|
|
59
|
+
raise requests.exceptions.ProxyError("Unable to connect to proxy: Tunnel connection failed: 403 Forbidden")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def install(put_storage, get_storage=None, put_fallback=None, get_fallback=None):
|
|
63
|
+
"""Route each requests call by URL: storage vs cellcog.ai."""
|
|
64
|
+
CALLS.clear()
|
|
65
|
+
|
|
66
|
+
def post(url, **k):
|
|
67
|
+
CALLS.append(("POST", url))
|
|
68
|
+
if url.endswith("/files/request-upload"):
|
|
69
|
+
return _Resp(payload={"file_id": "f1", "upload_url": STORAGE, "blob_name": "human_files/u/ref.png"})
|
|
70
|
+
return _Resp(payload={"success": True})
|
|
71
|
+
|
|
72
|
+
def put(url, **k):
|
|
73
|
+
CALLS.append(("PUT", url))
|
|
74
|
+
if url.startswith("https://storage.googleapis.com"):
|
|
75
|
+
return put_storage(url, **k)
|
|
76
|
+
return (put_fallback or (lambda *a, **kw: _Resp()))(url, **k)
|
|
77
|
+
|
|
78
|
+
def get(url, **k):
|
|
79
|
+
CALLS.append(("GET", url, k.get("params")))
|
|
80
|
+
if url.startswith("https://storage.googleapis.com"):
|
|
81
|
+
return (get_storage or proxy_block)(url, **k)
|
|
82
|
+
return (get_fallback or (lambda *a, **kw: _Resp(body=b"via-cellcog")))(url, **k)
|
|
83
|
+
|
|
84
|
+
F.requests.post, F.requests.put, F.requests.get = post, put, get
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def tmpfile(size=100, name="ref.png"):
|
|
88
|
+
d = tempfile.mkdtemp()
|
|
89
|
+
p = os.path.join(d, name)
|
|
90
|
+
with open(p, "wb") as f:
|
|
91
|
+
f.truncate(size)
|
|
92
|
+
return p
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def main():
|
|
96
|
+
print("— upload —")
|
|
97
|
+
install(lambda *a, **k: _Resp())
|
|
98
|
+
fp = F.FileProcessor(_Cfg())
|
|
99
|
+
blob = fp._upload_file(tmpfile())
|
|
100
|
+
check("direct storage PUT works: no cellcog.ai byte upload (PASS control)",
|
|
101
|
+
blob == "human_files/u/ref.png" and not any("upload-bytes" in c[1] for c in CALLS), str(CALLS))
|
|
102
|
+
|
|
103
|
+
install(proxy_block)
|
|
104
|
+
fp = F.FileProcessor(_Cfg())
|
|
105
|
+
blob = fp._upload_file(tmpfile())
|
|
106
|
+
check("a proxy-refused storage PUT falls back to /files/upload-bytes, then confirms",
|
|
107
|
+
blob == "human_files/u/ref.png" and ("PUT", f"{API}/files/upload-bytes/f1") in CALLS
|
|
108
|
+
and ("POST", f"{API}/files/confirm-upload/f1") in CALLS, str(CALLS))
|
|
109
|
+
check("the run remembers the block", fp._storage_blocked is True)
|
|
110
|
+
CALLS.clear()
|
|
111
|
+
fp._upload_file(tmpfile())
|
|
112
|
+
check("the next upload skips the direct storage attempt",
|
|
113
|
+
not any(c[1].startswith("https://storage") for c in CALLS) and any("upload-bytes" in c[1] for c in CALLS), str(CALLS))
|
|
114
|
+
|
|
115
|
+
install(lambda *a, **k: _Resp(status=403, payload={}))
|
|
116
|
+
fp = F.FileProcessor(_Cfg())
|
|
117
|
+
try:
|
|
118
|
+
fp._upload_file(tmpfile()); err = ""
|
|
119
|
+
except FileUploadError as e:
|
|
120
|
+
err = str(e)
|
|
121
|
+
check("[FAIL control] an HTTP 403 FROM storage is not a block: no fallback, clear error",
|
|
122
|
+
"HTTP 403" in err and not any("upload-bytes" in c[1] for c in CALLS) and "X-Goog-Signature" not in err, err)
|
|
123
|
+
|
|
124
|
+
install(proxy_block)
|
|
125
|
+
fp = F.FileProcessor(_Cfg())
|
|
126
|
+
try:
|
|
127
|
+
fp._upload_file(tmpfile(size=F.PROXY_UPLOAD_MAX_BYTES + 1, name="big.mp4")); err = ""
|
|
128
|
+
except FileUploadError as e:
|
|
129
|
+
err = str(e)
|
|
130
|
+
check("[FAIL control] a blocked file over 25 MB says why, and sends nothing through cellcog.ai",
|
|
131
|
+
"100 MB" in err and "big.mp4" in err and not any("upload-bytes" in c[1] for c in CALLS), err)
|
|
132
|
+
|
|
133
|
+
print("— loud failure on send —")
|
|
134
|
+
install(proxy_block, put_fallback=lambda *a, **k: _Resp(status=500, payload={"detail": "boom"}))
|
|
135
|
+
fp = F.FileProcessor(_Cfg())
|
|
136
|
+
path = tmpfile()
|
|
137
|
+
try:
|
|
138
|
+
fp.transform_outgoing(f"Use this <SHOW_FILE>{path}</SHOW_FILE> as reference"); err = ""
|
|
139
|
+
except FileUploadError as e:
|
|
140
|
+
err = str(e)
|
|
141
|
+
check("[FAIL control] a file that cannot be uploaded either way stops the send, naming the file",
|
|
142
|
+
"failed to upload" in err and path in err and "Nothing was sent" in err, err)
|
|
143
|
+
|
|
144
|
+
install(proxy_block)
|
|
145
|
+
fp = F.FileProcessor(_Cfg())
|
|
146
|
+
out, up = fp.transform_outgoing(f"Use this <SHOW_FILE>{path}</SHOW_FILE> as reference")
|
|
147
|
+
check("the same message with a working fallback sends the stored reference (PASS control)",
|
|
148
|
+
"human_files/u/ref.png" in out and len(up) == 1, out)
|
|
149
|
+
|
|
150
|
+
print("— download —")
|
|
151
|
+
chat_id = "6ae7d4b4a578585316b14e62"
|
|
152
|
+
ref = f"{chat_id}//home/app/img/legpress_A.png"
|
|
153
|
+
dest = os.path.join(tempfile.mkdtemp(), "out", "legpress_A.png")
|
|
154
|
+
install(proxy_block)
|
|
155
|
+
fp = F.FileProcessor(_Cfg())
|
|
156
|
+
fp._download_file(STORAGE, dest, chat_id=chat_id, ref=ref)
|
|
157
|
+
fb = [c for c in CALLS if c[0] == "GET" and c[1] == f"{API}/cellcog/chat/{chat_id}/file"]
|
|
158
|
+
check("a proxy-refused storage GET falls back to /cellcog/chat/{id}/file with the ref",
|
|
159
|
+
open(dest, "rb").read() == b"via-cellcog" and fb and fb[0][2] == {"ref": ref}, str(CALLS))
|
|
160
|
+
check("no .part file is left behind", not os.path.exists(dest + ".part"))
|
|
161
|
+
|
|
162
|
+
install(lambda *a, **k: _Resp(), get_storage=lambda *a, **k: _Resp(body=b"direct"))
|
|
163
|
+
fp = F.FileProcessor(_Cfg())
|
|
164
|
+
dest2 = os.path.join(tempfile.mkdtemp(), "a.png")
|
|
165
|
+
fp._download_file(STORAGE, dest2, chat_id=chat_id, ref=ref)
|
|
166
|
+
check("a working storage GET never touches cellcog.ai (PASS control)",
|
|
167
|
+
open(dest2, "rb").read() == b"direct" and not any("/file" in c[1] for c in CALLS if c[0] == "GET" and "cellcog.ai" in c[1]))
|
|
168
|
+
|
|
169
|
+
install(proxy_block, get_fallback=lambda *a, **k: _Resp(status=404, payload={"detail": "File not found"}))
|
|
170
|
+
fp = F.FileProcessor(_Cfg())
|
|
171
|
+
msgs = [{"messageFrom": "CellCog", "content": f"Here:\n<SHOW_FILE>{ref}</SHOW_FILE>", "createdAt": "x"}]
|
|
172
|
+
out = fp.transform_incoming_history(msgs, {ref: {"url": STORAGE}}, chat_id)
|
|
173
|
+
check("[FAIL control] a file that downloads neither way gets a visible note, not a silent missing file",
|
|
174
|
+
"[cellcog: this file was not downloaded:" in out[0]["content"] and "HTTP 404" in out[0]["content"]
|
|
175
|
+
and "X-Goog-Signature" not in out[0]["content"], out[0]["content"])
|
|
176
|
+
|
|
177
|
+
print(f"\n{PASS} passed, {FAIL} failed")
|
|
178
|
+
return FAIL
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def test_storage_fallback():
|
|
182
|
+
assert main() == 0
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
if __name__ == "__main__":
|
|
186
|
+
sys.exit(1 if main() else 0)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|