crabcode 0.1.6__py3-none-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. crabcode-0.1.6.dist-info/METADATA +63 -0
  2. crabcode-0.1.6.dist-info/RECORD +205 -0
  3. crabcode-0.1.6.dist-info/WHEEL +4 -0
  4. crabcode-0.1.6.dist-info/entry_points.txt +2 -0
  5. crabcode_cli/__init__.py +7 -0
  6. crabcode_cli/__main__.py +4 -0
  7. crabcode_cli/_native/crabcode-computer-use-host.exe +0 -0
  8. crabcode_cli/_native/runtime.json +6 -0
  9. crabcode_cli/app.py +776 -0
  10. crabcode_cli/banner.py +39 -0
  11. crabcode_cli/computer_use.py +286 -0
  12. crabcode_cli/model_manager.py +649 -0
  13. crabcode_cli/model_picker.py +168 -0
  14. crabcode_cli/pipe.py +148 -0
  15. crabcode_cli/render/__init__.py +0 -0
  16. crabcode_cli/render/messages.py +19 -0
  17. crabcode_cli/repl.py +4841 -0
  18. crabcode_cli/session_picker.py +239 -0
  19. crabcode_core/__init__.py +7 -0
  20. crabcode_core/_version.py +3 -0
  21. crabcode_core/_windows_process.py +68 -0
  22. crabcode_core/agent_manager.py +1731 -0
  23. crabcode_core/api/__init__.py +20 -0
  24. crabcode_core/api/anthropic_adapter.py +648 -0
  25. crabcode_core/api/azure_adapter.py +69 -0
  26. crabcode_core/api/base.py +233 -0
  27. crabcode_core/api/codex_adapter.py +1327 -0
  28. crabcode_core/api/error_logging.py +132 -0
  29. crabcode_core/api/gemini_adapter.py +26 -0
  30. crabcode_core/api/model_info.py +257 -0
  31. crabcode_core/api/network.py +62 -0
  32. crabcode_core/api/ollama_adapter.py +27 -0
  33. crabcode_core/api/openai_adapter.py +323 -0
  34. crabcode_core/api/registry.py +58 -0
  35. crabcode_core/command_line.py +18 -0
  36. crabcode_core/compact/__init__.py +17 -0
  37. crabcode_core/compact/compact.py +514 -0
  38. crabcode_core/compact/context_tokens.py +165 -0
  39. crabcode_core/computer_use_mode.py +38 -0
  40. crabcode_core/computer_use_observation.py +144 -0
  41. crabcode_core/config/__init__.py +3 -0
  42. crabcode_core/config/claudemd.py +61 -0
  43. crabcode_core/config/manager.py +237 -0
  44. crabcode_core/config/model_catalog.py +256 -0
  45. crabcode_core/events.py +4896 -0
  46. crabcode_core/file_lock.py +81 -0
  47. crabcode_core/filesystem.py +22 -0
  48. crabcode_core/goal.py +149 -0
  49. crabcode_core/hooks/__init__.py +5 -0
  50. crabcode_core/hooks/manager.py +290 -0
  51. crabcode_core/image_io.py +111 -0
  52. crabcode_core/io_worker.py +211 -0
  53. crabcode_core/logging_utils.py +168 -0
  54. crabcode_core/lsp/__init__.py +6 -0
  55. crabcode_core/lsp/client.py +786 -0
  56. crabcode_core/lsp/diagnostics.py +150 -0
  57. crabcode_core/lsp/manager.py +583 -0
  58. crabcode_core/mcp/__init__.py +4 -0
  59. crabcode_core/mcp/client.py +169 -0
  60. crabcode_core/mcp/config.py +57 -0
  61. crabcode_core/path_validation.py +34 -0
  62. crabcode_core/paths.py +24 -0
  63. crabcode_core/peer/__init__.py +10 -0
  64. crabcode_core/peer/runtime.py +518 -0
  65. crabcode_core/permissions/__init__.py +7 -0
  66. crabcode_core/permissions/ai_reviewer.py +184 -0
  67. crabcode_core/permissions/manager.py +187 -0
  68. crabcode_core/plan/__init__.py +5 -0
  69. crabcode_core/plan/executor.py +270 -0
  70. crabcode_core/plan/types.py +169 -0
  71. crabcode_core/prompts/__init__.py +0 -0
  72. crabcode_core/prompts/blocks.py +10 -0
  73. crabcode_core/prompts/context.py +229 -0
  74. crabcode_core/prompts/library.py +92 -0
  75. crabcode_core/prompts/profile.py +79 -0
  76. crabcode_core/prompts/system.py +356 -0
  77. crabcode_core/prompts/templates.py +104 -0
  78. crabcode_core/query/__init__.py +0 -0
  79. crabcode_core/query/computer_use_history.py +83 -0
  80. crabcode_core/query/loop.py +2446 -0
  81. crabcode_core/query/retry.py +116 -0
  82. crabcode_core/schedule/__init__.py +13 -0
  83. crabcode_core/schedule/manager.py +631 -0
  84. crabcode_core/schedule/models.py +204 -0
  85. crabcode_core/schedule/store.py +558 -0
  86. crabcode_core/schedule/timing.py +287 -0
  87. crabcode_core/session/__init__.py +12 -0
  88. crabcode_core/session/export.py +141 -0
  89. crabcode_core/session/meta_db.py +684 -0
  90. crabcode_core/session/storage.py +2032 -0
  91. crabcode_core/session/title_gen.py +52 -0
  92. crabcode_core/skills/__init__.py +6 -0
  93. crabcode_core/skills/context.py +23 -0
  94. crabcode_core/skills/loader.py +214 -0
  95. crabcode_core/skills/matcher.py +203 -0
  96. crabcode_core/snapshot/__init__.py +21 -0
  97. crabcode_core/snapshot/snapshot.py +436 -0
  98. crabcode_core/snapshot/tracker.py +359 -0
  99. crabcode_core/subprocess_utils.py +260 -0
  100. crabcode_core/team/__init__.py +27 -0
  101. crabcode_core/team/bridge.py +66 -0
  102. crabcode_core/team/inbox.py +237 -0
  103. crabcode_core/team/manager.py +1660 -0
  104. crabcode_core/team/message_bus.py +821 -0
  105. crabcode_core/team/models.py +191 -0
  106. crabcode_core/team/recovery.py +192 -0
  107. crabcode_core/text_io.py +52 -0
  108. crabcode_core/tools/__init__.py +112 -0
  109. crabcode_core/tools/_input_helpers.py +39 -0
  110. crabcode_core/tools/_input_schema.py +130 -0
  111. crabcode_core/tools/agent.py +413 -0
  112. crabcode_core/tools/apply_patch.py +587 -0
  113. crabcode_core/tools/ask_user.py +122 -0
  114. crabcode_core/tools/bash.py +154 -0
  115. crabcode_core/tools/browser.py +678 -0
  116. crabcode_core/tools/checklist.py +241 -0
  117. crabcode_core/tools/checkpoint.py +78 -0
  118. crabcode_core/tools/computer_use.py +599 -0
  119. crabcode_core/tools/diff_utils.py +123 -0
  120. crabcode_core/tools/file_edit.py +240 -0
  121. crabcode_core/tools/file_read.py +187 -0
  122. crabcode_core/tools/file_write.py +154 -0
  123. crabcode_core/tools/glob.py +123 -0
  124. crabcode_core/tools/goal.py +134 -0
  125. crabcode_core/tools/grep.py +247 -0
  126. crabcode_core/tools/image.py +173 -0
  127. crabcode_core/tools/image_generate.py +115 -0
  128. crabcode_core/tools/lint.py +388 -0
  129. crabcode_core/tools/loading.py +187 -0
  130. crabcode_core/tools/memory.py +456 -0
  131. crabcode_core/tools/monitor.py +677 -0
  132. crabcode_core/tools/peer.py +103 -0
  133. crabcode_core/tools/revert.py +90 -0
  134. crabcode_core/tools/schedule.py +683 -0
  135. crabcode_core/tools/skill.py +108 -0
  136. crabcode_core/tools/switch_mode.py +213 -0
  137. crabcode_core/tools/team.py +329 -0
  138. crabcode_core/tools/tool_search.py +64 -0
  139. crabcode_core/tools/web_search.py +444 -0
  140. crabcode_core/turn_details.py +64 -0
  141. crabcode_core/types/__init__.py +95 -0
  142. crabcode_core/types/config.py +434 -0
  143. crabcode_core/types/event.py +334 -0
  144. crabcode_core/types/message.py +263 -0
  145. crabcode_core/types/tool.py +208 -0
  146. crabcode_core/usage.py +289 -0
  147. crabcode_core/utf8_sanitize.py +29 -0
  148. crabcode_debugger/__init__.py +5 -0
  149. crabcode_debugger/adapters.py +332 -0
  150. crabcode_debugger/dap.py +304 -0
  151. crabcode_debugger/memory.py +1806 -0
  152. crabcode_debugger/process.py +707 -0
  153. crabcode_debugger/runtime.py +58 -0
  154. crabcode_debugger/sessions.py +387 -0
  155. crabcode_debugger/tool.py +731 -0
  156. crabcode_gateway/__init__.py +18 -0
  157. crabcode_gateway/acp/__init__.py +13 -0
  158. crabcode_gateway/acp/agent.py +1581 -0
  159. crabcode_gateway/acp/session.py +114 -0
  160. crabcode_gateway/acp/transport.py +39 -0
  161. crabcode_gateway/acp/types.py +90 -0
  162. crabcode_gateway/adapter.py +30 -0
  163. crabcode_gateway/auth.py +139 -0
  164. crabcode_gateway/computer_use.py +364 -0
  165. crabcode_gateway/document_engine.py +637 -0
  166. crabcode_gateway/document_worker.py +301 -0
  167. crabcode_gateway/event_bus.py +655 -0
  168. crabcode_gateway/grpc/__init__.py +0 -0
  169. crabcode_gateway/grpc/crabcode_pb2.py +70 -0
  170. crabcode_gateway/grpc/crabcode_pb2_grpc.py +474 -0
  171. crabcode_gateway/grpc/proto/crabcode.proto +160 -0
  172. crabcode_gateway/grpc/server.py +765 -0
  173. crabcode_gateway/middleware.py +344 -0
  174. crabcode_gateway/protocol.py +17 -0
  175. crabcode_gateway/routes/__init__.py +0 -0
  176. crabcode_gateway/routes/agent.py +392 -0
  177. crabcode_gateway/routes/auth.py +150 -0
  178. crabcode_gateway/routes/computer_use.py +101 -0
  179. crabcode_gateway/routes/config.py +1698 -0
  180. crabcode_gateway/routes/document.py +1810 -0
  181. crabcode_gateway/routes/event.py +3801 -0
  182. crabcode_gateway/routes/git_review.py +232 -0
  183. crabcode_gateway/routes/health.py +14 -0
  184. crabcode_gateway/routes/peer.py +78 -0
  185. crabcode_gateway/routes/permission.py +121 -0
  186. crabcode_gateway/routes/schedule.py +300 -0
  187. crabcode_gateway/routes/session.py +1496 -0
  188. crabcode_gateway/routes/snapshot.py +211 -0
  189. crabcode_gateway/routes/tasks.py +318 -0
  190. crabcode_gateway/routes/team.py +392 -0
  191. crabcode_gateway/routes/usage.py +49 -0
  192. crabcode_gateway/routes/workspace.py +299 -0
  193. crabcode_gateway/safe_file.py +63 -0
  194. crabcode_gateway/schemas.py +1820 -0
  195. crabcode_gateway/server.py +514 -0
  196. crabcode_gateway/session_registry.py +36 -0
  197. crabcode_gateway/task_registry.py +708 -0
  198. crabcode_gateway/turn_details.py +60 -0
  199. crabcode_search/__init__.py +11 -0
  200. crabcode_search/background.py +368 -0
  201. crabcode_search/chunker.py +517 -0
  202. crabcode_search/embedder.py +497 -0
  203. crabcode_search/indexer.py +215 -0
  204. crabcode_search/store.py +222 -0
  205. crabcode_search/tool.py +268 -0
@@ -0,0 +1,1810 @@
1
+ """Managed document-project import and artifact APIs."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import hashlib
7
+ import html
8
+ import ipaddress
9
+ import json
10
+ import mimetypes
11
+ import os
12
+ import re
13
+ import shutil
14
+ import socket
15
+ import tempfile
16
+ import threading
17
+ import uuid
18
+ import zipfile
19
+ from contextlib import contextmanager
20
+ from datetime import datetime, timezone
21
+ from pathlib import Path
22
+ from typing import Any, Literal
23
+ from urllib.parse import unquote, urljoin, urlsplit, urlunsplit
24
+
25
+ import httpx
26
+ from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile
27
+ from fastapi.responses import FileResponse
28
+ from pydantic import BaseModel, Field, HttpUrl
29
+ from pypdf import PdfReader
30
+
31
+ from crabcode_core.path_validation import validate_path_component
32
+ from crabcode_core.filesystem import replace_with_retry
33
+ from crabcode_core.subprocess_utils import (
34
+ decode_subprocess_output,
35
+ managed_process_command,
36
+ subprocess_group_options,
37
+ terminate_process_tree,
38
+ )
39
+ from crabcode_gateway.schemas import WorkspaceInfo
40
+
41
+
42
+ router = APIRouter(prefix="/document", tags=["document"])
43
+
44
+ MAX_DOCUMENT_BYTES = 100 * 1024 * 1024
45
+ MAX_LAYOUT_BYTES = 50 * 1024 * 1024
46
+ SUPPORTED_EXTENSIONS = {
47
+ ".pdf", ".doc", ".docx", ".odt", ".rtf", ".ppt", ".pptx",
48
+ ".txt", ".md", ".html", ".htm",
49
+ }
50
+ OFFICE_EXTENSIONS = SUPPORTED_EXTENSIONS - {".pdf"}
51
+ MANIFEST_RELATIVE_PATH = Path(".crabcode/document/manifest.json")
52
+ _MANIFEST_LOCKS_GUARD = threading.Lock()
53
+ _MANIFEST_LOCKS: dict[str, threading.RLock] = {}
54
+
55
+ MIME_EXTENSIONS = {
56
+ "application/pdf": ".pdf",
57
+ "application/msword": ".doc",
58
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
59
+ "application/vnd.oasis.opendocument.text": ".odt",
60
+ "application/rtf": ".rtf",
61
+ "text/rtf": ".rtf",
62
+ "application/vnd.ms-powerpoint": ".ppt",
63
+ "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
64
+ "text/plain": ".txt",
65
+ "text/markdown": ".md",
66
+ "text/html": ".html",
67
+ }
68
+
69
+
70
+ class DocumentUrlImportRequest(BaseModel):
71
+ workspace_path: str = Field(min_length=1)
72
+ project_id: str = Field(min_length=1, max_length=200)
73
+ project_name: str = Field(min_length=1, max_length=200)
74
+ url: HttpUrl
75
+
76
+
77
+ class DocumentLayoutRequest(BaseModel):
78
+ fingerprint: str = Field(min_length=1, max_length=500)
79
+ page_count: int = Field(ge=1, le=100_000)
80
+ pages: list[dict[str, Any]]
81
+
82
+
83
+ class DocumentBlogWriteRequest(BaseModel):
84
+ revision: str | None = None
85
+ markdown: str = Field(max_length=10_000_000)
86
+ language: str = Field(default="", max_length=50)
87
+
88
+
89
+ class DocumentAnnotationRect(BaseModel):
90
+ page: int = Field(ge=1, le=100_000)
91
+ x: float = Field(ge=0, le=1)
92
+ y: float = Field(ge=0, le=1)
93
+ width: float = Field(gt=0, le=1)
94
+ height: float = Field(gt=0, le=1)
95
+
96
+
97
+ class DocumentAnnotationWriteRequest(BaseModel):
98
+ id: str = Field(pattern=r"^[A-Za-z0-9_-]{1,100}$")
99
+ label: str = Field(default="", max_length=100)
100
+ note: str = Field(default="", max_length=20_000)
101
+ text: str = Field(min_length=1, max_length=50_000)
102
+ rects: list[DocumentAnnotationRect] = Field(min_length=1, max_length=1_000)
103
+ created_at: str = Field(default="", max_length=100)
104
+ updated_at: str = Field(default="", max_length=100)
105
+
106
+
107
+ @contextmanager
108
+ def _manifest_lock(workspace: Path):
109
+ key = os.path.normcase(str(workspace.resolve(strict=False)))
110
+ with _MANIFEST_LOCKS_GUARD:
111
+ lock = _MANIFEST_LOCKS.setdefault(key, threading.RLock())
112
+ with lock:
113
+ yield
114
+
115
+
116
+ def _workspace_roots(request: Request) -> tuple[Path, ...]:
117
+ info: WorkspaceInfo = request.app.state.workspace_info
118
+ return tuple(Path(value).resolve() for value in info.browse_roots)
119
+
120
+
121
+ def _is_within(path: Path, roots: tuple[Path, ...]) -> bool:
122
+ normalized = os.path.normcase(str(path))
123
+ for root in roots:
124
+ try:
125
+ if os.path.commonpath((normalized, os.path.normcase(str(root)))) == os.path.normcase(str(root)):
126
+ return True
127
+ except ValueError:
128
+ continue
129
+ return False
130
+
131
+
132
+ def _validated_new_workspace(value: str, roots: tuple[Path, ...]) -> Path:
133
+ candidate = Path(value).expanduser()
134
+ if not candidate.is_absolute() or not candidate.name:
135
+ raise HTTPException(status_code=400, detail="workspace_path must name an absolute directory")
136
+ candidate = candidate.resolve(strict=False)
137
+ if not _is_within(candidate, roots):
138
+ raise HTTPException(status_code=403, detail="Document workspace is outside the allowed browse roots")
139
+ if candidate.exists():
140
+ base = candidate
141
+ for index in range(2, 10_000):
142
+ candidate = base.with_name(f"{base.name}-{index}")
143
+ if not candidate.exists():
144
+ break
145
+ else:
146
+ raise HTTPException(status_code=409, detail="Unable to find an unused document workspace name")
147
+ ancestor = candidate.parent
148
+ while not ancestor.exists() and ancestor != ancestor.parent:
149
+ ancestor = ancestor.parent
150
+ try:
151
+ resolved_ancestor = ancestor.resolve(strict=True)
152
+ except OSError as exc:
153
+ raise HTTPException(status_code=400, detail="Unable to resolve document workspace parent") from exc
154
+ if not resolved_ancestor.is_dir() or not _is_within(resolved_ancestor, roots):
155
+ raise HTTPException(status_code=403, detail="Document workspace parent is not allowed")
156
+ return candidate
157
+
158
+
159
+ def _validated_workspace(value: str, roots: tuple[Path, ...]) -> Path:
160
+ candidate = Path(value).expanduser()
161
+ if not candidate.is_absolute():
162
+ raise HTTPException(status_code=400, detail="workspace must be absolute")
163
+ try:
164
+ resolved = candidate.resolve(strict=True)
165
+ except FileNotFoundError as exc:
166
+ raise HTTPException(status_code=404, detail="Document workspace not found") from exc
167
+ if not resolved.is_dir() or not _is_within(resolved, roots):
168
+ raise HTTPException(status_code=403, detail="Document workspace is not allowed")
169
+ manifest = resolved / MANIFEST_RELATIVE_PATH
170
+ if not manifest.is_file() or manifest.is_symlink():
171
+ raise HTTPException(status_code=404, detail="Document manifest not found")
172
+ return resolved
173
+
174
+
175
+ def _safe_name(value: str, fallback: str = "document") -> str:
176
+ name = Path(value.replace("\\", "/")).name.strip()
177
+ name = re.sub(r"[<>:\"/\\|?*\x00-\x1f]", "-", name).rstrip(". ")
178
+ name = name[:180] or fallback
179
+ try:
180
+ validate_path_component(name, "document filename")
181
+ except ValueError:
182
+ name = f"_{name}"[:180]
183
+ return name
184
+
185
+
186
+ def _file_extension(filename: str) -> str:
187
+ extension = Path(filename).suffix.lower()
188
+ if extension not in SUPPORTED_EXTENSIONS:
189
+ raise HTTPException(status_code=415, detail=f"Unsupported document format: {extension or 'unknown'}")
190
+ return extension
191
+
192
+
193
+ def _converter() -> str | None:
194
+ override = os.environ.get("CRABCODE_LIBREOFFICE_PATH", "").strip()
195
+ if override:
196
+ candidate = Path(override).expanduser()
197
+ return str(candidate.resolve()) if candidate.is_file() else None
198
+ executable = shutil.which("soffice") or shutil.which("libreoffice")
199
+ if executable or os.name != "nt":
200
+ return executable
201
+ for variable in ("ProgramFiles", "ProgramFiles(x86)", "LOCALAPPDATA"):
202
+ root = os.environ.get(variable)
203
+ if root:
204
+ candidate = Path(root) / "LibreOffice" / "program" / "soffice.exe"
205
+ if candidate.is_file():
206
+ return str(candidate)
207
+ return None
208
+
209
+
210
+ def _sha256(path: Path) -> str:
211
+ digest = hashlib.sha256()
212
+ with path.open("rb") as handle:
213
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
214
+ digest.update(chunk)
215
+ return digest.hexdigest()
216
+
217
+
218
+ def _atomic_write_bytes(path: Path, content: bytes) -> None:
219
+ path.parent.mkdir(parents=True, exist_ok=True)
220
+ with tempfile.NamedTemporaryFile(dir=path.parent, prefix=f".{path.name}.", delete=False) as handle:
221
+ temporary = Path(handle.name)
222
+ handle.write(content)
223
+ handle.flush()
224
+ os.fsync(handle.fileno())
225
+ try:
226
+ replace_with_retry(temporary, path)
227
+ finally:
228
+ temporary.unlink(missing_ok=True)
229
+
230
+
231
+ def _json_write(path: Path, value: Any) -> None:
232
+ _atomic_write_bytes(path, json.dumps(value, ensure_ascii=False, indent=2).encode("utf-8") + b"\n")
233
+
234
+
235
+ def _text_write(path: Path, value: str) -> None:
236
+ _atomic_write_bytes(path, value.encode("utf-8"))
237
+
238
+
239
+ def _read_manifest(workspace: Path) -> dict[str, Any]:
240
+ try:
241
+ value = json.loads((workspace / MANIFEST_RELATIVE_PATH).read_text(encoding="utf-8"))
242
+ except (OSError, json.JSONDecodeError) as exc:
243
+ raise HTTPException(status_code=500, detail="Document manifest is invalid") from exc
244
+ if not isinstance(value, dict):
245
+ raise HTTPException(status_code=500, detail="Document manifest is invalid")
246
+ return value
247
+
248
+
249
+ def _validate_signature(path: Path, extension: str) -> None:
250
+ with path.open("rb") as handle:
251
+ prefix = handle.read(8192)
252
+ if extension == ".pdf" and b"%PDF-" not in prefix[:1024]:
253
+ raise HTTPException(status_code=415, detail="The uploaded file is not a valid PDF")
254
+ if extension in {".docx", ".odt", ".pptx"} and not prefix.startswith(b"PK"):
255
+ raise HTTPException(status_code=415, detail="The uploaded file does not match its format")
256
+ if extension in {".docx", ".odt", ".pptx"}:
257
+ try:
258
+ with zipfile.ZipFile(path) as archive:
259
+ names = set(archive.namelist())
260
+ valid = (
261
+ extension == ".docx" and "word/document.xml" in names
262
+ or extension == ".pptx" and "ppt/presentation.xml" in names
263
+ or extension == ".odt" and "content.xml" in names and "META-INF/manifest.xml" in names
264
+ )
265
+ except (OSError, zipfile.BadZipFile):
266
+ valid = False
267
+ if not valid:
268
+ raise HTTPException(status_code=415, detail="The uploaded archive does not match its document format")
269
+ if extension in {".doc", ".ppt"} and not prefix.startswith(bytes.fromhex("d0cf11e0a1b11ae1")):
270
+ raise HTTPException(status_code=415, detail="The uploaded file does not match its format")
271
+ if extension == ".rtf" and not prefix.lstrip().startswith(b"{\\rtf"):
272
+ raise HTTPException(status_code=415, detail="The uploaded file is not valid RTF")
273
+ if extension in {".txt", ".md", ".html", ".htm"} and b"\x00" in prefix:
274
+ raise HTTPException(status_code=415, detail="The uploaded text document contains binary data")
275
+
276
+
277
+ def _validate_mime(extension: str, content_type: str) -> None:
278
+ media_type = content_type.split(";", 1)[0].strip().lower()
279
+ if not media_type or media_type == "application/octet-stream":
280
+ return
281
+ expected = MIME_EXTENSIONS.get(media_type)
282
+ compatible_text = extension in {".txt", ".md", ".html", ".htm"} and media_type.startswith("text/")
283
+ compatible_zip = extension in {".docx", ".odt", ".pptx"} and media_type in {"application/zip", "application/x-zip-compressed"}
284
+ if expected not in {extension, ".html" if extension == ".htm" else extension} and not compatible_text and not compatible_zip:
285
+ raise HTTPException(status_code=415, detail="Document MIME type does not match its filename")
286
+
287
+
288
+ def _pdf_page_count(path: Path) -> int:
289
+ try:
290
+ count = len(PdfReader(path, strict=False).pages)
291
+ except Exception as exc:
292
+ raise HTTPException(status_code=422, detail="The converted PDF is invalid or encrypted") from exc
293
+ if count < 1:
294
+ raise HTTPException(status_code=422, detail="The converted PDF has no pages")
295
+ return count
296
+
297
+
298
+ def _plain_html(text: str) -> str:
299
+ return (
300
+ "<!doctype html><meta charset=\"utf-8\"><style>"
301
+ "body{font:12pt/1.55 sans-serif;max-width:48rem;margin:2.5rem auto;}"
302
+ "pre{white-space:pre-wrap;font:inherit}</style><pre>"
303
+ f"{html.escape(text)}"
304
+ "</pre>"
305
+ )
306
+
307
+
308
+ def _prepare_conversion_input(source: Path, extension: str, internal: Path) -> Path:
309
+ if extension not in {".txt", ".md", ".html", ".htm"}:
310
+ return source
311
+ text = source.read_text(encoding="utf-8", errors="replace")
312
+ # Treat imported HTML as text. This deliberately prevents external resource
313
+ # loads and local-file references during headless conversion.
314
+ prepared = internal / "safe-source.html"
315
+ prepared.write_text(_plain_html(text), encoding="utf-8")
316
+ return prepared
317
+
318
+
319
+ async def _convert_to_pdf(source: Path, extension: str, internal: Path) -> Path:
320
+ rendered = internal / "rendered.pdf"
321
+ if extension == ".pdf":
322
+ shutil.copy2(source, rendered)
323
+ return rendered
324
+ converter = _converter()
325
+ if not converter:
326
+ raise HTTPException(status_code=503, detail="LibreOffice is required to convert this document format")
327
+ conversion_input = _prepare_conversion_input(source, extension, internal)
328
+ output_dir = internal / "conversion"
329
+ output_dir.mkdir()
330
+ profile_dir = internal / "libreoffice-profile"
331
+ profile_uri = profile_dir.resolve().as_uri()
332
+ process = await asyncio.create_subprocess_exec(
333
+ *managed_process_command([
334
+ converter,
335
+ "--headless",
336
+ f"-env:UserInstallation={profile_uri}",
337
+ "--convert-to",
338
+ "pdf",
339
+ "--outdir",
340
+ str(output_dir),
341
+ str(conversion_input),
342
+ ]),
343
+ stdout=asyncio.subprocess.PIPE,
344
+ stderr=asyncio.subprocess.PIPE,
345
+ **subprocess_group_options(),
346
+ )
347
+ try:
348
+ stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=120)
349
+ except asyncio.TimeoutError as exc:
350
+ await terminate_process_tree(process)
351
+ raise HTTPException(status_code=504, detail="Document conversion timed out") from exc
352
+ except asyncio.CancelledError:
353
+ await terminate_process_tree(process)
354
+ raise
355
+ candidates = list(output_dir.glob("*.pdf"))
356
+ if process.returncode != 0 or len(candidates) != 1:
357
+ detail = decode_subprocess_output(stderr or stdout).strip()[-500:]
358
+ raise HTTPException(status_code=422, detail=f"Document conversion failed: {detail or 'no PDF produced'}")
359
+ shutil.move(str(candidates[0]), rendered)
360
+ return rendered
361
+
362
+
363
+ async def _write_upload(upload: UploadFile, destination: Path) -> int:
364
+ total = 0
365
+ with destination.open("wb") as handle:
366
+ while True:
367
+ chunk = await upload.read(1024 * 1024)
368
+ if not chunk:
369
+ break
370
+ total += len(chunk)
371
+ if total > MAX_DOCUMENT_BYTES:
372
+ raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
373
+ handle.write(chunk)
374
+ if total == 0:
375
+ raise HTTPException(status_code=400, detail="Document is empty")
376
+ return total
377
+
378
+
379
+ def _assert_public_host(url: str) -> None:
380
+ parsed = urlsplit(url)
381
+ if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
382
+ raise HTTPException(status_code=400, detail="URL must be an unauthenticated HTTP or HTTPS address")
383
+ try:
384
+ addresses = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80))
385
+ except OSError as exc:
386
+ raise HTTPException(status_code=400, detail="Unable to resolve document URL") from exc
387
+ for address in addresses:
388
+ ip = ipaddress.ip_address(address[4][0])
389
+ if not ip.is_global:
390
+ raise HTTPException(status_code=400, detail="Document URL resolves to a private or local address")
391
+
392
+
393
+ async def _download(url: str, destination: Path) -> tuple[str, str, int]:
394
+ current = url
395
+ async with httpx.AsyncClient(
396
+ timeout=httpx.Timeout(120.0, connect=10.0),
397
+ follow_redirects=False,
398
+ trust_env=False,
399
+ ) as client:
400
+ for redirect in range(6):
401
+ try:
402
+ await asyncio.wait_for(asyncio.to_thread(_assert_public_host, current), timeout=10)
403
+ except asyncio.TimeoutError as exc:
404
+ raise HTTPException(status_code=400, detail="Document URL DNS lookup timed out") from exc
405
+ async with client.stream("GET", current, headers={"Accept": "application/pdf,application/*;q=.9,text/*;q=.5"}) as response:
406
+ if response.status_code in {301, 302, 303, 307, 308}:
407
+ location = response.headers.get("location")
408
+ if not location or redirect == 5:
409
+ raise HTTPException(status_code=400, detail="Too many or invalid document redirects")
410
+ current = urljoin(current, location)
411
+ continue
412
+ if response.status_code < 200 or response.status_code >= 300:
413
+ raise HTTPException(status_code=400, detail=f"Document download failed ({response.status_code})")
414
+ advertised = response.headers.get("content-length")
415
+ if advertised and advertised.isdigit() and int(advertised) > MAX_DOCUMENT_BYTES:
416
+ raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
417
+ total = 0
418
+ with destination.open("wb") as handle:
419
+ async for chunk in response.aiter_bytes(1024 * 1024):
420
+ total += len(chunk)
421
+ if total > MAX_DOCUMENT_BYTES:
422
+ raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
423
+ handle.write(chunk)
424
+ if total == 0:
425
+ raise HTTPException(status_code=400, detail="Downloaded document is empty")
426
+ content_type = response.headers.get("content-type", "").split(";", 1)[0]
427
+ parsed = urlsplit(str(response.url))
428
+ disposition = response.headers.get("content-disposition", "")
429
+ encoded_name = re.search(r"filename\*=UTF-8''([^;]+)", disposition, re.IGNORECASE)
430
+ plain_name = re.search(r'filename="?([^";]+)', disposition, re.IGNORECASE)
431
+ candidate_name = unquote(encoded_name.group(1)) if encoded_name else plain_name.group(1) if plain_name else Path(parsed.path).name
432
+ filename = _safe_name(unquote(candidate_name), "document")
433
+ if Path(filename).suffix.lower() not in SUPPORTED_EXTENSIONS:
434
+ inferred = MIME_EXTENSIONS.get(content_type.lower())
435
+ if inferred:
436
+ filename = f"{filename}{inferred}"
437
+ return filename, content_type, total
438
+ raise HTTPException(status_code=400, detail="Unable to download document")
439
+
440
+
441
+ def _redacted_url(value: str) -> str:
442
+ parsed = urlsplit(value)
443
+ return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
444
+
445
+
446
+ async def _finish_import(
447
+ *,
448
+ request: Request,
449
+ workspace_path: str,
450
+ project_id: str,
451
+ project_name: str,
452
+ filename: str,
453
+ content_type: str,
454
+ origin: Literal["upload", "url"],
455
+ source_url: str | None,
456
+ writer,
457
+ ) -> dict[str, Any]:
458
+ if not project_name.strip() or len(project_name) > 200:
459
+ raise HTTPException(status_code=400, detail="project_name must be between 1 and 200 characters")
460
+ if not project_id.strip() or len(project_id) > 200:
461
+ raise HTTPException(status_code=400, detail="project_id must be between 1 and 200 characters")
462
+ roots = _workspace_roots(request)
463
+ workspace = _validated_new_workspace(workspace_path, roots)
464
+ workspace.parent.mkdir(parents=True, exist_ok=True)
465
+ stage = workspace.parent / f".{workspace.name}.import-{uuid.uuid4().hex}"
466
+ stage.mkdir()
467
+ try:
468
+ source_dir = stage / "source"
469
+ internal = stage / ".crabcode" / "document"
470
+ source_dir.mkdir(parents=True)
471
+ internal.mkdir(parents=True)
472
+ safe_filename = _safe_name(filename)
473
+ extension = _file_extension(safe_filename)
474
+ _validate_mime(extension, content_type)
475
+ source = source_dir / safe_filename
476
+ size = await writer(source)
477
+ _validate_signature(source, extension)
478
+ rendered = await _convert_to_pdf(source, extension, internal)
479
+ page_count = await asyncio.to_thread(_pdf_page_count, rendered)
480
+ now = datetime.now(timezone.utc).isoformat()
481
+ manifest: dict[str, Any] = {
482
+ "schema_version": 1,
483
+ "project_id": project_id,
484
+ "project_name": project_name.strip(),
485
+ "workspace": str(workspace),
486
+ "source": {
487
+ "origin": origin,
488
+ "name": safe_filename,
489
+ "path": source.relative_to(stage).as_posix(),
490
+ "url": _redacted_url(source_url) if source_url else None,
491
+ "content_type": content_type or mimetypes.guess_type(safe_filename)[0] or "application/octet-stream",
492
+ "size": size,
493
+ "sha256": _sha256(source),
494
+ },
495
+ "pdf": {
496
+ "path": rendered.relative_to(stage).as_posix(),
497
+ "sha256": _sha256(rendered),
498
+ "page_count": page_count,
499
+ },
500
+ "layout": None,
501
+ "translations": {},
502
+ "blog": None,
503
+ "jobs": {},
504
+ "created_at": now,
505
+ "updated_at": now,
506
+ }
507
+ _json_write(stage / MANIFEST_RELATIVE_PATH, manifest)
508
+ replace_with_retry(stage, workspace)
509
+ return manifest
510
+ except Exception:
511
+ shutil.rmtree(stage, ignore_errors=True)
512
+ raise
513
+
514
+
515
+ @router.get("/capabilities")
516
+ async def document_capabilities(request: Request) -> dict[str, Any]:
517
+ from crabcode_gateway.document_engine import document_engine_status
518
+
519
+ converter = _converter()
520
+ precise = await asyncio.to_thread(document_engine_status)
521
+ info: WorkspaceInfo = request.app.state.workspace_info
522
+ available = [".pdf", *sorted(OFFICE_EXTENSIONS)] if converter else [".pdf"]
523
+ return {
524
+ "supported_extensions": sorted(SUPPORTED_EXTENSIONS),
525
+ "available_extensions": available,
526
+ "max_bytes": MAX_DOCUMENT_BYTES,
527
+ "documents_dir": info.documents_dir,
528
+ "libreoffice": {"available": bool(converter), "executable": converter},
529
+ "ocr": {"available": False},
530
+ "translation_engines": {
531
+ "default": "precise" if precise["available"] else "legacy",
532
+ "legacy": {"available": True, "status": "ready"},
533
+ "precise": precise,
534
+ },
535
+ }
536
+
537
+
538
+ @router.post("/import/upload", status_code=201)
539
+ async def import_document_upload(
540
+ request: Request,
541
+ workspace_path: str = Form(...),
542
+ project_id: str = Form(...),
543
+ project_name: str = Form(...),
544
+ file: UploadFile = File(...),
545
+ ) -> dict[str, Any]:
546
+ filename = _safe_name(file.filename or "document")
547
+
548
+ async def writer(path: Path) -> int:
549
+ return await _write_upload(file, path)
550
+
551
+ try:
552
+ return await _finish_import(
553
+ request=request,
554
+ workspace_path=workspace_path,
555
+ project_id=project_id,
556
+ project_name=project_name,
557
+ filename=filename,
558
+ content_type=file.content_type or "",
559
+ origin="upload",
560
+ source_url=None,
561
+ writer=writer,
562
+ )
563
+ finally:
564
+ await file.close()
565
+
566
+
567
+ @router.post("/import/url", status_code=201)
568
+ async def import_document_url(req: DocumentUrlImportRequest, request: Request) -> dict[str, Any]:
569
+ # Download into the source path selected from the URL. Resolve the filename
570
+ # before building the managed workspace so the original extension is kept.
571
+ with tempfile.TemporaryDirectory(prefix="crabcode-document-download-") as temp_dir:
572
+ download_path = Path(temp_dir) / "download"
573
+ filename, content_type, size = await _download(str(req.url), download_path)
574
+
575
+ async def copy_writer(path: Path) -> int:
576
+ shutil.copy2(download_path, path)
577
+ return size
578
+
579
+ return await _finish_import(
580
+ request=request,
581
+ workspace_path=req.workspace_path,
582
+ project_id=req.project_id,
583
+ project_name=req.project_name,
584
+ filename=filename,
585
+ content_type=content_type,
586
+ origin="url",
587
+ source_url=str(req.url),
588
+ writer=copy_writer,
589
+ )
590
+
591
+
592
+ @router.get("/manifest")
593
+ async def document_manifest(workspace: str, request: Request) -> dict[str, Any]:
594
+ return _read_manifest(_validated_workspace(workspace, _workspace_roots(request)))
595
+
596
+
597
+ @router.get("/asset")
598
+ async def document_asset(
599
+ workspace: str,
600
+ request: Request,
601
+ kind: Literal["pdf", "source"] = "pdf",
602
+ ) -> FileResponse:
603
+ root = _validated_workspace(workspace, _workspace_roots(request))
604
+ manifest = _read_manifest(root)
605
+ relative = manifest.get(kind, {}).get("path")
606
+ if not isinstance(relative, str):
607
+ raise HTTPException(status_code=404, detail="Document asset not found")
608
+ candidate = root / relative
609
+ if candidate.is_symlink():
610
+ raise HTTPException(status_code=403, detail="Document asset is invalid")
611
+ try:
612
+ path = candidate.resolve(strict=True)
613
+ except (FileNotFoundError, OSError) as exc:
614
+ raise HTTPException(status_code=404, detail="Document asset not found") from exc
615
+ if not _is_within(path, (root,)) or not path.is_file():
616
+ raise HTTPException(status_code=403, detail="Document asset is invalid")
617
+ media_type = "application/pdf" if kind == "pdf" else manifest.get("source", {}).get("content_type")
618
+ response = FileResponse(path, media_type=media_type, filename=path.name)
619
+ response.headers["Accept-Ranges"] = "bytes"
620
+ digest = manifest.get(kind, {}).get("sha256")
621
+ response.headers["ETag"] = f'"{digest if isinstance(digest, str) and digest else _sha256(path)}"'
622
+ return response
623
+
624
+
625
+ @router.get("/blog-asset")
626
+ async def document_blog_asset(workspace: str, path: str, request: Request) -> FileResponse:
627
+ root = _validated_workspace(workspace, _workspace_roots(request))
628
+ assets = root / "blog-assets"
629
+ try:
630
+ resolved_assets = assets.resolve(strict=True)
631
+ candidate = (assets / path).resolve(strict=True)
632
+ except (FileNotFoundError, OSError) as exc:
633
+ raise HTTPException(status_code=404, detail="Blog asset not found") from exc
634
+ if (
635
+ assets.is_symlink()
636
+ or not resolved_assets.is_dir()
637
+ or not _is_within(resolved_assets, (root,))
638
+ or not _is_within(candidate, (resolved_assets,))
639
+ or not candidate.is_file()
640
+ or candidate.is_symlink()
641
+ ):
642
+ raise HTTPException(status_code=403, detail="Blog asset is invalid")
643
+ return FileResponse(candidate, media_type=mimetypes.guess_type(candidate.name)[0] or "application/octet-stream")
644
+
645
+
646
+ @router.put("/layout")
647
+ async def save_document_layout(req: DocumentLayoutRequest, workspace: str, request: Request) -> dict[str, Any]:
648
+ root = _validated_workspace(workspace, _workspace_roots(request))
649
+ encoded = json.dumps(req.model_dump(), ensure_ascii=False).encode("utf-8")
650
+ if len(encoded) > MAX_LAYOUT_BYTES:
651
+ raise HTTPException(status_code=413, detail="Document layout exceeds the 50MiB limit")
652
+ try:
653
+ internal = _managed_internal_directory(root)
654
+ except ValueError as exc:
655
+ raise HTTPException(status_code=403, detail=str(exc)) from exc
656
+ with _manifest_lock(root):
657
+ return _save_document_layout_locked(root, internal, req)
658
+
659
+
660
+ def _document_annotations_path(root: Path) -> Path:
661
+ try:
662
+ internal = _managed_internal_directory(root)
663
+ except ValueError as exc:
664
+ raise HTTPException(status_code=403, detail=str(exc)) from exc
665
+ path = internal / "annotations.json"
666
+ if path.is_symlink():
667
+ raise HTTPException(status_code=403, detail="Document annotations path is invalid")
668
+ return path
669
+
670
+
671
+ def _read_document_annotations_locked(root: Path) -> list[dict[str, Any]]:
672
+ path = _document_annotations_path(root)
673
+ try:
674
+ value = json.loads(path.read_text(encoding="utf-8"))
675
+ except FileNotFoundError:
676
+ return []
677
+ except (OSError, json.JSONDecodeError) as exc:
678
+ raise HTTPException(status_code=500, detail="Document annotations are invalid") from exc
679
+ if not isinstance(value, list):
680
+ raise HTTPException(status_code=500, detail="Document annotations are invalid")
681
+ try:
682
+ return [DocumentAnnotationWriteRequest.model_validate(item).model_dump() for item in value]
683
+ except Exception as exc:
684
+ raise HTTPException(status_code=500, detail="Document annotations are invalid") from exc
685
+
686
+
687
+ @router.get("/annotations")
688
+ async def read_document_annotations(workspace: str, request: Request) -> list[dict[str, Any]]:
689
+ root = _validated_workspace(workspace, _workspace_roots(request))
690
+ with _manifest_lock(root):
691
+ return _read_document_annotations_locked(root)
692
+
693
+
694
+ @router.put("/annotations")
695
+ async def write_document_annotation(
696
+ req: DocumentAnnotationWriteRequest,
697
+ workspace: str,
698
+ request: Request,
699
+ ) -> dict[str, Any]:
700
+ root = _validated_workspace(workspace, _workspace_roots(request))
701
+ with _manifest_lock(root):
702
+ annotations = _read_document_annotations_locked(root)
703
+ now = datetime.now(timezone.utc).isoformat()
704
+ previous = next((item for item in annotations if item["id"] == req.id), None)
705
+ annotation = req.model_dump()
706
+ annotation["created_at"] = previous.get("created_at") if previous else now
707
+ annotation["updated_at"] = now
708
+ next_annotations = [item for item in annotations if item["id"] != req.id]
709
+ next_annotations.append(annotation)
710
+ _json_write(_document_annotations_path(root), next_annotations)
711
+ return annotation
712
+
713
+
714
+ @router.delete("/annotations/{annotation_id}", status_code=204)
715
+ async def delete_document_annotation(
716
+ annotation_id: str,
717
+ workspace: str,
718
+ request: Request,
719
+ ) -> Response:
720
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", annotation_id):
721
+ raise HTTPException(status_code=400, detail="Invalid annotation id")
722
+ root = _validated_workspace(workspace, _workspace_roots(request))
723
+ with _manifest_lock(root):
724
+ annotations = _read_document_annotations_locked(root)
725
+ next_annotations = [item for item in annotations if item["id"] != annotation_id]
726
+ if len(next_annotations) == len(annotations):
727
+ raise HTTPException(status_code=404, detail="Document annotation not found")
728
+ _json_write(_document_annotations_path(root), next_annotations)
729
+ return Response(status_code=204)
730
+
731
+
732
+ def _save_document_layout_locked(
733
+ root: Path,
734
+ internal: Path,
735
+ req: DocumentLayoutRequest,
736
+ ) -> dict[str, Any]:
737
+ layout_path = internal / "layout.json"
738
+ content_path = internal / "content.md"
739
+ if content_path.is_symlink():
740
+ raise HTTPException(status_code=403, detail="Document content path is invalid")
741
+ lines: list[str] = []
742
+ text_pages = 0
743
+ for page_number, page in enumerate(req.pages, 1):
744
+ blocks = page.get("blocks") if isinstance(page, dict) else None
745
+ page_lines = [str(block.get("text", "")).strip() for block in (blocks or []) if isinstance(block, dict)]
746
+ page_lines = [line for line in page_lines if line]
747
+ if page_lines:
748
+ text_pages += 1
749
+ lines.extend([f"\n<!-- page:{page_number} -->\n", *page_lines])
750
+ _json_write(layout_path, req.model_dump())
751
+ _text_write(content_path, "\n\n".join(lines).strip() + "\n")
752
+ manifest = _read_manifest(root)
753
+ previous_fingerprint = (manifest.get("layout") or {}).get("fingerprint")
754
+ if previous_fingerprint and previous_fingerprint != req.fingerprint:
755
+ translations = internal / "translations"
756
+ if translations.is_symlink():
757
+ translations.unlink(missing_ok=True)
758
+ elif translations.is_dir():
759
+ # A browser layout change invalidates only legacy block overlays.
760
+ # Precise PDFs are derived from the source PDF and remain valid.
761
+ for legacy_path in translations.glob("*.json"):
762
+ if legacy_path.is_file() and not legacy_path.is_symlink():
763
+ legacy_path.unlink(missing_ok=True)
764
+ (root / "blog.md").unlink(missing_ok=True)
765
+ current_translations = manifest.get("translations")
766
+ manifest["translations"] = {
767
+ locale: entry
768
+ for locale, entry in (current_translations.items() if isinstance(current_translations, dict) else [])
769
+ if isinstance(entry, dict) and entry.get("engine") == "precise"
770
+ }
771
+ manifest["blog"] = None
772
+ manifest["layout"] = {
773
+ "path": layout_path.relative_to(root).as_posix(),
774
+ "fingerprint": req.fingerprint,
775
+ "text_pages": text_pages,
776
+ "scanned_pages": req.page_count - text_pages,
777
+ }
778
+ manifest.setdefault("pdf", {})["page_count"] = req.page_count
779
+ manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
780
+ _json_write(root / MANIFEST_RELATIVE_PATH, manifest)
781
+ return manifest["layout"]
782
+
783
+
784
+ @router.get("/translation")
785
+ async def document_translation(workspace: str, locale: str, request: Request) -> dict[str, Any]:
786
+ root = _validated_workspace(workspace, _workspace_roots(request))
787
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
788
+ raise HTTPException(status_code=400, detail="Invalid translation locale")
789
+ try:
790
+ internal = _managed_internal_directory(root)
791
+ except ValueError as exc:
792
+ raise HTTPException(status_code=403, detail=str(exc)) from exc
793
+ with _manifest_lock(root):
794
+ return _read_document_translation_locked(root, internal, locale)
795
+
796
+
797
+ @router.delete("/translation")
798
+ async def clear_document_translation(workspace: str, locale: str, request: Request) -> dict[str, Any]:
799
+ root = _validated_workspace(workspace, _workspace_roots(request))
800
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
801
+ raise HTTPException(status_code=400, detail="Invalid translation locale")
802
+ try:
803
+ internal = _managed_internal_directory(root)
804
+ except ValueError as exc:
805
+ raise HTTPException(status_code=403, detail=str(exc)) from exc
806
+ with _manifest_lock(root):
807
+ return _clear_document_translation_locked(root, internal, locale)
808
+
809
+
810
+ def _read_document_translation_locked(root: Path, internal: Path, locale: str) -> dict[str, Any]:
811
+ translations = internal / "translations"
812
+ if translations.is_symlink():
813
+ raise HTTPException(status_code=403, detail="Managed translations directory is invalid")
814
+ manifest = _read_manifest(root)
815
+ entries = manifest.get("translations")
816
+ entry = entries.get(locale) if isinstance(entries, dict) else None
817
+ if isinstance(entry, dict) and entry.get("engine") == "precise":
818
+ relative = entry.get("path")
819
+ if not isinstance(relative, str):
820
+ raise HTTPException(status_code=500, detail="Precise translation metadata is invalid")
821
+ candidate = root / relative
822
+ if candidate.is_symlink():
823
+ raise HTTPException(status_code=403, detail="Precise translation path is invalid")
824
+ try:
825
+ path = candidate.resolve(strict=True)
826
+ except (FileNotFoundError, OSError) as exc:
827
+ raise HTTPException(status_code=404, detail="Translation not found") from exc
828
+ if not _is_within(path, (root.resolve(strict=True),)) or not path.is_file():
829
+ raise HTTPException(status_code=403, detail="Precise translation path is invalid")
830
+ source_sha256 = str(entry.get("source_sha256") or "")
831
+ if source_sha256 != str(manifest.get("source", {}).get("sha256") or ""):
832
+ raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
833
+ digest = str(entry.get("pdf_sha256") or entry.get("sha256") or "")
834
+ if digest and _sha256(path) != digest:
835
+ raise HTTPException(status_code=500, detail="Precise translation checksum failed")
836
+ return {
837
+ "engine": "precise",
838
+ "locale": locale,
839
+ "source_sha256": source_sha256,
840
+ "pdf_sha256": digest or _sha256(path),
841
+ "page_count": int(entry.get("page_count") or 0),
842
+ "engine_version": str(entry.get("engine_version") or ""),
843
+ "warnings": entry.get("warnings") if isinstance(entry.get("warnings"), list) else [],
844
+ }
845
+ path = translations / f"{locale}.json"
846
+ if path.is_symlink():
847
+ raise HTTPException(status_code=403, detail="Translation path is invalid")
848
+ try:
849
+ value = json.loads(path.read_text(encoding="utf-8"))
850
+ except FileNotFoundError as exc:
851
+ raise HTTPException(status_code=404, detail="Translation not found") from exc
852
+ except (OSError, json.JSONDecodeError) as exc:
853
+ raise HTTPException(status_code=500, detail="Translation is invalid") from exc
854
+ if not isinstance(value, dict):
855
+ raise HTTPException(status_code=500, detail="Translation is invalid")
856
+ if value.get("source_sha256") != manifest.get("source", {}).get("sha256"):
857
+ raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
858
+ if value.get("layout_fingerprint") != (manifest.get("layout") or {}).get("fingerprint"):
859
+ raise HTTPException(status_code=409, detail="Translation is stale for the current document layout")
860
+ return {**value, "engine": "legacy"}
861
+
862
+
863
+ @router.get("/translation/asset")
864
+ async def document_translation_asset(workspace: str, locale: str, request: Request) -> FileResponse:
865
+ root = _validated_workspace(workspace, _workspace_roots(request))
866
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
867
+ raise HTTPException(status_code=400, detail="Invalid translation locale")
868
+ with _manifest_lock(root):
869
+ manifest = _read_manifest(root)
870
+ entries = manifest.get("translations")
871
+ entry = entries.get(locale) if isinstance(entries, dict) else None
872
+ if not isinstance(entry, dict) or entry.get("engine") != "precise":
873
+ raise HTTPException(status_code=404, detail="Precise translation not found")
874
+ if entry.get("source_sha256") != manifest.get("source", {}).get("sha256"):
875
+ raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
876
+ relative = entry.get("path")
877
+ if not isinstance(relative, str):
878
+ raise HTTPException(status_code=500, detail="Precise translation metadata is invalid")
879
+ candidate = root / relative
880
+ if candidate.is_symlink():
881
+ raise HTTPException(status_code=403, detail="Precise translation path is invalid")
882
+ try:
883
+ path = candidate.resolve(strict=True)
884
+ except (FileNotFoundError, OSError) as exc:
885
+ raise HTTPException(status_code=404, detail="Precise translation not found") from exc
886
+ if not _is_within(path, (root.resolve(strict=True),)) or not path.is_file():
887
+ raise HTTPException(status_code=403, detail="Precise translation path is invalid")
888
+ actual_digest = _sha256(path)
889
+ digest = str(entry.get("pdf_sha256") or entry.get("sha256") or "")
890
+ if digest and digest != actual_digest:
891
+ raise HTTPException(status_code=500, detail="Precise translation checksum failed")
892
+ digest = digest or actual_digest
893
+ response = FileResponse(path, media_type="application/pdf", filename=f"translated-{locale}.pdf")
894
+ response.headers["Accept-Ranges"] = "bytes"
895
+ response.headers["ETag"] = f'"{digest}"'
896
+ return response
897
+
898
+
899
+ def _clear_document_translation_locked(root: Path, internal: Path, locale: str) -> dict[str, Any]:
900
+ manifest = _read_manifest(root)
901
+ jobs = manifest.get("jobs")
902
+ if not isinstance(jobs, dict):
903
+ jobs = {}
904
+ matching_job_ids = {
905
+ operation_id
906
+ for operation_id, job in jobs.items()
907
+ if isinstance(operation_id, str)
908
+ and re.fullmatch(r"[A-Za-z0-9_-]{1,100}", operation_id)
909
+ and isinstance(job, dict)
910
+ and job.get("action") == "translate"
911
+ and job.get("locale") == locale
912
+ }
913
+ if any(
914
+ isinstance(job, dict)
915
+ and job.get("action") == "translate"
916
+ and job.get("locale") == locale
917
+ and job.get("status") == "running"
918
+ for job in jobs.values()
919
+ ):
920
+ raise HTTPException(status_code=409, detail="Translation is still running")
921
+
922
+ jobs_dir = internal / "jobs"
923
+ if jobs_dir.is_symlink() or (jobs_dir.exists() and not jobs_dir.is_dir()):
924
+ raise HTTPException(status_code=403, detail="Managed document jobs directory is invalid")
925
+ if jobs_dir.is_dir():
926
+ for job_dir in jobs_dir.iterdir():
927
+ if job_dir.is_symlink() or not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", job_dir.name):
928
+ continue
929
+ staged = job_dir / "translation.json"
930
+ precise_cache = job_dir / "precise-cache.json"
931
+ for candidate in (staged, precise_cache):
932
+ if not candidate.is_file() or candidate.is_symlink():
933
+ continue
934
+ value = _read_json_file(candidate)
935
+ if isinstance(value, dict) and value.get("locale") == locale:
936
+ matching_job_ids.add(job_dir.name)
937
+ break
938
+
939
+ translations_dir = internal / "translations"
940
+ if translations_dir.is_symlink() or (translations_dir.exists() and not translations_dir.is_dir()):
941
+ raise HTTPException(status_code=403, detail="Managed translations directory is invalid")
942
+ translation_path = translations_dir / f"{locale}.json"
943
+ if translation_path.is_symlink() or (translation_path.exists() and not translation_path.is_file()):
944
+ raise HTTPException(status_code=403, detail="Translation path is invalid")
945
+ precise_path = translations_dir / locale
946
+ if precise_path.is_symlink() or (precise_path.exists() and not precise_path.is_dir()):
947
+ raise HTTPException(status_code=403, detail="Precise translation directory is invalid")
948
+ removed_translation = translation_path.is_file() or precise_path.is_dir()
949
+ translation_path.unlink(missing_ok=True)
950
+ if precise_path.is_dir():
951
+ shutil.rmtree(precise_path)
952
+
953
+ for operation_id in matching_job_ids:
954
+ _cleanup_document_job(root, operation_id)
955
+
956
+ translations = manifest.get("translations")
957
+ if not isinstance(translations, dict):
958
+ translations = {}
959
+ translations.pop(locale, None)
960
+ manifest["translations"] = translations
961
+ manifest["jobs"] = {
962
+ operation_id: job
963
+ for operation_id, job in jobs.items()
964
+ if operation_id not in matching_job_ids
965
+ }
966
+ manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
967
+ _json_write(root / MANIFEST_RELATIVE_PATH, manifest)
968
+ return {
969
+ "locale": locale,
970
+ "removed_translation": removed_translation,
971
+ "removed_jobs": len(matching_job_ids),
972
+ }
973
+
974
+
975
+ def _blog_revision(markdown: str) -> str:
976
+ return hashlib.sha256(markdown.encode("utf-8")).hexdigest()
977
+
978
+
979
+ def _managed_internal_directory(workspace: Path) -> Path:
980
+ try:
981
+ resolved_workspace = workspace.resolve(strict=True)
982
+ except OSError as exc:
983
+ raise ValueError("document workspace is unavailable") from exc
984
+ crabcode = workspace / ".crabcode"
985
+ internal = crabcode / "document"
986
+ if crabcode.is_symlink() or internal.is_symlink():
987
+ raise ValueError("managed document directories cannot be symlinks")
988
+ try:
989
+ resolved_internal = internal.resolve(strict=True)
990
+ except OSError as exc:
991
+ raise ValueError("managed document directory is unavailable") from exc
992
+ if not resolved_internal.is_dir() or not _is_within(resolved_internal, (resolved_workspace,)):
993
+ raise ValueError("managed document directory is outside the workspace")
994
+ return internal
995
+
996
+
997
+ def _document_job_directory(workspace: Path, operation_id: str) -> Path:
998
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", operation_id):
999
+ raise ValueError("invalid document operation id")
1000
+ internal = _managed_internal_directory(workspace)
1001
+ jobs = internal / "jobs"
1002
+ if jobs.is_symlink() or not _is_within(jobs.resolve(strict=False), (workspace.resolve(strict=True),)):
1003
+ raise ValueError("document jobs directory is invalid")
1004
+ return jobs / operation_id
1005
+
1006
+
1007
+ def _document_job_total(workspace: Path, action: str, engine: str = "legacy") -> int:
1008
+ if action != "translate":
1009
+ return 1
1010
+ if engine == "precise":
1011
+ return 100
1012
+ return sum(len(page) for page in _translation_source_pages(workspace))
1013
+
1014
+
1015
+ def _document_pdf_path(workspace: Path) -> Path:
1016
+ manifest = _read_manifest(workspace)
1017
+ relative = manifest.get("pdf", {}).get("path")
1018
+ if not isinstance(relative, str):
1019
+ raise ValueError("document PDF is unavailable")
1020
+ candidate = workspace / relative
1021
+ if candidate.is_symlink():
1022
+ raise ValueError("document PDF path is invalid")
1023
+ try:
1024
+ path = candidate.resolve(strict=True)
1025
+ except OSError as exc:
1026
+ raise ValueError("document PDF is unavailable") from exc
1027
+ if not path.is_file() or not _is_within(path, (workspace.resolve(strict=True),)):
1028
+ raise ValueError("document PDF path is invalid")
1029
+ return path
1030
+
1031
+
1032
+ def _translation_source_pages(workspace: Path) -> list[list[tuple[str, str]]]:
1033
+ """Return exact non-empty layout blocks grouped by page."""
1034
+ try:
1035
+ internal = _managed_internal_directory(workspace)
1036
+ layout_path = internal / "layout.json"
1037
+ if layout_path.is_symlink():
1038
+ raise ValueError("document layout is invalid")
1039
+ layout = json.loads(layout_path.read_text(encoding="utf-8"))
1040
+ raw_pages = layout.get("pages")
1041
+ if not isinstance(raw_pages, list):
1042
+ raise ValueError("document layout is not ready")
1043
+ except (OSError, json.JSONDecodeError, AttributeError, TypeError) as exc:
1044
+ raise ValueError("document layout is not ready") from exc
1045
+
1046
+ pages: list[list[tuple[str, str]]] = []
1047
+ seen: set[str] = set()
1048
+ for raw_page in raw_pages:
1049
+ if not isinstance(raw_page, dict) or not isinstance(raw_page.get("blocks", []), list):
1050
+ raise ValueError("document layout contains invalid pages")
1051
+ page: list[tuple[str, str]] = []
1052
+ for block in raw_page.get("blocks", []):
1053
+ if not isinstance(block, dict):
1054
+ raise ValueError("document layout contains invalid blocks")
1055
+ text = block.get("text")
1056
+ if not isinstance(text, str) or not text.strip():
1057
+ continue
1058
+ block_id = block.get("id")
1059
+ if not isinstance(block_id, str) or not block_id or block_id in seen:
1060
+ raise ValueError("document layout contains invalid or duplicate block ids")
1061
+ seen.add(block_id)
1062
+ page.append((block_id, text))
1063
+ pages.append(page)
1064
+ return pages
1065
+
1066
+
1067
+ def _translation_preserved_blocks(workspace: Path) -> dict[str, str]:
1068
+ """Return fixed-layout blocks that must bypass model translation."""
1069
+ try:
1070
+ internal = _managed_internal_directory(workspace)
1071
+ layout_path = internal / "layout.json"
1072
+ if layout_path.is_symlink():
1073
+ raise ValueError("document layout is invalid")
1074
+ layout = json.loads(layout_path.read_text(encoding="utf-8"))
1075
+ raw_pages = layout.get("pages")
1076
+ if not isinstance(raw_pages, list):
1077
+ raise ValueError("document layout is not ready")
1078
+ except (OSError, json.JSONDecodeError, AttributeError, TypeError) as exc:
1079
+ raise ValueError("document layout is not ready") from exc
1080
+
1081
+ preserved: dict[str, str] = {}
1082
+ for raw_page in raw_pages:
1083
+ if not isinstance(raw_page, dict) or not isinstance(raw_page.get("blocks", []), list):
1084
+ raise ValueError("document layout contains invalid pages")
1085
+ for block in raw_page.get("blocks", []):
1086
+ if not isinstance(block, dict) or block.get("kind") not in {"formula", "graphic"}:
1087
+ continue
1088
+ block_id = block.get("id")
1089
+ text = block.get("text")
1090
+ if not isinstance(block_id, str) or not block_id or not isinstance(text, str) or not text.strip():
1091
+ raise ValueError("document layout contains invalid preserved blocks")
1092
+ preserved[block_id] = text
1093
+ return preserved
1094
+
1095
+
1096
+ def _validated_translation_file(
1097
+ workspace: Path,
1098
+ path: Path,
1099
+ locale: str | None,
1100
+ ) -> dict[str, str]:
1101
+ if path.is_symlink() or path.parent.is_symlink():
1102
+ raise ValueError("translation job file is invalid")
1103
+ try:
1104
+ staged = json.loads(path.read_text(encoding="utf-8"))
1105
+ manifest = _read_manifest(workspace)
1106
+ except (OSError, json.JSONDecodeError) as exc:
1107
+ raise ValueError("translation job file is invalid") from exc
1108
+ if not isinstance(staged, dict):
1109
+ raise ValueError("translation job file is invalid")
1110
+ staged_locale = staged.get("locale")
1111
+ if locale is not None and staged_locale is not None and staged_locale != locale:
1112
+ raise ValueError("translation job locale does not match")
1113
+ if staged.get("source_sha256") != manifest.get("source", {}).get("sha256"):
1114
+ raise ValueError("translation was generated for a different source document")
1115
+ if staged.get("layout_fingerprint") != (manifest.get("layout") or {}).get("fingerprint"):
1116
+ raise ValueError("translation was generated for a different document layout")
1117
+ blocks = staged.get("blocks")
1118
+ if not isinstance(blocks, list):
1119
+ raise ValueError("translation JSON is missing blocks")
1120
+
1121
+ expected_ids = {
1122
+ block_id
1123
+ for page in _translation_source_pages(workspace)
1124
+ for block_id, _ in page
1125
+ }
1126
+ translated: dict[str, str] = {}
1127
+ for block in blocks:
1128
+ if not isinstance(block, dict):
1129
+ raise ValueError("translation blocks must contain id and translated_text")
1130
+ block_id = block.get("id")
1131
+ translated_text = block.get("translated_text")
1132
+ if not isinstance(block_id, str) or not isinstance(translated_text, str):
1133
+ raise ValueError("translation blocks must contain id and translated_text")
1134
+ if not translated_text.strip():
1135
+ raise ValueError(f"translation is empty for block id: {block_id}")
1136
+ if block_id in translated:
1137
+ raise ValueError(f"translation contains duplicate block id: {block_id}")
1138
+ if block_id not in expected_ids:
1139
+ raise ValueError(f"translation contains unknown block id: {block_id}")
1140
+ translated[block_id] = translated_text
1141
+ return translated
1142
+
1143
+
1144
+ def _write_translation_mapping(
1145
+ workspace: Path,
1146
+ operation_id: str,
1147
+ locale: str,
1148
+ translated: dict[str, str],
1149
+ ) -> int:
1150
+ pages = _translation_source_pages(workspace)
1151
+ expected_ids = [block_id for page in pages for block_id, _ in page]
1152
+ expected = set(expected_ids)
1153
+ if not set(translated).issubset(expected):
1154
+ raise ValueError("translation contains unknown block ids")
1155
+ if any(not isinstance(text, str) or not text.strip() for text in translated.values()):
1156
+ raise ValueError("translation contains empty text")
1157
+ manifest = _read_manifest(workspace)
1158
+ target = _document_job_directory(workspace, operation_id) / "translation.json"
1159
+ if target.is_symlink() or target.parent.is_symlink():
1160
+ raise ValueError("translation job file is invalid")
1161
+ _json_write(target, {
1162
+ "locale": locale,
1163
+ "source_sha256": manifest.get("source", {}).get("sha256"),
1164
+ "layout_fingerprint": (manifest.get("layout") or {}).get("fingerprint"),
1165
+ "blocks": [
1166
+ {"id": block_id, "translated_text": translated[block_id]}
1167
+ for block_id in expected_ids
1168
+ if block_id in translated
1169
+ ],
1170
+ })
1171
+ return len(translated)
1172
+
1173
+
1174
+ def _translation_job_blocks(
1175
+ workspace: Path,
1176
+ operation_id: str,
1177
+ locale: str,
1178
+ ) -> dict[str, str]:
1179
+ target = _document_job_directory(workspace, operation_id) / "translation.json"
1180
+ if not target.is_file():
1181
+ return {}
1182
+ return _validated_translation_file(workspace, target, locale)
1183
+
1184
+
1185
+ def _store_translation_batch(
1186
+ workspace: Path,
1187
+ operation_id: str,
1188
+ locale: str,
1189
+ batch: dict[str, str],
1190
+ ) -> int:
1191
+ """Validate and atomically merge one model-produced translation batch."""
1192
+ translated = _translation_job_blocks(workspace, operation_id, locale)
1193
+ translated.update(batch)
1194
+ return _write_translation_mapping(workspace, operation_id, locale, translated)
1195
+
1196
+
1197
+ def _cleanup_document_job(workspace: Path, operation_id: str) -> None:
1198
+ try:
1199
+ job = _document_job_directory(workspace, operation_id)
1200
+ except ValueError:
1201
+ return
1202
+ if job.is_symlink():
1203
+ job.unlink(missing_ok=True)
1204
+ else:
1205
+ shutil.rmtree(job, ignore_errors=True)
1206
+
1207
+
1208
+ def _translation_job_progress(workspace: Path, operation_id: str) -> int:
1209
+ path = _document_job_directory(workspace, operation_id) / "translation.json"
1210
+ if not path.is_file() or path.is_symlink():
1211
+ return 0
1212
+ try:
1213
+ return len(_validated_translation_file(workspace, path, None))
1214
+ except ValueError:
1215
+ return 0
1216
+
1217
+
1218
+ def _recover_translation_job(
1219
+ workspace: Path,
1220
+ operation_id: str,
1221
+ locale: str,
1222
+ engine: str = "legacy",
1223
+ ) -> int:
1224
+ """Seed a new translation operation from the newest valid partial batch."""
1225
+ internal = _managed_internal_directory(workspace)
1226
+ jobs = internal / "jobs"
1227
+ if jobs.is_symlink():
1228
+ raise ValueError("document jobs directory is invalid")
1229
+ target = _document_job_directory(workspace, operation_id)
1230
+ target.mkdir(parents=True, exist_ok=True)
1231
+ if engine == "precise":
1232
+ manifest = _read_manifest(workspace)
1233
+ source_sha256 = str(manifest.get("source", {}).get("sha256") or "")
1234
+ candidates_with_time: list[tuple[float, Path]] = []
1235
+ for path in jobs.glob("*/precise-cache.json"):
1236
+ if path.parent.name == operation_id or path.is_symlink() or path.parent.is_symlink():
1237
+ continue
1238
+ try:
1239
+ candidates_with_time.append((path.stat().st_mtime, path))
1240
+ except OSError:
1241
+ continue
1242
+ for _, candidate in sorted(candidates_with_time, reverse=True):
1243
+ value = _read_json_file(candidate)
1244
+ if (
1245
+ isinstance(value, dict)
1246
+ and value.get("locale") == locale
1247
+ and value.get("source_sha256") == source_sha256
1248
+ and value.get("engine_version") == "0.6.4"
1249
+ and isinstance(value.get("entries"), dict)
1250
+ ):
1251
+ shutil.copy2(candidate, target / "precise-cache.json")
1252
+ # Cached paragraph count is not a stable percentage because
1253
+ # BabelDOC has not parsed this run yet.
1254
+ return 0
1255
+ return 0
1256
+ _translation_source_pages(workspace)
1257
+ candidates_with_time: list[tuple[float, Path]] = []
1258
+ for path in jobs.glob("*/translation.json"):
1259
+ if path.parent.name == operation_id or path.is_symlink() or path.parent.is_symlink():
1260
+ continue
1261
+ try:
1262
+ candidates_with_time.append((path.stat().st_mtime, path))
1263
+ except OSError:
1264
+ continue
1265
+ candidates = [path for _, path in sorted(candidates_with_time, reverse=True)]
1266
+ for candidate in candidates:
1267
+ try:
1268
+ translated = _validated_translation_file(workspace, candidate, locale)
1269
+ except ValueError:
1270
+ continue
1271
+ if translated:
1272
+ return _write_translation_mapping(workspace, operation_id, locale, translated)
1273
+ return 0
1274
+
1275
+
1276
+ def _read_json_file(path: Path) -> dict[str, Any] | None:
1277
+ try:
1278
+ value = json.loads(path.read_text(encoding="utf-8"))
1279
+ except (OSError, json.JSONDecodeError):
1280
+ return None
1281
+ return value if isinstance(value, dict) else None
1282
+
1283
+
1284
+ def _update_document_job_status(
1285
+ workspace: Path,
1286
+ operation_id: str,
1287
+ *,
1288
+ action: str,
1289
+ status: str,
1290
+ locale: str,
1291
+ source: str,
1292
+ current: int,
1293
+ total: int,
1294
+ message: str,
1295
+ language: str = "",
1296
+ engine: str = "legacy",
1297
+ ) -> None:
1298
+ with _manifest_lock(workspace):
1299
+ _update_document_job_status_unlocked(
1300
+ workspace,
1301
+ operation_id,
1302
+ action=action,
1303
+ status=status,
1304
+ locale=locale,
1305
+ source=source,
1306
+ current=current,
1307
+ total=total,
1308
+ message=message,
1309
+ language=language,
1310
+ engine=engine,
1311
+ )
1312
+
1313
+
1314
+ def _update_document_job_status_unlocked(
1315
+ workspace: Path,
1316
+ operation_id: str,
1317
+ *,
1318
+ action: str,
1319
+ status: str,
1320
+ locale: str,
1321
+ source: str,
1322
+ current: int,
1323
+ total: int,
1324
+ message: str,
1325
+ language: str = "",
1326
+ engine: str = "legacy",
1327
+ ) -> None:
1328
+ _managed_internal_directory(workspace)
1329
+ manifest = _read_manifest(workspace)
1330
+ jobs = manifest.get("jobs")
1331
+ if not isinstance(jobs, dict):
1332
+ jobs = {}
1333
+ jobs[operation_id] = {
1334
+ "action": action,
1335
+ "status": status,
1336
+ "locale": locale,
1337
+ "language": language,
1338
+ "source": source,
1339
+ "engine": engine,
1340
+ "current": current,
1341
+ "total": total,
1342
+ "message": message[:500],
1343
+ "updated_at": datetime.now(timezone.utc).isoformat(),
1344
+ }
1345
+ if len(jobs) > 50:
1346
+ jobs = dict(sorted(jobs.items(), key=lambda item: item[1].get("updated_at", ""), reverse=True)[:50])
1347
+ manifest["jobs"] = jobs
1348
+ manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
1349
+ _json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
1350
+
1351
+
1352
+ def _document_action_hash(
1353
+ workspace: Path,
1354
+ source: str,
1355
+ locale: str,
1356
+ engine: str = "legacy",
1357
+ ) -> str:
1358
+ with _manifest_lock(workspace):
1359
+ return _document_action_hash_unlocked(workspace, source, locale, engine)
1360
+
1361
+
1362
+ def _document_action_hash_unlocked(
1363
+ workspace: Path,
1364
+ source: str,
1365
+ locale: str,
1366
+ engine: str = "legacy",
1367
+ ) -> str:
1368
+ internal = _managed_internal_directory(workspace)
1369
+ manifest = _read_manifest(workspace)
1370
+ source_hash = str(manifest.get("source", {}).get("sha256", ""))
1371
+ layout_hash = str((manifest.get("layout") or {}).get("fingerprint", ""))
1372
+ if not source_hash or (engine == "legacy" and not layout_hash):
1373
+ raise ValueError("document layout is not ready" if engine == "legacy" else "document source is not ready")
1374
+ selected_hash = source_hash
1375
+ if source == "translation":
1376
+ entries = manifest.get("translations")
1377
+ entry = entries.get(locale) if isinstance(entries, dict) else None
1378
+ relative = entry.get("path") if isinstance(entry, dict) else None
1379
+ translation = workspace / relative if isinstance(relative, str) else internal / "translations" / f"{locale}.json"
1380
+ if not translation.is_file() or translation.is_symlink() or not _is_within(
1381
+ translation.resolve(strict=True),
1382
+ (workspace.resolve(strict=True),),
1383
+ ):
1384
+ raise ValueError("selected translation is not available")
1385
+ selected_hash = _sha256(translation)
1386
+ engine_version = ""
1387
+ if engine == "precise":
1388
+ from crabcode_gateway.document_engine import BABELDOC_VERSION
1389
+
1390
+ engine_version = BABELDOC_VERSION
1391
+ return hashlib.sha256(
1392
+ f"{source_hash}:{layout_hash if engine == 'legacy' else ''}:{source}:{selected_hash}:{engine}:{engine_version}".encode("utf-8")
1393
+ ).hexdigest()
1394
+
1395
+
1396
+ def _finalize_precise_translation_unlocked(
1397
+ workspace: Path,
1398
+ internal: Path,
1399
+ job: Path,
1400
+ manifest: dict[str, Any],
1401
+ operation_id: str,
1402
+ locale: str,
1403
+ now: str,
1404
+ ) -> tuple[int, int]:
1405
+ from crabcode_gateway.document_engine import BABELDOC_VERSION
1406
+
1407
+ staged_pdf = job / "translated.pdf"
1408
+ staged_content = job / "content.json"
1409
+ staged_diagnostics = job / "diagnostics.json"
1410
+ for path in (staged_pdf, staged_content, staged_diagnostics):
1411
+ if path.is_symlink() or not path.is_file():
1412
+ raise ValueError(f"precise translation is missing {path.name}")
1413
+ if staged_pdf.stat().st_size <= 8 or staged_pdf.stat().st_size > MAX_DOCUMENT_BYTES * 4:
1414
+ raise ValueError("precise translated PDF has an invalid size")
1415
+ if staged_content.stat().st_size > 50 * 1024 * 1024 or staged_diagnostics.stat().st_size > 5 * 1024 * 1024:
1416
+ raise ValueError("precise translation metadata is too large")
1417
+ with staged_pdf.open("rb") as handle:
1418
+ if b"%PDF-" not in handle.read(1024):
1419
+ raise ValueError("precise translation did not produce a valid PDF")
1420
+ diagnostics = _read_json_file(staged_diagnostics)
1421
+ if diagnostics is None:
1422
+ raise ValueError("precise translation diagnostics are invalid")
1423
+ content = _read_json_file(staged_content)
1424
+ content_pages = content.get("pages") if isinstance(content, dict) else None
1425
+ if (
1426
+ content is None
1427
+ or content.get("schema_version") != 1
1428
+ or content.get("engine") != "precise"
1429
+ or not isinstance(content_pages, list)
1430
+ ):
1431
+ raise ValueError("precise translation content is invalid")
1432
+ source_sha256 = str(manifest.get("source", {}).get("sha256") or "")
1433
+ if diagnostics.get("source_sha256") != source_sha256:
1434
+ raise ValueError("precise translation was generated for a different source document")
1435
+ if diagnostics.get("engine_version") != BABELDOC_VERSION:
1436
+ raise ValueError("precise translation engine version does not match")
1437
+
1438
+ source_pdf = _document_pdf_path(workspace)
1439
+ try:
1440
+ source_reader = PdfReader(str(source_pdf))
1441
+ translated_reader = PdfReader(str(staged_pdf))
1442
+ except Exception as exc:
1443
+ raise ValueError("precise translated PDF cannot be opened") from exc
1444
+ if len(source_reader.pages) != len(translated_reader.pages):
1445
+ raise ValueError("precise translated PDF page count changed")
1446
+ if len(content_pages) != len(translated_reader.pages):
1447
+ raise ValueError("precise translation content page count changed")
1448
+ source_has_text = any((page.extract_text() or "").strip() for page in source_reader.pages)
1449
+ translated_has_text = any((page.extract_text() or "").strip() for page in translated_reader.pages)
1450
+ if source_has_text and not translated_has_text:
1451
+ raise ValueError("precise translated PDF has no selectable text")
1452
+ for source_page, translated_page in zip(source_reader.pages, translated_reader.pages, strict=True):
1453
+ for source_box, translated_box in (
1454
+ (source_page.mediabox, translated_page.mediabox),
1455
+ (source_page.cropbox, translated_page.cropbox),
1456
+ ):
1457
+ source_size = (float(source_box.width), float(source_box.height))
1458
+ translated_size = (float(translated_box.width), float(translated_box.height))
1459
+ if any(abs(left - right) > 0.5 for left, right in zip(source_size, translated_size, strict=True)):
1460
+ raise ValueError("precise translated PDF page dimensions changed")
1461
+ if int(source_page.get("/Rotate", 0) or 0) % 360 != int(translated_page.get("/Rotate", 0) or 0) % 360:
1462
+ raise ValueError("precise translated PDF page rotation changed")
1463
+
1464
+ translations_dir = internal / "translations"
1465
+ if translations_dir.is_symlink():
1466
+ raise ValueError("managed translations directory is invalid")
1467
+ translations_dir.mkdir(parents=True, exist_ok=True)
1468
+ target = translations_dir / locale
1469
+ if target.is_symlink() or (target.exists() and not target.is_dir()):
1470
+ raise ValueError("precise translation output directory is invalid")
1471
+ stage = translations_dir / f".{locale}-stage-{uuid.uuid4().hex}"
1472
+ backup = translations_dir / f".{locale}-backup-{uuid.uuid4().hex}"
1473
+ stage.mkdir()
1474
+ replaced = False
1475
+ try:
1476
+ shutil.copy2(staged_pdf, stage / "translated.pdf")
1477
+ shutil.copy2(staged_content, stage / "content.json")
1478
+ shutil.copy2(staged_diagnostics, stage / "diagnostics.json")
1479
+ digest = _sha256(stage / "translated.pdf")
1480
+ warnings = diagnostics.get("warnings") if isinstance(diagnostics.get("warnings"), list) else []
1481
+ metadata = {
1482
+ "schema_version": 1,
1483
+ "engine": "precise",
1484
+ "locale": locale,
1485
+ "source_sha256": source_sha256,
1486
+ "pdf_sha256": digest,
1487
+ "page_count": len(translated_reader.pages),
1488
+ "engine_version": BABELDOC_VERSION,
1489
+ "warnings": [str(item)[:500] for item in warnings],
1490
+ "updated_at": now,
1491
+ }
1492
+ _json_write(stage / "metadata.json", metadata)
1493
+ if target.exists():
1494
+ replace_with_retry(target, backup)
1495
+ replaced = True
1496
+ replace_with_retry(stage, target)
1497
+ translations = manifest.get("translations")
1498
+ if not isinstance(translations, dict):
1499
+ translations = {}
1500
+ manifest["translations"] = translations
1501
+ translations[locale] = {
1502
+ "engine": "precise",
1503
+ "path": (target / "translated.pdf").relative_to(workspace).as_posix(),
1504
+ "content_path": (target / "content.json").relative_to(workspace).as_posix(),
1505
+ "diagnostics_path": (target / "diagnostics.json").relative_to(workspace).as_posix(),
1506
+ "source_sha256": source_sha256,
1507
+ "pdf_sha256": digest,
1508
+ "page_count": len(translated_reader.pages),
1509
+ "engine_version": BABELDOC_VERSION,
1510
+ "warnings": metadata["warnings"],
1511
+ "updated_at": now,
1512
+ }
1513
+ manifest["updated_at"] = now
1514
+ try:
1515
+ _json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
1516
+ except Exception:
1517
+ shutil.rmtree(target, ignore_errors=True)
1518
+ if replaced and backup.exists():
1519
+ replace_with_retry(backup, target)
1520
+ raise
1521
+ shutil.rmtree(backup, ignore_errors=True)
1522
+ except Exception:
1523
+ if replaced and backup.exists() and not target.exists():
1524
+ replace_with_retry(backup, target)
1525
+ raise
1526
+ finally:
1527
+ shutil.rmtree(stage, ignore_errors=True)
1528
+ _cleanup_document_job(workspace, operation_id)
1529
+ return 100, 100
1530
+
1531
+
1532
+ def _finalize_document_job_unlocked(
1533
+ workspace: Path,
1534
+ operation_id: str,
1535
+ action: str,
1536
+ locale: str,
1537
+ source: str,
1538
+ expected_document_hash: str = "",
1539
+ language: str = "",
1540
+ engine: str = "legacy",
1541
+ ) -> tuple[int, int]:
1542
+ """Validate an Agent's staged artifact and publish it atomically."""
1543
+ if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
1544
+ raise ValueError("invalid document locale")
1545
+ if language and not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", language):
1546
+ raise ValueError("invalid Blog language")
1547
+ if engine not in {"legacy", "precise"}:
1548
+ raise ValueError("invalid document translation engine")
1549
+ if expected_document_hash and _document_action_hash(workspace, source, locale, engine) != expected_document_hash:
1550
+ raise ValueError("source document changed while the operation was running")
1551
+ internal = _managed_internal_directory(workspace)
1552
+ manifest = _read_manifest(workspace)
1553
+ now = datetime.now(timezone.utc).isoformat()
1554
+ if action == "generate_blog":
1555
+ # Finalization may be retried after the first call has atomically moved
1556
+ # blog.md and removed the staging directory. Treat that exact publish
1557
+ # as idempotently complete instead of reporting a missing artifact.
1558
+ published = manifest.get("blog")
1559
+ expected_language = locale if language == "source" and source == "translation" else language
1560
+ expected_language = expected_language or (locale if source == "translation" else "source")
1561
+ output = workspace / "blog.md"
1562
+ if (
1563
+ isinstance(published, dict)
1564
+ and published.get("operation_id") == operation_id
1565
+ and published.get("source") == source
1566
+ and output.is_file()
1567
+ and not output.is_symlink()
1568
+ ):
1569
+ if published.get("language") != expected_language:
1570
+ published["language"] = expected_language
1571
+ manifest["updated_at"] = now
1572
+ _json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
1573
+ return 1, 1
1574
+ job = _document_job_directory(workspace, operation_id)
1575
+ if job.is_symlink():
1576
+ raise ValueError("document job directory cannot be a symlink")
1577
+ if action == "translate" and engine == "precise":
1578
+ return _finalize_precise_translation_unlocked(
1579
+ workspace,
1580
+ internal,
1581
+ job,
1582
+ manifest,
1583
+ operation_id,
1584
+ locale,
1585
+ now,
1586
+ )
1587
+ blog_backup: bytes | None = None
1588
+ blog_existed = False
1589
+ translation_output: Path | None = None
1590
+ translation_backup: bytes | None = None
1591
+ translation_existed = False
1592
+ if action == "translate":
1593
+ total = _document_job_total(workspace, action, engine)
1594
+ try:
1595
+ layout_path = internal / "layout.json"
1596
+ staged_path = job / "translation.json"
1597
+ if layout_path.is_symlink() or staged_path.is_symlink():
1598
+ raise ValueError("document translation inputs cannot be symlinks")
1599
+ staged = json.loads(staged_path.read_text(encoding="utf-8"))
1600
+ except (OSError, json.JSONDecodeError) as exc:
1601
+ raise ValueError("Agent did not produce valid translation JSON") from exc
1602
+ expected_ids = [
1603
+ block_id
1604
+ for page in _translation_source_pages(workspace)
1605
+ for block_id, _ in page
1606
+ ]
1607
+ blocks = staged.get("blocks") if isinstance(staged, dict) else None
1608
+ if not isinstance(blocks, list):
1609
+ raise ValueError("translation JSON is missing blocks")
1610
+ if staged.get("locale") not in {None, locale}:
1611
+ raise ValueError("translation job locale does not match")
1612
+ translated: dict[str, str] = {}
1613
+ for block in blocks:
1614
+ if not isinstance(block, dict) or not isinstance(block.get("id"), str) or not isinstance(block.get("translated_text"), str):
1615
+ raise ValueError("translation blocks must contain id and translated_text")
1616
+ block_id = block["id"]
1617
+ if not block["translated_text"].strip():
1618
+ raise ValueError(f"translation is empty for block id: {block_id}")
1619
+ if block_id in translated:
1620
+ raise ValueError(f"translation contains duplicate block id: {block_id}")
1621
+ translated[block_id] = block["translated_text"]
1622
+ if len(expected_ids) != total or set(translated) != set(expected_ids):
1623
+ missing = len(set(expected_ids) - set(translated))
1624
+ extra = len(set(translated) - set(expected_ids))
1625
+ raise ValueError(f"translation block ids do not match layout (missing={missing}, extra={extra})")
1626
+ source_sha256 = str(manifest.get("source", {}).get("sha256", ""))
1627
+ if staged.get("source_sha256") != source_sha256:
1628
+ raise ValueError("translation was generated for a different source document")
1629
+ layout_fingerprint = (manifest.get("layout") or {}).get("fingerprint", "")
1630
+ if staged.get("layout_fingerprint") != layout_fingerprint:
1631
+ raise ValueError("translation was generated for a different document layout")
1632
+ canonical = {
1633
+ "engine": "legacy",
1634
+ "locale": locale,
1635
+ "source_sha256": source_sha256,
1636
+ "layout_fingerprint": layout_fingerprint,
1637
+ "blocks": [{"id": block_id, "translated_text": translated[block_id]} for block_id in expected_ids],
1638
+ }
1639
+ translations_dir = internal / "translations"
1640
+ if translations_dir.is_symlink():
1641
+ raise ValueError("managed translations directory is invalid")
1642
+ output = translations_dir / f"{locale}.json"
1643
+ if output.is_symlink():
1644
+ raise ValueError("translation output path is invalid")
1645
+ translation_output = output
1646
+ if output.exists():
1647
+ translation_backup = output.read_bytes()
1648
+ translation_existed = True
1649
+ _json_write(output, canonical)
1650
+ translations = manifest.get("translations")
1651
+ if not isinstance(translations, dict):
1652
+ translations = {}
1653
+ manifest["translations"] = translations
1654
+ translations[locale] = {
1655
+ "engine": "legacy",
1656
+ "path": output.relative_to(workspace).as_posix(),
1657
+ "source_sha256": source_sha256,
1658
+ "blocks": total,
1659
+ "updated_at": now,
1660
+ }
1661
+ current = total
1662
+ elif action == "generate_blog":
1663
+ staged_blog = job / "blog.md"
1664
+ try:
1665
+ if staged_blog.is_symlink() or staged_blog.stat().st_size > 10_000_000:
1666
+ raise ValueError("generated Blog is invalid or too large")
1667
+ markdown = staged_blog.read_text(encoding="utf-8")
1668
+ except OSError as exc:
1669
+ raise ValueError("Agent did not produce blog.md") from exc
1670
+ if not markdown.strip():
1671
+ raise ValueError("generated Blog is empty")
1672
+ output = workspace / "blog.md"
1673
+ if output.is_symlink():
1674
+ raise ValueError("Blog path is invalid")
1675
+ if output.exists():
1676
+ blog_backup = output.read_bytes()
1677
+ blog_existed = True
1678
+ replace_with_retry(staged_blog, output)
1679
+ revision = _blog_revision(markdown)
1680
+ blog_language = locale if language == "source" and source == "translation" else language
1681
+ manifest["blog"] = {
1682
+ "path": "blog.md",
1683
+ "revision": revision,
1684
+ "language": blog_language or (locale if source == "translation" else "source"),
1685
+ "source": source,
1686
+ "operation_id": operation_id,
1687
+ "updated_at": now,
1688
+ }
1689
+ current = total = 1
1690
+ else:
1691
+ raise ValueError("unknown document action")
1692
+ manifest["updated_at"] = now
1693
+ try:
1694
+ _json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
1695
+ except Exception:
1696
+ if action == "generate_blog":
1697
+ if blog_existed and blog_backup is not None:
1698
+ _atomic_write_bytes(workspace / "blog.md", blog_backup)
1699
+ else:
1700
+ (workspace / "blog.md").unlink(missing_ok=True)
1701
+ if action == "translate" and translation_output is not None:
1702
+ if translation_existed and translation_backup is not None:
1703
+ _atomic_write_bytes(translation_output, translation_backup)
1704
+ else:
1705
+ translation_output.unlink(missing_ok=True)
1706
+ raise
1707
+ _cleanup_document_job(workspace, operation_id)
1708
+ return current, total
1709
+
1710
+
1711
+ def _finalize_document_job(
1712
+ workspace: Path,
1713
+ operation_id: str,
1714
+ action: str,
1715
+ locale: str,
1716
+ source: str,
1717
+ expected_document_hash: str = "",
1718
+ language: str = "",
1719
+ engine: str = "legacy",
1720
+ ) -> tuple[int, int]:
1721
+ with _manifest_lock(workspace):
1722
+ return _finalize_document_job_unlocked(
1723
+ workspace,
1724
+ operation_id,
1725
+ action,
1726
+ locale,
1727
+ source,
1728
+ expected_document_hash,
1729
+ language,
1730
+ engine,
1731
+ )
1732
+
1733
+
1734
+ @router.get("/blog")
1735
+ async def read_document_blog(workspace: str, request: Request, response: Response) -> dict[str, Any]:
1736
+ root = _validated_workspace(workspace, _workspace_roots(request))
1737
+ with _manifest_lock(root):
1738
+ path = root / "blog.md"
1739
+ if path.is_symlink():
1740
+ raise HTTPException(status_code=403, detail="Blog path is invalid")
1741
+ if path.exists() and not path.is_file():
1742
+ raise HTTPException(status_code=403, detail="Blog path is invalid")
1743
+ try:
1744
+ markdown = path.read_text(encoding="utf-8")
1745
+ except FileNotFoundError as exc:
1746
+ raise HTTPException(status_code=404, detail="Blog not found") from exc
1747
+ revision = _blog_revision(markdown)
1748
+ response.headers["ETag"] = f'"{revision}"'
1749
+ manifest = _read_manifest(root)
1750
+ return {"markdown": markdown, "revision": revision, "language": (manifest.get("blog") or {}).get("language", "")}
1751
+
1752
+
1753
+ @router.put("/blog")
1754
+ async def write_document_blog(req: DocumentBlogWriteRequest, workspace: str, request: Request) -> dict[str, Any]:
1755
+ root = _validated_workspace(workspace, _workspace_roots(request))
1756
+ with _manifest_lock(root):
1757
+ path = root / "blog.md"
1758
+ if path.is_symlink():
1759
+ raise HTTPException(status_code=403, detail="Blog path is invalid")
1760
+ current = path.read_text(encoding="utf-8") if path.exists() else ""
1761
+ current_revision = _blog_revision(current) if path.exists() else None
1762
+ if req.revision != current_revision:
1763
+ raise HTTPException(status_code=409, detail="Blog changed since it was opened")
1764
+ temporary = root / f".blog-{uuid.uuid4().hex}.md"
1765
+ try:
1766
+ temporary.write_text(req.markdown, encoding="utf-8")
1767
+ replace_with_retry(temporary, path)
1768
+ finally:
1769
+ temporary.unlink(missing_ok=True)
1770
+ revision = _blog_revision(req.markdown)
1771
+ manifest = _read_manifest(root)
1772
+ manifest["blog"] = {"path": "blog.md", "revision": revision, "language": req.language}
1773
+ manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
1774
+ _json_write(root / MANIFEST_RELATIVE_PATH, manifest)
1775
+ return {"markdown": req.markdown, "revision": revision, "language": req.language}
1776
+
1777
+
1778
+ @router.delete("/blog")
1779
+ async def delete_document_blog(
1780
+ workspace: str,
1781
+ request: Request,
1782
+ ) -> dict[str, Any]:
1783
+ root = _validated_workspace(workspace, _workspace_roots(request))
1784
+ with _manifest_lock(root):
1785
+ path = root / "blog.md"
1786
+ if path.is_symlink():
1787
+ raise HTTPException(status_code=403, detail="Blog path is invalid")
1788
+ if path.exists() and not path.is_file():
1789
+ raise HTTPException(status_code=403, detail="Blog path is invalid")
1790
+ manifest = _read_manifest(root)
1791
+ entry = manifest.get("blog")
1792
+ blog_entry = entry if isinstance(entry, dict) else {}
1793
+ assets = root / "blog-assets"
1794
+ if assets.is_symlink() or (assets.exists() and not assets.is_dir()):
1795
+ raise HTTPException(status_code=403, detail="Blog assets path is invalid")
1796
+ removed = path.exists()
1797
+ if removed:
1798
+ path.unlink()
1799
+ removed_assets = assets.is_dir()
1800
+ if removed_assets:
1801
+ shutil.rmtree(assets)
1802
+ if entry is not None or removed or removed_assets:
1803
+ manifest["blog"] = None
1804
+ manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
1805
+ _json_write(root / MANIFEST_RELATIVE_PATH, manifest)
1806
+ return {
1807
+ "removed": removed or removed_assets,
1808
+ "removed_assets": removed_assets,
1809
+ "language": blog_entry.get("language", ""),
1810
+ }