crabcode 0.1.6__py3-none-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- crabcode-0.1.6.dist-info/METADATA +63 -0
- crabcode-0.1.6.dist-info/RECORD +205 -0
- crabcode-0.1.6.dist-info/WHEEL +4 -0
- crabcode-0.1.6.dist-info/entry_points.txt +2 -0
- crabcode_cli/__init__.py +7 -0
- crabcode_cli/__main__.py +4 -0
- crabcode_cli/_native/crabcode-computer-use-host.exe +0 -0
- crabcode_cli/_native/runtime.json +6 -0
- crabcode_cli/app.py +776 -0
- crabcode_cli/banner.py +39 -0
- crabcode_cli/computer_use.py +286 -0
- crabcode_cli/model_manager.py +649 -0
- crabcode_cli/model_picker.py +168 -0
- crabcode_cli/pipe.py +148 -0
- crabcode_cli/render/__init__.py +0 -0
- crabcode_cli/render/messages.py +19 -0
- crabcode_cli/repl.py +4841 -0
- crabcode_cli/session_picker.py +239 -0
- crabcode_core/__init__.py +7 -0
- crabcode_core/_version.py +3 -0
- crabcode_core/_windows_process.py +68 -0
- crabcode_core/agent_manager.py +1731 -0
- crabcode_core/api/__init__.py +20 -0
- crabcode_core/api/anthropic_adapter.py +648 -0
- crabcode_core/api/azure_adapter.py +69 -0
- crabcode_core/api/base.py +233 -0
- crabcode_core/api/codex_adapter.py +1327 -0
- crabcode_core/api/error_logging.py +132 -0
- crabcode_core/api/gemini_adapter.py +26 -0
- crabcode_core/api/model_info.py +257 -0
- crabcode_core/api/network.py +62 -0
- crabcode_core/api/ollama_adapter.py +27 -0
- crabcode_core/api/openai_adapter.py +323 -0
- crabcode_core/api/registry.py +58 -0
- crabcode_core/command_line.py +18 -0
- crabcode_core/compact/__init__.py +17 -0
- crabcode_core/compact/compact.py +514 -0
- crabcode_core/compact/context_tokens.py +165 -0
- crabcode_core/computer_use_mode.py +38 -0
- crabcode_core/computer_use_observation.py +144 -0
- crabcode_core/config/__init__.py +3 -0
- crabcode_core/config/claudemd.py +61 -0
- crabcode_core/config/manager.py +237 -0
- crabcode_core/config/model_catalog.py +256 -0
- crabcode_core/events.py +4896 -0
- crabcode_core/file_lock.py +81 -0
- crabcode_core/filesystem.py +22 -0
- crabcode_core/goal.py +149 -0
- crabcode_core/hooks/__init__.py +5 -0
- crabcode_core/hooks/manager.py +290 -0
- crabcode_core/image_io.py +111 -0
- crabcode_core/io_worker.py +211 -0
- crabcode_core/logging_utils.py +168 -0
- crabcode_core/lsp/__init__.py +6 -0
- crabcode_core/lsp/client.py +786 -0
- crabcode_core/lsp/diagnostics.py +150 -0
- crabcode_core/lsp/manager.py +583 -0
- crabcode_core/mcp/__init__.py +4 -0
- crabcode_core/mcp/client.py +169 -0
- crabcode_core/mcp/config.py +57 -0
- crabcode_core/path_validation.py +34 -0
- crabcode_core/paths.py +24 -0
- crabcode_core/peer/__init__.py +10 -0
- crabcode_core/peer/runtime.py +518 -0
- crabcode_core/permissions/__init__.py +7 -0
- crabcode_core/permissions/ai_reviewer.py +184 -0
- crabcode_core/permissions/manager.py +187 -0
- crabcode_core/plan/__init__.py +5 -0
- crabcode_core/plan/executor.py +270 -0
- crabcode_core/plan/types.py +169 -0
- crabcode_core/prompts/__init__.py +0 -0
- crabcode_core/prompts/blocks.py +10 -0
- crabcode_core/prompts/context.py +229 -0
- crabcode_core/prompts/library.py +92 -0
- crabcode_core/prompts/profile.py +79 -0
- crabcode_core/prompts/system.py +356 -0
- crabcode_core/prompts/templates.py +104 -0
- crabcode_core/query/__init__.py +0 -0
- crabcode_core/query/computer_use_history.py +83 -0
- crabcode_core/query/loop.py +2446 -0
- crabcode_core/query/retry.py +116 -0
- crabcode_core/schedule/__init__.py +13 -0
- crabcode_core/schedule/manager.py +631 -0
- crabcode_core/schedule/models.py +204 -0
- crabcode_core/schedule/store.py +558 -0
- crabcode_core/schedule/timing.py +287 -0
- crabcode_core/session/__init__.py +12 -0
- crabcode_core/session/export.py +141 -0
- crabcode_core/session/meta_db.py +684 -0
- crabcode_core/session/storage.py +2032 -0
- crabcode_core/session/title_gen.py +52 -0
- crabcode_core/skills/__init__.py +6 -0
- crabcode_core/skills/context.py +23 -0
- crabcode_core/skills/loader.py +214 -0
- crabcode_core/skills/matcher.py +203 -0
- crabcode_core/snapshot/__init__.py +21 -0
- crabcode_core/snapshot/snapshot.py +436 -0
- crabcode_core/snapshot/tracker.py +359 -0
- crabcode_core/subprocess_utils.py +260 -0
- crabcode_core/team/__init__.py +27 -0
- crabcode_core/team/bridge.py +66 -0
- crabcode_core/team/inbox.py +237 -0
- crabcode_core/team/manager.py +1660 -0
- crabcode_core/team/message_bus.py +821 -0
- crabcode_core/team/models.py +191 -0
- crabcode_core/team/recovery.py +192 -0
- crabcode_core/text_io.py +52 -0
- crabcode_core/tools/__init__.py +112 -0
- crabcode_core/tools/_input_helpers.py +39 -0
- crabcode_core/tools/_input_schema.py +130 -0
- crabcode_core/tools/agent.py +413 -0
- crabcode_core/tools/apply_patch.py +587 -0
- crabcode_core/tools/ask_user.py +122 -0
- crabcode_core/tools/bash.py +154 -0
- crabcode_core/tools/browser.py +678 -0
- crabcode_core/tools/checklist.py +241 -0
- crabcode_core/tools/checkpoint.py +78 -0
- crabcode_core/tools/computer_use.py +599 -0
- crabcode_core/tools/diff_utils.py +123 -0
- crabcode_core/tools/file_edit.py +240 -0
- crabcode_core/tools/file_read.py +187 -0
- crabcode_core/tools/file_write.py +154 -0
- crabcode_core/tools/glob.py +123 -0
- crabcode_core/tools/goal.py +134 -0
- crabcode_core/tools/grep.py +247 -0
- crabcode_core/tools/image.py +173 -0
- crabcode_core/tools/image_generate.py +115 -0
- crabcode_core/tools/lint.py +388 -0
- crabcode_core/tools/loading.py +187 -0
- crabcode_core/tools/memory.py +456 -0
- crabcode_core/tools/monitor.py +677 -0
- crabcode_core/tools/peer.py +103 -0
- crabcode_core/tools/revert.py +90 -0
- crabcode_core/tools/schedule.py +683 -0
- crabcode_core/tools/skill.py +108 -0
- crabcode_core/tools/switch_mode.py +213 -0
- crabcode_core/tools/team.py +329 -0
- crabcode_core/tools/tool_search.py +64 -0
- crabcode_core/tools/web_search.py +444 -0
- crabcode_core/turn_details.py +64 -0
- crabcode_core/types/__init__.py +95 -0
- crabcode_core/types/config.py +434 -0
- crabcode_core/types/event.py +334 -0
- crabcode_core/types/message.py +263 -0
- crabcode_core/types/tool.py +208 -0
- crabcode_core/usage.py +289 -0
- crabcode_core/utf8_sanitize.py +29 -0
- crabcode_debugger/__init__.py +5 -0
- crabcode_debugger/adapters.py +332 -0
- crabcode_debugger/dap.py +304 -0
- crabcode_debugger/memory.py +1806 -0
- crabcode_debugger/process.py +707 -0
- crabcode_debugger/runtime.py +58 -0
- crabcode_debugger/sessions.py +387 -0
- crabcode_debugger/tool.py +731 -0
- crabcode_gateway/__init__.py +18 -0
- crabcode_gateway/acp/__init__.py +13 -0
- crabcode_gateway/acp/agent.py +1581 -0
- crabcode_gateway/acp/session.py +114 -0
- crabcode_gateway/acp/transport.py +39 -0
- crabcode_gateway/acp/types.py +90 -0
- crabcode_gateway/adapter.py +30 -0
- crabcode_gateway/auth.py +139 -0
- crabcode_gateway/computer_use.py +364 -0
- crabcode_gateway/document_engine.py +637 -0
- crabcode_gateway/document_worker.py +301 -0
- crabcode_gateway/event_bus.py +655 -0
- crabcode_gateway/grpc/__init__.py +0 -0
- crabcode_gateway/grpc/crabcode_pb2.py +70 -0
- crabcode_gateway/grpc/crabcode_pb2_grpc.py +474 -0
- crabcode_gateway/grpc/proto/crabcode.proto +160 -0
- crabcode_gateway/grpc/server.py +765 -0
- crabcode_gateway/middleware.py +344 -0
- crabcode_gateway/protocol.py +17 -0
- crabcode_gateway/routes/__init__.py +0 -0
- crabcode_gateway/routes/agent.py +392 -0
- crabcode_gateway/routes/auth.py +150 -0
- crabcode_gateway/routes/computer_use.py +101 -0
- crabcode_gateway/routes/config.py +1698 -0
- crabcode_gateway/routes/document.py +1810 -0
- crabcode_gateway/routes/event.py +3801 -0
- crabcode_gateway/routes/git_review.py +232 -0
- crabcode_gateway/routes/health.py +14 -0
- crabcode_gateway/routes/peer.py +78 -0
- crabcode_gateway/routes/permission.py +121 -0
- crabcode_gateway/routes/schedule.py +300 -0
- crabcode_gateway/routes/session.py +1496 -0
- crabcode_gateway/routes/snapshot.py +211 -0
- crabcode_gateway/routes/tasks.py +318 -0
- crabcode_gateway/routes/team.py +392 -0
- crabcode_gateway/routes/usage.py +49 -0
- crabcode_gateway/routes/workspace.py +299 -0
- crabcode_gateway/safe_file.py +63 -0
- crabcode_gateway/schemas.py +1820 -0
- crabcode_gateway/server.py +514 -0
- crabcode_gateway/session_registry.py +36 -0
- crabcode_gateway/task_registry.py +708 -0
- crabcode_gateway/turn_details.py +60 -0
- crabcode_search/__init__.py +11 -0
- crabcode_search/background.py +368 -0
- crabcode_search/chunker.py +517 -0
- crabcode_search/embedder.py +497 -0
- crabcode_search/indexer.py +215 -0
- crabcode_search/store.py +222 -0
- crabcode_search/tool.py +268 -0
|
@@ -0,0 +1,1810 @@
|
|
|
1
|
+
"""Managed document-project import and artifact APIs."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import html
|
|
8
|
+
import ipaddress
|
|
9
|
+
import json
|
|
10
|
+
import mimetypes
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
import socket
|
|
15
|
+
import tempfile
|
|
16
|
+
import threading
|
|
17
|
+
import uuid
|
|
18
|
+
import zipfile
|
|
19
|
+
from contextlib import contextmanager
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any, Literal
|
|
23
|
+
from urllib.parse import unquote, urljoin, urlsplit, urlunsplit
|
|
24
|
+
|
|
25
|
+
import httpx
|
|
26
|
+
from fastapi import APIRouter, File, Form, HTTPException, Request, Response, UploadFile
|
|
27
|
+
from fastapi.responses import FileResponse
|
|
28
|
+
from pydantic import BaseModel, Field, HttpUrl
|
|
29
|
+
from pypdf import PdfReader
|
|
30
|
+
|
|
31
|
+
from crabcode_core.path_validation import validate_path_component
|
|
32
|
+
from crabcode_core.filesystem import replace_with_retry
|
|
33
|
+
from crabcode_core.subprocess_utils import (
|
|
34
|
+
decode_subprocess_output,
|
|
35
|
+
managed_process_command,
|
|
36
|
+
subprocess_group_options,
|
|
37
|
+
terminate_process_tree,
|
|
38
|
+
)
|
|
39
|
+
from crabcode_gateway.schemas import WorkspaceInfo
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
router = APIRouter(prefix="/document", tags=["document"])
|
|
43
|
+
|
|
44
|
+
MAX_DOCUMENT_BYTES = 100 * 1024 * 1024
|
|
45
|
+
MAX_LAYOUT_BYTES = 50 * 1024 * 1024
|
|
46
|
+
SUPPORTED_EXTENSIONS = {
|
|
47
|
+
".pdf", ".doc", ".docx", ".odt", ".rtf", ".ppt", ".pptx",
|
|
48
|
+
".txt", ".md", ".html", ".htm",
|
|
49
|
+
}
|
|
50
|
+
OFFICE_EXTENSIONS = SUPPORTED_EXTENSIONS - {".pdf"}
|
|
51
|
+
MANIFEST_RELATIVE_PATH = Path(".crabcode/document/manifest.json")
|
|
52
|
+
_MANIFEST_LOCKS_GUARD = threading.Lock()
|
|
53
|
+
_MANIFEST_LOCKS: dict[str, threading.RLock] = {}
|
|
54
|
+
|
|
55
|
+
MIME_EXTENSIONS = {
|
|
56
|
+
"application/pdf": ".pdf",
|
|
57
|
+
"application/msword": ".doc",
|
|
58
|
+
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",
|
|
59
|
+
"application/vnd.oasis.opendocument.text": ".odt",
|
|
60
|
+
"application/rtf": ".rtf",
|
|
61
|
+
"text/rtf": ".rtf",
|
|
62
|
+
"application/vnd.ms-powerpoint": ".ppt",
|
|
63
|
+
"application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",
|
|
64
|
+
"text/plain": ".txt",
|
|
65
|
+
"text/markdown": ".md",
|
|
66
|
+
"text/html": ".html",
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class DocumentUrlImportRequest(BaseModel):
|
|
71
|
+
workspace_path: str = Field(min_length=1)
|
|
72
|
+
project_id: str = Field(min_length=1, max_length=200)
|
|
73
|
+
project_name: str = Field(min_length=1, max_length=200)
|
|
74
|
+
url: HttpUrl
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class DocumentLayoutRequest(BaseModel):
|
|
78
|
+
fingerprint: str = Field(min_length=1, max_length=500)
|
|
79
|
+
page_count: int = Field(ge=1, le=100_000)
|
|
80
|
+
pages: list[dict[str, Any]]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class DocumentBlogWriteRequest(BaseModel):
|
|
84
|
+
revision: str | None = None
|
|
85
|
+
markdown: str = Field(max_length=10_000_000)
|
|
86
|
+
language: str = Field(default="", max_length=50)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class DocumentAnnotationRect(BaseModel):
|
|
90
|
+
page: int = Field(ge=1, le=100_000)
|
|
91
|
+
x: float = Field(ge=0, le=1)
|
|
92
|
+
y: float = Field(ge=0, le=1)
|
|
93
|
+
width: float = Field(gt=0, le=1)
|
|
94
|
+
height: float = Field(gt=0, le=1)
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class DocumentAnnotationWriteRequest(BaseModel):
|
|
98
|
+
id: str = Field(pattern=r"^[A-Za-z0-9_-]{1,100}$")
|
|
99
|
+
label: str = Field(default="", max_length=100)
|
|
100
|
+
note: str = Field(default="", max_length=20_000)
|
|
101
|
+
text: str = Field(min_length=1, max_length=50_000)
|
|
102
|
+
rects: list[DocumentAnnotationRect] = Field(min_length=1, max_length=1_000)
|
|
103
|
+
created_at: str = Field(default="", max_length=100)
|
|
104
|
+
updated_at: str = Field(default="", max_length=100)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@contextmanager
|
|
108
|
+
def _manifest_lock(workspace: Path):
|
|
109
|
+
key = os.path.normcase(str(workspace.resolve(strict=False)))
|
|
110
|
+
with _MANIFEST_LOCKS_GUARD:
|
|
111
|
+
lock = _MANIFEST_LOCKS.setdefault(key, threading.RLock())
|
|
112
|
+
with lock:
|
|
113
|
+
yield
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _workspace_roots(request: Request) -> tuple[Path, ...]:
|
|
117
|
+
info: WorkspaceInfo = request.app.state.workspace_info
|
|
118
|
+
return tuple(Path(value).resolve() for value in info.browse_roots)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _is_within(path: Path, roots: tuple[Path, ...]) -> bool:
|
|
122
|
+
normalized = os.path.normcase(str(path))
|
|
123
|
+
for root in roots:
|
|
124
|
+
try:
|
|
125
|
+
if os.path.commonpath((normalized, os.path.normcase(str(root)))) == os.path.normcase(str(root)):
|
|
126
|
+
return True
|
|
127
|
+
except ValueError:
|
|
128
|
+
continue
|
|
129
|
+
return False
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _validated_new_workspace(value: str, roots: tuple[Path, ...]) -> Path:
|
|
133
|
+
candidate = Path(value).expanduser()
|
|
134
|
+
if not candidate.is_absolute() or not candidate.name:
|
|
135
|
+
raise HTTPException(status_code=400, detail="workspace_path must name an absolute directory")
|
|
136
|
+
candidate = candidate.resolve(strict=False)
|
|
137
|
+
if not _is_within(candidate, roots):
|
|
138
|
+
raise HTTPException(status_code=403, detail="Document workspace is outside the allowed browse roots")
|
|
139
|
+
if candidate.exists():
|
|
140
|
+
base = candidate
|
|
141
|
+
for index in range(2, 10_000):
|
|
142
|
+
candidate = base.with_name(f"{base.name}-{index}")
|
|
143
|
+
if not candidate.exists():
|
|
144
|
+
break
|
|
145
|
+
else:
|
|
146
|
+
raise HTTPException(status_code=409, detail="Unable to find an unused document workspace name")
|
|
147
|
+
ancestor = candidate.parent
|
|
148
|
+
while not ancestor.exists() and ancestor != ancestor.parent:
|
|
149
|
+
ancestor = ancestor.parent
|
|
150
|
+
try:
|
|
151
|
+
resolved_ancestor = ancestor.resolve(strict=True)
|
|
152
|
+
except OSError as exc:
|
|
153
|
+
raise HTTPException(status_code=400, detail="Unable to resolve document workspace parent") from exc
|
|
154
|
+
if not resolved_ancestor.is_dir() or not _is_within(resolved_ancestor, roots):
|
|
155
|
+
raise HTTPException(status_code=403, detail="Document workspace parent is not allowed")
|
|
156
|
+
return candidate
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _validated_workspace(value: str, roots: tuple[Path, ...]) -> Path:
|
|
160
|
+
candidate = Path(value).expanduser()
|
|
161
|
+
if not candidate.is_absolute():
|
|
162
|
+
raise HTTPException(status_code=400, detail="workspace must be absolute")
|
|
163
|
+
try:
|
|
164
|
+
resolved = candidate.resolve(strict=True)
|
|
165
|
+
except FileNotFoundError as exc:
|
|
166
|
+
raise HTTPException(status_code=404, detail="Document workspace not found") from exc
|
|
167
|
+
if not resolved.is_dir() or not _is_within(resolved, roots):
|
|
168
|
+
raise HTTPException(status_code=403, detail="Document workspace is not allowed")
|
|
169
|
+
manifest = resolved / MANIFEST_RELATIVE_PATH
|
|
170
|
+
if not manifest.is_file() or manifest.is_symlink():
|
|
171
|
+
raise HTTPException(status_code=404, detail="Document manifest not found")
|
|
172
|
+
return resolved
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _safe_name(value: str, fallback: str = "document") -> str:
|
|
176
|
+
name = Path(value.replace("\\", "/")).name.strip()
|
|
177
|
+
name = re.sub(r"[<>:\"/\\|?*\x00-\x1f]", "-", name).rstrip(". ")
|
|
178
|
+
name = name[:180] or fallback
|
|
179
|
+
try:
|
|
180
|
+
validate_path_component(name, "document filename")
|
|
181
|
+
except ValueError:
|
|
182
|
+
name = f"_{name}"[:180]
|
|
183
|
+
return name
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _file_extension(filename: str) -> str:
|
|
187
|
+
extension = Path(filename).suffix.lower()
|
|
188
|
+
if extension not in SUPPORTED_EXTENSIONS:
|
|
189
|
+
raise HTTPException(status_code=415, detail=f"Unsupported document format: {extension or 'unknown'}")
|
|
190
|
+
return extension
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def _converter() -> str | None:
|
|
194
|
+
override = os.environ.get("CRABCODE_LIBREOFFICE_PATH", "").strip()
|
|
195
|
+
if override:
|
|
196
|
+
candidate = Path(override).expanduser()
|
|
197
|
+
return str(candidate.resolve()) if candidate.is_file() else None
|
|
198
|
+
executable = shutil.which("soffice") or shutil.which("libreoffice")
|
|
199
|
+
if executable or os.name != "nt":
|
|
200
|
+
return executable
|
|
201
|
+
for variable in ("ProgramFiles", "ProgramFiles(x86)", "LOCALAPPDATA"):
|
|
202
|
+
root = os.environ.get(variable)
|
|
203
|
+
if root:
|
|
204
|
+
candidate = Path(root) / "LibreOffice" / "program" / "soffice.exe"
|
|
205
|
+
if candidate.is_file():
|
|
206
|
+
return str(candidate)
|
|
207
|
+
return None
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _sha256(path: Path) -> str:
|
|
211
|
+
digest = hashlib.sha256()
|
|
212
|
+
with path.open("rb") as handle:
|
|
213
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
214
|
+
digest.update(chunk)
|
|
215
|
+
return digest.hexdigest()
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _atomic_write_bytes(path: Path, content: bytes) -> None:
|
|
219
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
220
|
+
with tempfile.NamedTemporaryFile(dir=path.parent, prefix=f".{path.name}.", delete=False) as handle:
|
|
221
|
+
temporary = Path(handle.name)
|
|
222
|
+
handle.write(content)
|
|
223
|
+
handle.flush()
|
|
224
|
+
os.fsync(handle.fileno())
|
|
225
|
+
try:
|
|
226
|
+
replace_with_retry(temporary, path)
|
|
227
|
+
finally:
|
|
228
|
+
temporary.unlink(missing_ok=True)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _json_write(path: Path, value: Any) -> None:
|
|
232
|
+
_atomic_write_bytes(path, json.dumps(value, ensure_ascii=False, indent=2).encode("utf-8") + b"\n")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _text_write(path: Path, value: str) -> None:
|
|
236
|
+
_atomic_write_bytes(path, value.encode("utf-8"))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _read_manifest(workspace: Path) -> dict[str, Any]:
|
|
240
|
+
try:
|
|
241
|
+
value = json.loads((workspace / MANIFEST_RELATIVE_PATH).read_text(encoding="utf-8"))
|
|
242
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
243
|
+
raise HTTPException(status_code=500, detail="Document manifest is invalid") from exc
|
|
244
|
+
if not isinstance(value, dict):
|
|
245
|
+
raise HTTPException(status_code=500, detail="Document manifest is invalid")
|
|
246
|
+
return value
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _validate_signature(path: Path, extension: str) -> None:
|
|
250
|
+
with path.open("rb") as handle:
|
|
251
|
+
prefix = handle.read(8192)
|
|
252
|
+
if extension == ".pdf" and b"%PDF-" not in prefix[:1024]:
|
|
253
|
+
raise HTTPException(status_code=415, detail="The uploaded file is not a valid PDF")
|
|
254
|
+
if extension in {".docx", ".odt", ".pptx"} and not prefix.startswith(b"PK"):
|
|
255
|
+
raise HTTPException(status_code=415, detail="The uploaded file does not match its format")
|
|
256
|
+
if extension in {".docx", ".odt", ".pptx"}:
|
|
257
|
+
try:
|
|
258
|
+
with zipfile.ZipFile(path) as archive:
|
|
259
|
+
names = set(archive.namelist())
|
|
260
|
+
valid = (
|
|
261
|
+
extension == ".docx" and "word/document.xml" in names
|
|
262
|
+
or extension == ".pptx" and "ppt/presentation.xml" in names
|
|
263
|
+
or extension == ".odt" and "content.xml" in names and "META-INF/manifest.xml" in names
|
|
264
|
+
)
|
|
265
|
+
except (OSError, zipfile.BadZipFile):
|
|
266
|
+
valid = False
|
|
267
|
+
if not valid:
|
|
268
|
+
raise HTTPException(status_code=415, detail="The uploaded archive does not match its document format")
|
|
269
|
+
if extension in {".doc", ".ppt"} and not prefix.startswith(bytes.fromhex("d0cf11e0a1b11ae1")):
|
|
270
|
+
raise HTTPException(status_code=415, detail="The uploaded file does not match its format")
|
|
271
|
+
if extension == ".rtf" and not prefix.lstrip().startswith(b"{\\rtf"):
|
|
272
|
+
raise HTTPException(status_code=415, detail="The uploaded file is not valid RTF")
|
|
273
|
+
if extension in {".txt", ".md", ".html", ".htm"} and b"\x00" in prefix:
|
|
274
|
+
raise HTTPException(status_code=415, detail="The uploaded text document contains binary data")
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _validate_mime(extension: str, content_type: str) -> None:
|
|
278
|
+
media_type = content_type.split(";", 1)[0].strip().lower()
|
|
279
|
+
if not media_type or media_type == "application/octet-stream":
|
|
280
|
+
return
|
|
281
|
+
expected = MIME_EXTENSIONS.get(media_type)
|
|
282
|
+
compatible_text = extension in {".txt", ".md", ".html", ".htm"} and media_type.startswith("text/")
|
|
283
|
+
compatible_zip = extension in {".docx", ".odt", ".pptx"} and media_type in {"application/zip", "application/x-zip-compressed"}
|
|
284
|
+
if expected not in {extension, ".html" if extension == ".htm" else extension} and not compatible_text and not compatible_zip:
|
|
285
|
+
raise HTTPException(status_code=415, detail="Document MIME type does not match its filename")
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _pdf_page_count(path: Path) -> int:
|
|
289
|
+
try:
|
|
290
|
+
count = len(PdfReader(path, strict=False).pages)
|
|
291
|
+
except Exception as exc:
|
|
292
|
+
raise HTTPException(status_code=422, detail="The converted PDF is invalid or encrypted") from exc
|
|
293
|
+
if count < 1:
|
|
294
|
+
raise HTTPException(status_code=422, detail="The converted PDF has no pages")
|
|
295
|
+
return count
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _plain_html(text: str) -> str:
|
|
299
|
+
return (
|
|
300
|
+
"<!doctype html><meta charset=\"utf-8\"><style>"
|
|
301
|
+
"body{font:12pt/1.55 sans-serif;max-width:48rem;margin:2.5rem auto;}"
|
|
302
|
+
"pre{white-space:pre-wrap;font:inherit}</style><pre>"
|
|
303
|
+
f"{html.escape(text)}"
|
|
304
|
+
"</pre>"
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def _prepare_conversion_input(source: Path, extension: str, internal: Path) -> Path:
|
|
309
|
+
if extension not in {".txt", ".md", ".html", ".htm"}:
|
|
310
|
+
return source
|
|
311
|
+
text = source.read_text(encoding="utf-8", errors="replace")
|
|
312
|
+
# Treat imported HTML as text. This deliberately prevents external resource
|
|
313
|
+
# loads and local-file references during headless conversion.
|
|
314
|
+
prepared = internal / "safe-source.html"
|
|
315
|
+
prepared.write_text(_plain_html(text), encoding="utf-8")
|
|
316
|
+
return prepared
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
async def _convert_to_pdf(source: Path, extension: str, internal: Path) -> Path:
|
|
320
|
+
rendered = internal / "rendered.pdf"
|
|
321
|
+
if extension == ".pdf":
|
|
322
|
+
shutil.copy2(source, rendered)
|
|
323
|
+
return rendered
|
|
324
|
+
converter = _converter()
|
|
325
|
+
if not converter:
|
|
326
|
+
raise HTTPException(status_code=503, detail="LibreOffice is required to convert this document format")
|
|
327
|
+
conversion_input = _prepare_conversion_input(source, extension, internal)
|
|
328
|
+
output_dir = internal / "conversion"
|
|
329
|
+
output_dir.mkdir()
|
|
330
|
+
profile_dir = internal / "libreoffice-profile"
|
|
331
|
+
profile_uri = profile_dir.resolve().as_uri()
|
|
332
|
+
process = await asyncio.create_subprocess_exec(
|
|
333
|
+
*managed_process_command([
|
|
334
|
+
converter,
|
|
335
|
+
"--headless",
|
|
336
|
+
f"-env:UserInstallation={profile_uri}",
|
|
337
|
+
"--convert-to",
|
|
338
|
+
"pdf",
|
|
339
|
+
"--outdir",
|
|
340
|
+
str(output_dir),
|
|
341
|
+
str(conversion_input),
|
|
342
|
+
]),
|
|
343
|
+
stdout=asyncio.subprocess.PIPE,
|
|
344
|
+
stderr=asyncio.subprocess.PIPE,
|
|
345
|
+
**subprocess_group_options(),
|
|
346
|
+
)
|
|
347
|
+
try:
|
|
348
|
+
stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=120)
|
|
349
|
+
except asyncio.TimeoutError as exc:
|
|
350
|
+
await terminate_process_tree(process)
|
|
351
|
+
raise HTTPException(status_code=504, detail="Document conversion timed out") from exc
|
|
352
|
+
except asyncio.CancelledError:
|
|
353
|
+
await terminate_process_tree(process)
|
|
354
|
+
raise
|
|
355
|
+
candidates = list(output_dir.glob("*.pdf"))
|
|
356
|
+
if process.returncode != 0 or len(candidates) != 1:
|
|
357
|
+
detail = decode_subprocess_output(stderr or stdout).strip()[-500:]
|
|
358
|
+
raise HTTPException(status_code=422, detail=f"Document conversion failed: {detail or 'no PDF produced'}")
|
|
359
|
+
shutil.move(str(candidates[0]), rendered)
|
|
360
|
+
return rendered
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
async def _write_upload(upload: UploadFile, destination: Path) -> int:
|
|
364
|
+
total = 0
|
|
365
|
+
with destination.open("wb") as handle:
|
|
366
|
+
while True:
|
|
367
|
+
chunk = await upload.read(1024 * 1024)
|
|
368
|
+
if not chunk:
|
|
369
|
+
break
|
|
370
|
+
total += len(chunk)
|
|
371
|
+
if total > MAX_DOCUMENT_BYTES:
|
|
372
|
+
raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
|
|
373
|
+
handle.write(chunk)
|
|
374
|
+
if total == 0:
|
|
375
|
+
raise HTTPException(status_code=400, detail="Document is empty")
|
|
376
|
+
return total
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _assert_public_host(url: str) -> None:
|
|
380
|
+
parsed = urlsplit(url)
|
|
381
|
+
if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username or parsed.password:
|
|
382
|
+
raise HTTPException(status_code=400, detail="URL must be an unauthenticated HTTP or HTTPS address")
|
|
383
|
+
try:
|
|
384
|
+
addresses = socket.getaddrinfo(parsed.hostname, parsed.port or (443 if parsed.scheme == "https" else 80))
|
|
385
|
+
except OSError as exc:
|
|
386
|
+
raise HTTPException(status_code=400, detail="Unable to resolve document URL") from exc
|
|
387
|
+
for address in addresses:
|
|
388
|
+
ip = ipaddress.ip_address(address[4][0])
|
|
389
|
+
if not ip.is_global:
|
|
390
|
+
raise HTTPException(status_code=400, detail="Document URL resolves to a private or local address")
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
async def _download(url: str, destination: Path) -> tuple[str, str, int]:
|
|
394
|
+
current = url
|
|
395
|
+
async with httpx.AsyncClient(
|
|
396
|
+
timeout=httpx.Timeout(120.0, connect=10.0),
|
|
397
|
+
follow_redirects=False,
|
|
398
|
+
trust_env=False,
|
|
399
|
+
) as client:
|
|
400
|
+
for redirect in range(6):
|
|
401
|
+
try:
|
|
402
|
+
await asyncio.wait_for(asyncio.to_thread(_assert_public_host, current), timeout=10)
|
|
403
|
+
except asyncio.TimeoutError as exc:
|
|
404
|
+
raise HTTPException(status_code=400, detail="Document URL DNS lookup timed out") from exc
|
|
405
|
+
async with client.stream("GET", current, headers={"Accept": "application/pdf,application/*;q=.9,text/*;q=.5"}) as response:
|
|
406
|
+
if response.status_code in {301, 302, 303, 307, 308}:
|
|
407
|
+
location = response.headers.get("location")
|
|
408
|
+
if not location or redirect == 5:
|
|
409
|
+
raise HTTPException(status_code=400, detail="Too many or invalid document redirects")
|
|
410
|
+
current = urljoin(current, location)
|
|
411
|
+
continue
|
|
412
|
+
if response.status_code < 200 or response.status_code >= 300:
|
|
413
|
+
raise HTTPException(status_code=400, detail=f"Document download failed ({response.status_code})")
|
|
414
|
+
advertised = response.headers.get("content-length")
|
|
415
|
+
if advertised and advertised.isdigit() and int(advertised) > MAX_DOCUMENT_BYTES:
|
|
416
|
+
raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
|
|
417
|
+
total = 0
|
|
418
|
+
with destination.open("wb") as handle:
|
|
419
|
+
async for chunk in response.aiter_bytes(1024 * 1024):
|
|
420
|
+
total += len(chunk)
|
|
421
|
+
if total > MAX_DOCUMENT_BYTES:
|
|
422
|
+
raise HTTPException(status_code=413, detail="Document exceeds the 100MiB limit")
|
|
423
|
+
handle.write(chunk)
|
|
424
|
+
if total == 0:
|
|
425
|
+
raise HTTPException(status_code=400, detail="Downloaded document is empty")
|
|
426
|
+
content_type = response.headers.get("content-type", "").split(";", 1)[0]
|
|
427
|
+
parsed = urlsplit(str(response.url))
|
|
428
|
+
disposition = response.headers.get("content-disposition", "")
|
|
429
|
+
encoded_name = re.search(r"filename\*=UTF-8''([^;]+)", disposition, re.IGNORECASE)
|
|
430
|
+
plain_name = re.search(r'filename="?([^";]+)', disposition, re.IGNORECASE)
|
|
431
|
+
candidate_name = unquote(encoded_name.group(1)) if encoded_name else plain_name.group(1) if plain_name else Path(parsed.path).name
|
|
432
|
+
filename = _safe_name(unquote(candidate_name), "document")
|
|
433
|
+
if Path(filename).suffix.lower() not in SUPPORTED_EXTENSIONS:
|
|
434
|
+
inferred = MIME_EXTENSIONS.get(content_type.lower())
|
|
435
|
+
if inferred:
|
|
436
|
+
filename = f"{filename}{inferred}"
|
|
437
|
+
return filename, content_type, total
|
|
438
|
+
raise HTTPException(status_code=400, detail="Unable to download document")
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _redacted_url(value: str) -> str:
|
|
442
|
+
parsed = urlsplit(value)
|
|
443
|
+
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", ""))
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
async def _finish_import(
|
|
447
|
+
*,
|
|
448
|
+
request: Request,
|
|
449
|
+
workspace_path: str,
|
|
450
|
+
project_id: str,
|
|
451
|
+
project_name: str,
|
|
452
|
+
filename: str,
|
|
453
|
+
content_type: str,
|
|
454
|
+
origin: Literal["upload", "url"],
|
|
455
|
+
source_url: str | None,
|
|
456
|
+
writer,
|
|
457
|
+
) -> dict[str, Any]:
|
|
458
|
+
if not project_name.strip() or len(project_name) > 200:
|
|
459
|
+
raise HTTPException(status_code=400, detail="project_name must be between 1 and 200 characters")
|
|
460
|
+
if not project_id.strip() or len(project_id) > 200:
|
|
461
|
+
raise HTTPException(status_code=400, detail="project_id must be between 1 and 200 characters")
|
|
462
|
+
roots = _workspace_roots(request)
|
|
463
|
+
workspace = _validated_new_workspace(workspace_path, roots)
|
|
464
|
+
workspace.parent.mkdir(parents=True, exist_ok=True)
|
|
465
|
+
stage = workspace.parent / f".{workspace.name}.import-{uuid.uuid4().hex}"
|
|
466
|
+
stage.mkdir()
|
|
467
|
+
try:
|
|
468
|
+
source_dir = stage / "source"
|
|
469
|
+
internal = stage / ".crabcode" / "document"
|
|
470
|
+
source_dir.mkdir(parents=True)
|
|
471
|
+
internal.mkdir(parents=True)
|
|
472
|
+
safe_filename = _safe_name(filename)
|
|
473
|
+
extension = _file_extension(safe_filename)
|
|
474
|
+
_validate_mime(extension, content_type)
|
|
475
|
+
source = source_dir / safe_filename
|
|
476
|
+
size = await writer(source)
|
|
477
|
+
_validate_signature(source, extension)
|
|
478
|
+
rendered = await _convert_to_pdf(source, extension, internal)
|
|
479
|
+
page_count = await asyncio.to_thread(_pdf_page_count, rendered)
|
|
480
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
481
|
+
manifest: dict[str, Any] = {
|
|
482
|
+
"schema_version": 1,
|
|
483
|
+
"project_id": project_id,
|
|
484
|
+
"project_name": project_name.strip(),
|
|
485
|
+
"workspace": str(workspace),
|
|
486
|
+
"source": {
|
|
487
|
+
"origin": origin,
|
|
488
|
+
"name": safe_filename,
|
|
489
|
+
"path": source.relative_to(stage).as_posix(),
|
|
490
|
+
"url": _redacted_url(source_url) if source_url else None,
|
|
491
|
+
"content_type": content_type or mimetypes.guess_type(safe_filename)[0] or "application/octet-stream",
|
|
492
|
+
"size": size,
|
|
493
|
+
"sha256": _sha256(source),
|
|
494
|
+
},
|
|
495
|
+
"pdf": {
|
|
496
|
+
"path": rendered.relative_to(stage).as_posix(),
|
|
497
|
+
"sha256": _sha256(rendered),
|
|
498
|
+
"page_count": page_count,
|
|
499
|
+
},
|
|
500
|
+
"layout": None,
|
|
501
|
+
"translations": {},
|
|
502
|
+
"blog": None,
|
|
503
|
+
"jobs": {},
|
|
504
|
+
"created_at": now,
|
|
505
|
+
"updated_at": now,
|
|
506
|
+
}
|
|
507
|
+
_json_write(stage / MANIFEST_RELATIVE_PATH, manifest)
|
|
508
|
+
replace_with_retry(stage, workspace)
|
|
509
|
+
return manifest
|
|
510
|
+
except Exception:
|
|
511
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
512
|
+
raise
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
@router.get("/capabilities")
|
|
516
|
+
async def document_capabilities(request: Request) -> dict[str, Any]:
|
|
517
|
+
from crabcode_gateway.document_engine import document_engine_status
|
|
518
|
+
|
|
519
|
+
converter = _converter()
|
|
520
|
+
precise = await asyncio.to_thread(document_engine_status)
|
|
521
|
+
info: WorkspaceInfo = request.app.state.workspace_info
|
|
522
|
+
available = [".pdf", *sorted(OFFICE_EXTENSIONS)] if converter else [".pdf"]
|
|
523
|
+
return {
|
|
524
|
+
"supported_extensions": sorted(SUPPORTED_EXTENSIONS),
|
|
525
|
+
"available_extensions": available,
|
|
526
|
+
"max_bytes": MAX_DOCUMENT_BYTES,
|
|
527
|
+
"documents_dir": info.documents_dir,
|
|
528
|
+
"libreoffice": {"available": bool(converter), "executable": converter},
|
|
529
|
+
"ocr": {"available": False},
|
|
530
|
+
"translation_engines": {
|
|
531
|
+
"default": "precise" if precise["available"] else "legacy",
|
|
532
|
+
"legacy": {"available": True, "status": "ready"},
|
|
533
|
+
"precise": precise,
|
|
534
|
+
},
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
@router.post("/import/upload", status_code=201)
|
|
539
|
+
async def import_document_upload(
|
|
540
|
+
request: Request,
|
|
541
|
+
workspace_path: str = Form(...),
|
|
542
|
+
project_id: str = Form(...),
|
|
543
|
+
project_name: str = Form(...),
|
|
544
|
+
file: UploadFile = File(...),
|
|
545
|
+
) -> dict[str, Any]:
|
|
546
|
+
filename = _safe_name(file.filename or "document")
|
|
547
|
+
|
|
548
|
+
async def writer(path: Path) -> int:
|
|
549
|
+
return await _write_upload(file, path)
|
|
550
|
+
|
|
551
|
+
try:
|
|
552
|
+
return await _finish_import(
|
|
553
|
+
request=request,
|
|
554
|
+
workspace_path=workspace_path,
|
|
555
|
+
project_id=project_id,
|
|
556
|
+
project_name=project_name,
|
|
557
|
+
filename=filename,
|
|
558
|
+
content_type=file.content_type or "",
|
|
559
|
+
origin="upload",
|
|
560
|
+
source_url=None,
|
|
561
|
+
writer=writer,
|
|
562
|
+
)
|
|
563
|
+
finally:
|
|
564
|
+
await file.close()
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
@router.post("/import/url", status_code=201)
|
|
568
|
+
async def import_document_url(req: DocumentUrlImportRequest, request: Request) -> dict[str, Any]:
|
|
569
|
+
# Download into the source path selected from the URL. Resolve the filename
|
|
570
|
+
# before building the managed workspace so the original extension is kept.
|
|
571
|
+
with tempfile.TemporaryDirectory(prefix="crabcode-document-download-") as temp_dir:
|
|
572
|
+
download_path = Path(temp_dir) / "download"
|
|
573
|
+
filename, content_type, size = await _download(str(req.url), download_path)
|
|
574
|
+
|
|
575
|
+
async def copy_writer(path: Path) -> int:
|
|
576
|
+
shutil.copy2(download_path, path)
|
|
577
|
+
return size
|
|
578
|
+
|
|
579
|
+
return await _finish_import(
|
|
580
|
+
request=request,
|
|
581
|
+
workspace_path=req.workspace_path,
|
|
582
|
+
project_id=req.project_id,
|
|
583
|
+
project_name=req.project_name,
|
|
584
|
+
filename=filename,
|
|
585
|
+
content_type=content_type,
|
|
586
|
+
origin="url",
|
|
587
|
+
source_url=str(req.url),
|
|
588
|
+
writer=copy_writer,
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
@router.get("/manifest")
|
|
593
|
+
async def document_manifest(workspace: str, request: Request) -> dict[str, Any]:
|
|
594
|
+
return _read_manifest(_validated_workspace(workspace, _workspace_roots(request)))
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
@router.get("/asset")
|
|
598
|
+
async def document_asset(
|
|
599
|
+
workspace: str,
|
|
600
|
+
request: Request,
|
|
601
|
+
kind: Literal["pdf", "source"] = "pdf",
|
|
602
|
+
) -> FileResponse:
|
|
603
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
604
|
+
manifest = _read_manifest(root)
|
|
605
|
+
relative = manifest.get(kind, {}).get("path")
|
|
606
|
+
if not isinstance(relative, str):
|
|
607
|
+
raise HTTPException(status_code=404, detail="Document asset not found")
|
|
608
|
+
candidate = root / relative
|
|
609
|
+
if candidate.is_symlink():
|
|
610
|
+
raise HTTPException(status_code=403, detail="Document asset is invalid")
|
|
611
|
+
try:
|
|
612
|
+
path = candidate.resolve(strict=True)
|
|
613
|
+
except (FileNotFoundError, OSError) as exc:
|
|
614
|
+
raise HTTPException(status_code=404, detail="Document asset not found") from exc
|
|
615
|
+
if not _is_within(path, (root,)) or not path.is_file():
|
|
616
|
+
raise HTTPException(status_code=403, detail="Document asset is invalid")
|
|
617
|
+
media_type = "application/pdf" if kind == "pdf" else manifest.get("source", {}).get("content_type")
|
|
618
|
+
response = FileResponse(path, media_type=media_type, filename=path.name)
|
|
619
|
+
response.headers["Accept-Ranges"] = "bytes"
|
|
620
|
+
digest = manifest.get(kind, {}).get("sha256")
|
|
621
|
+
response.headers["ETag"] = f'"{digest if isinstance(digest, str) and digest else _sha256(path)}"'
|
|
622
|
+
return response
|
|
623
|
+
|
|
624
|
+
|
|
625
|
+
@router.get("/blog-asset")
|
|
626
|
+
async def document_blog_asset(workspace: str, path: str, request: Request) -> FileResponse:
|
|
627
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
628
|
+
assets = root / "blog-assets"
|
|
629
|
+
try:
|
|
630
|
+
resolved_assets = assets.resolve(strict=True)
|
|
631
|
+
candidate = (assets / path).resolve(strict=True)
|
|
632
|
+
except (FileNotFoundError, OSError) as exc:
|
|
633
|
+
raise HTTPException(status_code=404, detail="Blog asset not found") from exc
|
|
634
|
+
if (
|
|
635
|
+
assets.is_symlink()
|
|
636
|
+
or not resolved_assets.is_dir()
|
|
637
|
+
or not _is_within(resolved_assets, (root,))
|
|
638
|
+
or not _is_within(candidate, (resolved_assets,))
|
|
639
|
+
or not candidate.is_file()
|
|
640
|
+
or candidate.is_symlink()
|
|
641
|
+
):
|
|
642
|
+
raise HTTPException(status_code=403, detail="Blog asset is invalid")
|
|
643
|
+
return FileResponse(candidate, media_type=mimetypes.guess_type(candidate.name)[0] or "application/octet-stream")
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
@router.put("/layout")
|
|
647
|
+
async def save_document_layout(req: DocumentLayoutRequest, workspace: str, request: Request) -> dict[str, Any]:
|
|
648
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
649
|
+
encoded = json.dumps(req.model_dump(), ensure_ascii=False).encode("utf-8")
|
|
650
|
+
if len(encoded) > MAX_LAYOUT_BYTES:
|
|
651
|
+
raise HTTPException(status_code=413, detail="Document layout exceeds the 50MiB limit")
|
|
652
|
+
try:
|
|
653
|
+
internal = _managed_internal_directory(root)
|
|
654
|
+
except ValueError as exc:
|
|
655
|
+
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
|
656
|
+
with _manifest_lock(root):
|
|
657
|
+
return _save_document_layout_locked(root, internal, req)
|
|
658
|
+
|
|
659
|
+
|
|
660
|
+
def _document_annotations_path(root: Path) -> Path:
|
|
661
|
+
try:
|
|
662
|
+
internal = _managed_internal_directory(root)
|
|
663
|
+
except ValueError as exc:
|
|
664
|
+
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
|
665
|
+
path = internal / "annotations.json"
|
|
666
|
+
if path.is_symlink():
|
|
667
|
+
raise HTTPException(status_code=403, detail="Document annotations path is invalid")
|
|
668
|
+
return path
|
|
669
|
+
|
|
670
|
+
|
|
671
|
+
def _read_document_annotations_locked(root: Path) -> list[dict[str, Any]]:
|
|
672
|
+
path = _document_annotations_path(root)
|
|
673
|
+
try:
|
|
674
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
675
|
+
except FileNotFoundError:
|
|
676
|
+
return []
|
|
677
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
678
|
+
raise HTTPException(status_code=500, detail="Document annotations are invalid") from exc
|
|
679
|
+
if not isinstance(value, list):
|
|
680
|
+
raise HTTPException(status_code=500, detail="Document annotations are invalid")
|
|
681
|
+
try:
|
|
682
|
+
return [DocumentAnnotationWriteRequest.model_validate(item).model_dump() for item in value]
|
|
683
|
+
except Exception as exc:
|
|
684
|
+
raise HTTPException(status_code=500, detail="Document annotations are invalid") from exc
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
@router.get("/annotations")
|
|
688
|
+
async def read_document_annotations(workspace: str, request: Request) -> list[dict[str, Any]]:
|
|
689
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
690
|
+
with _manifest_lock(root):
|
|
691
|
+
return _read_document_annotations_locked(root)
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
@router.put("/annotations")
|
|
695
|
+
async def write_document_annotation(
|
|
696
|
+
req: DocumentAnnotationWriteRequest,
|
|
697
|
+
workspace: str,
|
|
698
|
+
request: Request,
|
|
699
|
+
) -> dict[str, Any]:
|
|
700
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
701
|
+
with _manifest_lock(root):
|
|
702
|
+
annotations = _read_document_annotations_locked(root)
|
|
703
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
704
|
+
previous = next((item for item in annotations if item["id"] == req.id), None)
|
|
705
|
+
annotation = req.model_dump()
|
|
706
|
+
annotation["created_at"] = previous.get("created_at") if previous else now
|
|
707
|
+
annotation["updated_at"] = now
|
|
708
|
+
next_annotations = [item for item in annotations if item["id"] != req.id]
|
|
709
|
+
next_annotations.append(annotation)
|
|
710
|
+
_json_write(_document_annotations_path(root), next_annotations)
|
|
711
|
+
return annotation
|
|
712
|
+
|
|
713
|
+
|
|
714
|
+
@router.delete("/annotations/{annotation_id}", status_code=204)
|
|
715
|
+
async def delete_document_annotation(
|
|
716
|
+
annotation_id: str,
|
|
717
|
+
workspace: str,
|
|
718
|
+
request: Request,
|
|
719
|
+
) -> Response:
|
|
720
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", annotation_id):
|
|
721
|
+
raise HTTPException(status_code=400, detail="Invalid annotation id")
|
|
722
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
723
|
+
with _manifest_lock(root):
|
|
724
|
+
annotations = _read_document_annotations_locked(root)
|
|
725
|
+
next_annotations = [item for item in annotations if item["id"] != annotation_id]
|
|
726
|
+
if len(next_annotations) == len(annotations):
|
|
727
|
+
raise HTTPException(status_code=404, detail="Document annotation not found")
|
|
728
|
+
_json_write(_document_annotations_path(root), next_annotations)
|
|
729
|
+
return Response(status_code=204)
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def _save_document_layout_locked(
|
|
733
|
+
root: Path,
|
|
734
|
+
internal: Path,
|
|
735
|
+
req: DocumentLayoutRequest,
|
|
736
|
+
) -> dict[str, Any]:
|
|
737
|
+
layout_path = internal / "layout.json"
|
|
738
|
+
content_path = internal / "content.md"
|
|
739
|
+
if content_path.is_symlink():
|
|
740
|
+
raise HTTPException(status_code=403, detail="Document content path is invalid")
|
|
741
|
+
lines: list[str] = []
|
|
742
|
+
text_pages = 0
|
|
743
|
+
for page_number, page in enumerate(req.pages, 1):
|
|
744
|
+
blocks = page.get("blocks") if isinstance(page, dict) else None
|
|
745
|
+
page_lines = [str(block.get("text", "")).strip() for block in (blocks or []) if isinstance(block, dict)]
|
|
746
|
+
page_lines = [line for line in page_lines if line]
|
|
747
|
+
if page_lines:
|
|
748
|
+
text_pages += 1
|
|
749
|
+
lines.extend([f"\n<!-- page:{page_number} -->\n", *page_lines])
|
|
750
|
+
_json_write(layout_path, req.model_dump())
|
|
751
|
+
_text_write(content_path, "\n\n".join(lines).strip() + "\n")
|
|
752
|
+
manifest = _read_manifest(root)
|
|
753
|
+
previous_fingerprint = (manifest.get("layout") or {}).get("fingerprint")
|
|
754
|
+
if previous_fingerprint and previous_fingerprint != req.fingerprint:
|
|
755
|
+
translations = internal / "translations"
|
|
756
|
+
if translations.is_symlink():
|
|
757
|
+
translations.unlink(missing_ok=True)
|
|
758
|
+
elif translations.is_dir():
|
|
759
|
+
# A browser layout change invalidates only legacy block overlays.
|
|
760
|
+
# Precise PDFs are derived from the source PDF and remain valid.
|
|
761
|
+
for legacy_path in translations.glob("*.json"):
|
|
762
|
+
if legacy_path.is_file() and not legacy_path.is_symlink():
|
|
763
|
+
legacy_path.unlink(missing_ok=True)
|
|
764
|
+
(root / "blog.md").unlink(missing_ok=True)
|
|
765
|
+
current_translations = manifest.get("translations")
|
|
766
|
+
manifest["translations"] = {
|
|
767
|
+
locale: entry
|
|
768
|
+
for locale, entry in (current_translations.items() if isinstance(current_translations, dict) else [])
|
|
769
|
+
if isinstance(entry, dict) and entry.get("engine") == "precise"
|
|
770
|
+
}
|
|
771
|
+
manifest["blog"] = None
|
|
772
|
+
manifest["layout"] = {
|
|
773
|
+
"path": layout_path.relative_to(root).as_posix(),
|
|
774
|
+
"fingerprint": req.fingerprint,
|
|
775
|
+
"text_pages": text_pages,
|
|
776
|
+
"scanned_pages": req.page_count - text_pages,
|
|
777
|
+
}
|
|
778
|
+
manifest.setdefault("pdf", {})["page_count"] = req.page_count
|
|
779
|
+
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
780
|
+
_json_write(root / MANIFEST_RELATIVE_PATH, manifest)
|
|
781
|
+
return manifest["layout"]
|
|
782
|
+
|
|
783
|
+
|
|
784
|
+
@router.get("/translation")
|
|
785
|
+
async def document_translation(workspace: str, locale: str, request: Request) -> dict[str, Any]:
|
|
786
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
787
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
|
|
788
|
+
raise HTTPException(status_code=400, detail="Invalid translation locale")
|
|
789
|
+
try:
|
|
790
|
+
internal = _managed_internal_directory(root)
|
|
791
|
+
except ValueError as exc:
|
|
792
|
+
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
|
793
|
+
with _manifest_lock(root):
|
|
794
|
+
return _read_document_translation_locked(root, internal, locale)
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
@router.delete("/translation")
|
|
798
|
+
async def clear_document_translation(workspace: str, locale: str, request: Request) -> dict[str, Any]:
|
|
799
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
800
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
|
|
801
|
+
raise HTTPException(status_code=400, detail="Invalid translation locale")
|
|
802
|
+
try:
|
|
803
|
+
internal = _managed_internal_directory(root)
|
|
804
|
+
except ValueError as exc:
|
|
805
|
+
raise HTTPException(status_code=403, detail=str(exc)) from exc
|
|
806
|
+
with _manifest_lock(root):
|
|
807
|
+
return _clear_document_translation_locked(root, internal, locale)
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _read_document_translation_locked(root: Path, internal: Path, locale: str) -> dict[str, Any]:
|
|
811
|
+
translations = internal / "translations"
|
|
812
|
+
if translations.is_symlink():
|
|
813
|
+
raise HTTPException(status_code=403, detail="Managed translations directory is invalid")
|
|
814
|
+
manifest = _read_manifest(root)
|
|
815
|
+
entries = manifest.get("translations")
|
|
816
|
+
entry = entries.get(locale) if isinstance(entries, dict) else None
|
|
817
|
+
if isinstance(entry, dict) and entry.get("engine") == "precise":
|
|
818
|
+
relative = entry.get("path")
|
|
819
|
+
if not isinstance(relative, str):
|
|
820
|
+
raise HTTPException(status_code=500, detail="Precise translation metadata is invalid")
|
|
821
|
+
candidate = root / relative
|
|
822
|
+
if candidate.is_symlink():
|
|
823
|
+
raise HTTPException(status_code=403, detail="Precise translation path is invalid")
|
|
824
|
+
try:
|
|
825
|
+
path = candidate.resolve(strict=True)
|
|
826
|
+
except (FileNotFoundError, OSError) as exc:
|
|
827
|
+
raise HTTPException(status_code=404, detail="Translation not found") from exc
|
|
828
|
+
if not _is_within(path, (root.resolve(strict=True),)) or not path.is_file():
|
|
829
|
+
raise HTTPException(status_code=403, detail="Precise translation path is invalid")
|
|
830
|
+
source_sha256 = str(entry.get("source_sha256") or "")
|
|
831
|
+
if source_sha256 != str(manifest.get("source", {}).get("sha256") or ""):
|
|
832
|
+
raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
|
|
833
|
+
digest = str(entry.get("pdf_sha256") or entry.get("sha256") or "")
|
|
834
|
+
if digest and _sha256(path) != digest:
|
|
835
|
+
raise HTTPException(status_code=500, detail="Precise translation checksum failed")
|
|
836
|
+
return {
|
|
837
|
+
"engine": "precise",
|
|
838
|
+
"locale": locale,
|
|
839
|
+
"source_sha256": source_sha256,
|
|
840
|
+
"pdf_sha256": digest or _sha256(path),
|
|
841
|
+
"page_count": int(entry.get("page_count") or 0),
|
|
842
|
+
"engine_version": str(entry.get("engine_version") or ""),
|
|
843
|
+
"warnings": entry.get("warnings") if isinstance(entry.get("warnings"), list) else [],
|
|
844
|
+
}
|
|
845
|
+
path = translations / f"{locale}.json"
|
|
846
|
+
if path.is_symlink():
|
|
847
|
+
raise HTTPException(status_code=403, detail="Translation path is invalid")
|
|
848
|
+
try:
|
|
849
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
850
|
+
except FileNotFoundError as exc:
|
|
851
|
+
raise HTTPException(status_code=404, detail="Translation not found") from exc
|
|
852
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
853
|
+
raise HTTPException(status_code=500, detail="Translation is invalid") from exc
|
|
854
|
+
if not isinstance(value, dict):
|
|
855
|
+
raise HTTPException(status_code=500, detail="Translation is invalid")
|
|
856
|
+
if value.get("source_sha256") != manifest.get("source", {}).get("sha256"):
|
|
857
|
+
raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
|
|
858
|
+
if value.get("layout_fingerprint") != (manifest.get("layout") or {}).get("fingerprint"):
|
|
859
|
+
raise HTTPException(status_code=409, detail="Translation is stale for the current document layout")
|
|
860
|
+
return {**value, "engine": "legacy"}
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
@router.get("/translation/asset")
|
|
864
|
+
async def document_translation_asset(workspace: str, locale: str, request: Request) -> FileResponse:
|
|
865
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
866
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
|
|
867
|
+
raise HTTPException(status_code=400, detail="Invalid translation locale")
|
|
868
|
+
with _manifest_lock(root):
|
|
869
|
+
manifest = _read_manifest(root)
|
|
870
|
+
entries = manifest.get("translations")
|
|
871
|
+
entry = entries.get(locale) if isinstance(entries, dict) else None
|
|
872
|
+
if not isinstance(entry, dict) or entry.get("engine") != "precise":
|
|
873
|
+
raise HTTPException(status_code=404, detail="Precise translation not found")
|
|
874
|
+
if entry.get("source_sha256") != manifest.get("source", {}).get("sha256"):
|
|
875
|
+
raise HTTPException(status_code=409, detail="Translation is stale for the current source document")
|
|
876
|
+
relative = entry.get("path")
|
|
877
|
+
if not isinstance(relative, str):
|
|
878
|
+
raise HTTPException(status_code=500, detail="Precise translation metadata is invalid")
|
|
879
|
+
candidate = root / relative
|
|
880
|
+
if candidate.is_symlink():
|
|
881
|
+
raise HTTPException(status_code=403, detail="Precise translation path is invalid")
|
|
882
|
+
try:
|
|
883
|
+
path = candidate.resolve(strict=True)
|
|
884
|
+
except (FileNotFoundError, OSError) as exc:
|
|
885
|
+
raise HTTPException(status_code=404, detail="Precise translation not found") from exc
|
|
886
|
+
if not _is_within(path, (root.resolve(strict=True),)) or not path.is_file():
|
|
887
|
+
raise HTTPException(status_code=403, detail="Precise translation path is invalid")
|
|
888
|
+
actual_digest = _sha256(path)
|
|
889
|
+
digest = str(entry.get("pdf_sha256") or entry.get("sha256") or "")
|
|
890
|
+
if digest and digest != actual_digest:
|
|
891
|
+
raise HTTPException(status_code=500, detail="Precise translation checksum failed")
|
|
892
|
+
digest = digest or actual_digest
|
|
893
|
+
response = FileResponse(path, media_type="application/pdf", filename=f"translated-{locale}.pdf")
|
|
894
|
+
response.headers["Accept-Ranges"] = "bytes"
|
|
895
|
+
response.headers["ETag"] = f'"{digest}"'
|
|
896
|
+
return response
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _clear_document_translation_locked(root: Path, internal: Path, locale: str) -> dict[str, Any]:
|
|
900
|
+
manifest = _read_manifest(root)
|
|
901
|
+
jobs = manifest.get("jobs")
|
|
902
|
+
if not isinstance(jobs, dict):
|
|
903
|
+
jobs = {}
|
|
904
|
+
matching_job_ids = {
|
|
905
|
+
operation_id
|
|
906
|
+
for operation_id, job in jobs.items()
|
|
907
|
+
if isinstance(operation_id, str)
|
|
908
|
+
and re.fullmatch(r"[A-Za-z0-9_-]{1,100}", operation_id)
|
|
909
|
+
and isinstance(job, dict)
|
|
910
|
+
and job.get("action") == "translate"
|
|
911
|
+
and job.get("locale") == locale
|
|
912
|
+
}
|
|
913
|
+
if any(
|
|
914
|
+
isinstance(job, dict)
|
|
915
|
+
and job.get("action") == "translate"
|
|
916
|
+
and job.get("locale") == locale
|
|
917
|
+
and job.get("status") == "running"
|
|
918
|
+
for job in jobs.values()
|
|
919
|
+
):
|
|
920
|
+
raise HTTPException(status_code=409, detail="Translation is still running")
|
|
921
|
+
|
|
922
|
+
jobs_dir = internal / "jobs"
|
|
923
|
+
if jobs_dir.is_symlink() or (jobs_dir.exists() and not jobs_dir.is_dir()):
|
|
924
|
+
raise HTTPException(status_code=403, detail="Managed document jobs directory is invalid")
|
|
925
|
+
if jobs_dir.is_dir():
|
|
926
|
+
for job_dir in jobs_dir.iterdir():
|
|
927
|
+
if job_dir.is_symlink() or not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", job_dir.name):
|
|
928
|
+
continue
|
|
929
|
+
staged = job_dir / "translation.json"
|
|
930
|
+
precise_cache = job_dir / "precise-cache.json"
|
|
931
|
+
for candidate in (staged, precise_cache):
|
|
932
|
+
if not candidate.is_file() or candidate.is_symlink():
|
|
933
|
+
continue
|
|
934
|
+
value = _read_json_file(candidate)
|
|
935
|
+
if isinstance(value, dict) and value.get("locale") == locale:
|
|
936
|
+
matching_job_ids.add(job_dir.name)
|
|
937
|
+
break
|
|
938
|
+
|
|
939
|
+
translations_dir = internal / "translations"
|
|
940
|
+
if translations_dir.is_symlink() or (translations_dir.exists() and not translations_dir.is_dir()):
|
|
941
|
+
raise HTTPException(status_code=403, detail="Managed translations directory is invalid")
|
|
942
|
+
translation_path = translations_dir / f"{locale}.json"
|
|
943
|
+
if translation_path.is_symlink() or (translation_path.exists() and not translation_path.is_file()):
|
|
944
|
+
raise HTTPException(status_code=403, detail="Translation path is invalid")
|
|
945
|
+
precise_path = translations_dir / locale
|
|
946
|
+
if precise_path.is_symlink() or (precise_path.exists() and not precise_path.is_dir()):
|
|
947
|
+
raise HTTPException(status_code=403, detail="Precise translation directory is invalid")
|
|
948
|
+
removed_translation = translation_path.is_file() or precise_path.is_dir()
|
|
949
|
+
translation_path.unlink(missing_ok=True)
|
|
950
|
+
if precise_path.is_dir():
|
|
951
|
+
shutil.rmtree(precise_path)
|
|
952
|
+
|
|
953
|
+
for operation_id in matching_job_ids:
|
|
954
|
+
_cleanup_document_job(root, operation_id)
|
|
955
|
+
|
|
956
|
+
translations = manifest.get("translations")
|
|
957
|
+
if not isinstance(translations, dict):
|
|
958
|
+
translations = {}
|
|
959
|
+
translations.pop(locale, None)
|
|
960
|
+
manifest["translations"] = translations
|
|
961
|
+
manifest["jobs"] = {
|
|
962
|
+
operation_id: job
|
|
963
|
+
for operation_id, job in jobs.items()
|
|
964
|
+
if operation_id not in matching_job_ids
|
|
965
|
+
}
|
|
966
|
+
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
967
|
+
_json_write(root / MANIFEST_RELATIVE_PATH, manifest)
|
|
968
|
+
return {
|
|
969
|
+
"locale": locale,
|
|
970
|
+
"removed_translation": removed_translation,
|
|
971
|
+
"removed_jobs": len(matching_job_ids),
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _blog_revision(markdown: str) -> str:
|
|
976
|
+
return hashlib.sha256(markdown.encode("utf-8")).hexdigest()
|
|
977
|
+
|
|
978
|
+
|
|
979
|
+
def _managed_internal_directory(workspace: Path) -> Path:
|
|
980
|
+
try:
|
|
981
|
+
resolved_workspace = workspace.resolve(strict=True)
|
|
982
|
+
except OSError as exc:
|
|
983
|
+
raise ValueError("document workspace is unavailable") from exc
|
|
984
|
+
crabcode = workspace / ".crabcode"
|
|
985
|
+
internal = crabcode / "document"
|
|
986
|
+
if crabcode.is_symlink() or internal.is_symlink():
|
|
987
|
+
raise ValueError("managed document directories cannot be symlinks")
|
|
988
|
+
try:
|
|
989
|
+
resolved_internal = internal.resolve(strict=True)
|
|
990
|
+
except OSError as exc:
|
|
991
|
+
raise ValueError("managed document directory is unavailable") from exc
|
|
992
|
+
if not resolved_internal.is_dir() or not _is_within(resolved_internal, (resolved_workspace,)):
|
|
993
|
+
raise ValueError("managed document directory is outside the workspace")
|
|
994
|
+
return internal
|
|
995
|
+
|
|
996
|
+
|
|
997
|
+
def _document_job_directory(workspace: Path, operation_id: str) -> Path:
|
|
998
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,100}", operation_id):
|
|
999
|
+
raise ValueError("invalid document operation id")
|
|
1000
|
+
internal = _managed_internal_directory(workspace)
|
|
1001
|
+
jobs = internal / "jobs"
|
|
1002
|
+
if jobs.is_symlink() or not _is_within(jobs.resolve(strict=False), (workspace.resolve(strict=True),)):
|
|
1003
|
+
raise ValueError("document jobs directory is invalid")
|
|
1004
|
+
return jobs / operation_id
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def _document_job_total(workspace: Path, action: str, engine: str = "legacy") -> int:
|
|
1008
|
+
if action != "translate":
|
|
1009
|
+
return 1
|
|
1010
|
+
if engine == "precise":
|
|
1011
|
+
return 100
|
|
1012
|
+
return sum(len(page) for page in _translation_source_pages(workspace))
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def _document_pdf_path(workspace: Path) -> Path:
|
|
1016
|
+
manifest = _read_manifest(workspace)
|
|
1017
|
+
relative = manifest.get("pdf", {}).get("path")
|
|
1018
|
+
if not isinstance(relative, str):
|
|
1019
|
+
raise ValueError("document PDF is unavailable")
|
|
1020
|
+
candidate = workspace / relative
|
|
1021
|
+
if candidate.is_symlink():
|
|
1022
|
+
raise ValueError("document PDF path is invalid")
|
|
1023
|
+
try:
|
|
1024
|
+
path = candidate.resolve(strict=True)
|
|
1025
|
+
except OSError as exc:
|
|
1026
|
+
raise ValueError("document PDF is unavailable") from exc
|
|
1027
|
+
if not path.is_file() or not _is_within(path, (workspace.resolve(strict=True),)):
|
|
1028
|
+
raise ValueError("document PDF path is invalid")
|
|
1029
|
+
return path
|
|
1030
|
+
|
|
1031
|
+
|
|
1032
|
+
def _translation_source_pages(workspace: Path) -> list[list[tuple[str, str]]]:
|
|
1033
|
+
"""Return exact non-empty layout blocks grouped by page."""
|
|
1034
|
+
try:
|
|
1035
|
+
internal = _managed_internal_directory(workspace)
|
|
1036
|
+
layout_path = internal / "layout.json"
|
|
1037
|
+
if layout_path.is_symlink():
|
|
1038
|
+
raise ValueError("document layout is invalid")
|
|
1039
|
+
layout = json.loads(layout_path.read_text(encoding="utf-8"))
|
|
1040
|
+
raw_pages = layout.get("pages")
|
|
1041
|
+
if not isinstance(raw_pages, list):
|
|
1042
|
+
raise ValueError("document layout is not ready")
|
|
1043
|
+
except (OSError, json.JSONDecodeError, AttributeError, TypeError) as exc:
|
|
1044
|
+
raise ValueError("document layout is not ready") from exc
|
|
1045
|
+
|
|
1046
|
+
pages: list[list[tuple[str, str]]] = []
|
|
1047
|
+
seen: set[str] = set()
|
|
1048
|
+
for raw_page in raw_pages:
|
|
1049
|
+
if not isinstance(raw_page, dict) or not isinstance(raw_page.get("blocks", []), list):
|
|
1050
|
+
raise ValueError("document layout contains invalid pages")
|
|
1051
|
+
page: list[tuple[str, str]] = []
|
|
1052
|
+
for block in raw_page.get("blocks", []):
|
|
1053
|
+
if not isinstance(block, dict):
|
|
1054
|
+
raise ValueError("document layout contains invalid blocks")
|
|
1055
|
+
text = block.get("text")
|
|
1056
|
+
if not isinstance(text, str) or not text.strip():
|
|
1057
|
+
continue
|
|
1058
|
+
block_id = block.get("id")
|
|
1059
|
+
if not isinstance(block_id, str) or not block_id or block_id in seen:
|
|
1060
|
+
raise ValueError("document layout contains invalid or duplicate block ids")
|
|
1061
|
+
seen.add(block_id)
|
|
1062
|
+
page.append((block_id, text))
|
|
1063
|
+
pages.append(page)
|
|
1064
|
+
return pages
|
|
1065
|
+
|
|
1066
|
+
|
|
1067
|
+
def _translation_preserved_blocks(workspace: Path) -> dict[str, str]:
|
|
1068
|
+
"""Return fixed-layout blocks that must bypass model translation."""
|
|
1069
|
+
try:
|
|
1070
|
+
internal = _managed_internal_directory(workspace)
|
|
1071
|
+
layout_path = internal / "layout.json"
|
|
1072
|
+
if layout_path.is_symlink():
|
|
1073
|
+
raise ValueError("document layout is invalid")
|
|
1074
|
+
layout = json.loads(layout_path.read_text(encoding="utf-8"))
|
|
1075
|
+
raw_pages = layout.get("pages")
|
|
1076
|
+
if not isinstance(raw_pages, list):
|
|
1077
|
+
raise ValueError("document layout is not ready")
|
|
1078
|
+
except (OSError, json.JSONDecodeError, AttributeError, TypeError) as exc:
|
|
1079
|
+
raise ValueError("document layout is not ready") from exc
|
|
1080
|
+
|
|
1081
|
+
preserved: dict[str, str] = {}
|
|
1082
|
+
for raw_page in raw_pages:
|
|
1083
|
+
if not isinstance(raw_page, dict) or not isinstance(raw_page.get("blocks", []), list):
|
|
1084
|
+
raise ValueError("document layout contains invalid pages")
|
|
1085
|
+
for block in raw_page.get("blocks", []):
|
|
1086
|
+
if not isinstance(block, dict) or block.get("kind") not in {"formula", "graphic"}:
|
|
1087
|
+
continue
|
|
1088
|
+
block_id = block.get("id")
|
|
1089
|
+
text = block.get("text")
|
|
1090
|
+
if not isinstance(block_id, str) or not block_id or not isinstance(text, str) or not text.strip():
|
|
1091
|
+
raise ValueError("document layout contains invalid preserved blocks")
|
|
1092
|
+
preserved[block_id] = text
|
|
1093
|
+
return preserved
|
|
1094
|
+
|
|
1095
|
+
|
|
1096
|
+
def _validated_translation_file(
|
|
1097
|
+
workspace: Path,
|
|
1098
|
+
path: Path,
|
|
1099
|
+
locale: str | None,
|
|
1100
|
+
) -> dict[str, str]:
|
|
1101
|
+
if path.is_symlink() or path.parent.is_symlink():
|
|
1102
|
+
raise ValueError("translation job file is invalid")
|
|
1103
|
+
try:
|
|
1104
|
+
staged = json.loads(path.read_text(encoding="utf-8"))
|
|
1105
|
+
manifest = _read_manifest(workspace)
|
|
1106
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
1107
|
+
raise ValueError("translation job file is invalid") from exc
|
|
1108
|
+
if not isinstance(staged, dict):
|
|
1109
|
+
raise ValueError("translation job file is invalid")
|
|
1110
|
+
staged_locale = staged.get("locale")
|
|
1111
|
+
if locale is not None and staged_locale is not None and staged_locale != locale:
|
|
1112
|
+
raise ValueError("translation job locale does not match")
|
|
1113
|
+
if staged.get("source_sha256") != manifest.get("source", {}).get("sha256"):
|
|
1114
|
+
raise ValueError("translation was generated for a different source document")
|
|
1115
|
+
if staged.get("layout_fingerprint") != (manifest.get("layout") or {}).get("fingerprint"):
|
|
1116
|
+
raise ValueError("translation was generated for a different document layout")
|
|
1117
|
+
blocks = staged.get("blocks")
|
|
1118
|
+
if not isinstance(blocks, list):
|
|
1119
|
+
raise ValueError("translation JSON is missing blocks")
|
|
1120
|
+
|
|
1121
|
+
expected_ids = {
|
|
1122
|
+
block_id
|
|
1123
|
+
for page in _translation_source_pages(workspace)
|
|
1124
|
+
for block_id, _ in page
|
|
1125
|
+
}
|
|
1126
|
+
translated: dict[str, str] = {}
|
|
1127
|
+
for block in blocks:
|
|
1128
|
+
if not isinstance(block, dict):
|
|
1129
|
+
raise ValueError("translation blocks must contain id and translated_text")
|
|
1130
|
+
block_id = block.get("id")
|
|
1131
|
+
translated_text = block.get("translated_text")
|
|
1132
|
+
if not isinstance(block_id, str) or not isinstance(translated_text, str):
|
|
1133
|
+
raise ValueError("translation blocks must contain id and translated_text")
|
|
1134
|
+
if not translated_text.strip():
|
|
1135
|
+
raise ValueError(f"translation is empty for block id: {block_id}")
|
|
1136
|
+
if block_id in translated:
|
|
1137
|
+
raise ValueError(f"translation contains duplicate block id: {block_id}")
|
|
1138
|
+
if block_id not in expected_ids:
|
|
1139
|
+
raise ValueError(f"translation contains unknown block id: {block_id}")
|
|
1140
|
+
translated[block_id] = translated_text
|
|
1141
|
+
return translated
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def _write_translation_mapping(
|
|
1145
|
+
workspace: Path,
|
|
1146
|
+
operation_id: str,
|
|
1147
|
+
locale: str,
|
|
1148
|
+
translated: dict[str, str],
|
|
1149
|
+
) -> int:
|
|
1150
|
+
pages = _translation_source_pages(workspace)
|
|
1151
|
+
expected_ids = [block_id for page in pages for block_id, _ in page]
|
|
1152
|
+
expected = set(expected_ids)
|
|
1153
|
+
if not set(translated).issubset(expected):
|
|
1154
|
+
raise ValueError("translation contains unknown block ids")
|
|
1155
|
+
if any(not isinstance(text, str) or not text.strip() for text in translated.values()):
|
|
1156
|
+
raise ValueError("translation contains empty text")
|
|
1157
|
+
manifest = _read_manifest(workspace)
|
|
1158
|
+
target = _document_job_directory(workspace, operation_id) / "translation.json"
|
|
1159
|
+
if target.is_symlink() or target.parent.is_symlink():
|
|
1160
|
+
raise ValueError("translation job file is invalid")
|
|
1161
|
+
_json_write(target, {
|
|
1162
|
+
"locale": locale,
|
|
1163
|
+
"source_sha256": manifest.get("source", {}).get("sha256"),
|
|
1164
|
+
"layout_fingerprint": (manifest.get("layout") or {}).get("fingerprint"),
|
|
1165
|
+
"blocks": [
|
|
1166
|
+
{"id": block_id, "translated_text": translated[block_id]}
|
|
1167
|
+
for block_id in expected_ids
|
|
1168
|
+
if block_id in translated
|
|
1169
|
+
],
|
|
1170
|
+
})
|
|
1171
|
+
return len(translated)
|
|
1172
|
+
|
|
1173
|
+
|
|
1174
|
+
def _translation_job_blocks(
|
|
1175
|
+
workspace: Path,
|
|
1176
|
+
operation_id: str,
|
|
1177
|
+
locale: str,
|
|
1178
|
+
) -> dict[str, str]:
|
|
1179
|
+
target = _document_job_directory(workspace, operation_id) / "translation.json"
|
|
1180
|
+
if not target.is_file():
|
|
1181
|
+
return {}
|
|
1182
|
+
return _validated_translation_file(workspace, target, locale)
|
|
1183
|
+
|
|
1184
|
+
|
|
1185
|
+
def _store_translation_batch(
|
|
1186
|
+
workspace: Path,
|
|
1187
|
+
operation_id: str,
|
|
1188
|
+
locale: str,
|
|
1189
|
+
batch: dict[str, str],
|
|
1190
|
+
) -> int:
|
|
1191
|
+
"""Validate and atomically merge one model-produced translation batch."""
|
|
1192
|
+
translated = _translation_job_blocks(workspace, operation_id, locale)
|
|
1193
|
+
translated.update(batch)
|
|
1194
|
+
return _write_translation_mapping(workspace, operation_id, locale, translated)
|
|
1195
|
+
|
|
1196
|
+
|
|
1197
|
+
def _cleanup_document_job(workspace: Path, operation_id: str) -> None:
|
|
1198
|
+
try:
|
|
1199
|
+
job = _document_job_directory(workspace, operation_id)
|
|
1200
|
+
except ValueError:
|
|
1201
|
+
return
|
|
1202
|
+
if job.is_symlink():
|
|
1203
|
+
job.unlink(missing_ok=True)
|
|
1204
|
+
else:
|
|
1205
|
+
shutil.rmtree(job, ignore_errors=True)
|
|
1206
|
+
|
|
1207
|
+
|
|
1208
|
+
def _translation_job_progress(workspace: Path, operation_id: str) -> int:
|
|
1209
|
+
path = _document_job_directory(workspace, operation_id) / "translation.json"
|
|
1210
|
+
if not path.is_file() or path.is_symlink():
|
|
1211
|
+
return 0
|
|
1212
|
+
try:
|
|
1213
|
+
return len(_validated_translation_file(workspace, path, None))
|
|
1214
|
+
except ValueError:
|
|
1215
|
+
return 0
|
|
1216
|
+
|
|
1217
|
+
|
|
1218
|
+
def _recover_translation_job(
|
|
1219
|
+
workspace: Path,
|
|
1220
|
+
operation_id: str,
|
|
1221
|
+
locale: str,
|
|
1222
|
+
engine: str = "legacy",
|
|
1223
|
+
) -> int:
|
|
1224
|
+
"""Seed a new translation operation from the newest valid partial batch."""
|
|
1225
|
+
internal = _managed_internal_directory(workspace)
|
|
1226
|
+
jobs = internal / "jobs"
|
|
1227
|
+
if jobs.is_symlink():
|
|
1228
|
+
raise ValueError("document jobs directory is invalid")
|
|
1229
|
+
target = _document_job_directory(workspace, operation_id)
|
|
1230
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
1231
|
+
if engine == "precise":
|
|
1232
|
+
manifest = _read_manifest(workspace)
|
|
1233
|
+
source_sha256 = str(manifest.get("source", {}).get("sha256") or "")
|
|
1234
|
+
candidates_with_time: list[tuple[float, Path]] = []
|
|
1235
|
+
for path in jobs.glob("*/precise-cache.json"):
|
|
1236
|
+
if path.parent.name == operation_id or path.is_symlink() or path.parent.is_symlink():
|
|
1237
|
+
continue
|
|
1238
|
+
try:
|
|
1239
|
+
candidates_with_time.append((path.stat().st_mtime, path))
|
|
1240
|
+
except OSError:
|
|
1241
|
+
continue
|
|
1242
|
+
for _, candidate in sorted(candidates_with_time, reverse=True):
|
|
1243
|
+
value = _read_json_file(candidate)
|
|
1244
|
+
if (
|
|
1245
|
+
isinstance(value, dict)
|
|
1246
|
+
and value.get("locale") == locale
|
|
1247
|
+
and value.get("source_sha256") == source_sha256
|
|
1248
|
+
and value.get("engine_version") == "0.6.4"
|
|
1249
|
+
and isinstance(value.get("entries"), dict)
|
|
1250
|
+
):
|
|
1251
|
+
shutil.copy2(candidate, target / "precise-cache.json")
|
|
1252
|
+
# Cached paragraph count is not a stable percentage because
|
|
1253
|
+
# BabelDOC has not parsed this run yet.
|
|
1254
|
+
return 0
|
|
1255
|
+
return 0
|
|
1256
|
+
_translation_source_pages(workspace)
|
|
1257
|
+
candidates_with_time: list[tuple[float, Path]] = []
|
|
1258
|
+
for path in jobs.glob("*/translation.json"):
|
|
1259
|
+
if path.parent.name == operation_id or path.is_symlink() or path.parent.is_symlink():
|
|
1260
|
+
continue
|
|
1261
|
+
try:
|
|
1262
|
+
candidates_with_time.append((path.stat().st_mtime, path))
|
|
1263
|
+
except OSError:
|
|
1264
|
+
continue
|
|
1265
|
+
candidates = [path for _, path in sorted(candidates_with_time, reverse=True)]
|
|
1266
|
+
for candidate in candidates:
|
|
1267
|
+
try:
|
|
1268
|
+
translated = _validated_translation_file(workspace, candidate, locale)
|
|
1269
|
+
except ValueError:
|
|
1270
|
+
continue
|
|
1271
|
+
if translated:
|
|
1272
|
+
return _write_translation_mapping(workspace, operation_id, locale, translated)
|
|
1273
|
+
return 0
|
|
1274
|
+
|
|
1275
|
+
|
|
1276
|
+
def _read_json_file(path: Path) -> dict[str, Any] | None:
|
|
1277
|
+
try:
|
|
1278
|
+
value = json.loads(path.read_text(encoding="utf-8"))
|
|
1279
|
+
except (OSError, json.JSONDecodeError):
|
|
1280
|
+
return None
|
|
1281
|
+
return value if isinstance(value, dict) else None
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
def _update_document_job_status(
|
|
1285
|
+
workspace: Path,
|
|
1286
|
+
operation_id: str,
|
|
1287
|
+
*,
|
|
1288
|
+
action: str,
|
|
1289
|
+
status: str,
|
|
1290
|
+
locale: str,
|
|
1291
|
+
source: str,
|
|
1292
|
+
current: int,
|
|
1293
|
+
total: int,
|
|
1294
|
+
message: str,
|
|
1295
|
+
language: str = "",
|
|
1296
|
+
engine: str = "legacy",
|
|
1297
|
+
) -> None:
|
|
1298
|
+
with _manifest_lock(workspace):
|
|
1299
|
+
_update_document_job_status_unlocked(
|
|
1300
|
+
workspace,
|
|
1301
|
+
operation_id,
|
|
1302
|
+
action=action,
|
|
1303
|
+
status=status,
|
|
1304
|
+
locale=locale,
|
|
1305
|
+
source=source,
|
|
1306
|
+
current=current,
|
|
1307
|
+
total=total,
|
|
1308
|
+
message=message,
|
|
1309
|
+
language=language,
|
|
1310
|
+
engine=engine,
|
|
1311
|
+
)
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def _update_document_job_status_unlocked(
|
|
1315
|
+
workspace: Path,
|
|
1316
|
+
operation_id: str,
|
|
1317
|
+
*,
|
|
1318
|
+
action: str,
|
|
1319
|
+
status: str,
|
|
1320
|
+
locale: str,
|
|
1321
|
+
source: str,
|
|
1322
|
+
current: int,
|
|
1323
|
+
total: int,
|
|
1324
|
+
message: str,
|
|
1325
|
+
language: str = "",
|
|
1326
|
+
engine: str = "legacy",
|
|
1327
|
+
) -> None:
|
|
1328
|
+
_managed_internal_directory(workspace)
|
|
1329
|
+
manifest = _read_manifest(workspace)
|
|
1330
|
+
jobs = manifest.get("jobs")
|
|
1331
|
+
if not isinstance(jobs, dict):
|
|
1332
|
+
jobs = {}
|
|
1333
|
+
jobs[operation_id] = {
|
|
1334
|
+
"action": action,
|
|
1335
|
+
"status": status,
|
|
1336
|
+
"locale": locale,
|
|
1337
|
+
"language": language,
|
|
1338
|
+
"source": source,
|
|
1339
|
+
"engine": engine,
|
|
1340
|
+
"current": current,
|
|
1341
|
+
"total": total,
|
|
1342
|
+
"message": message[:500],
|
|
1343
|
+
"updated_at": datetime.now(timezone.utc).isoformat(),
|
|
1344
|
+
}
|
|
1345
|
+
if len(jobs) > 50:
|
|
1346
|
+
jobs = dict(sorted(jobs.items(), key=lambda item: item[1].get("updated_at", ""), reverse=True)[:50])
|
|
1347
|
+
manifest["jobs"] = jobs
|
|
1348
|
+
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
1349
|
+
_json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
|
|
1350
|
+
|
|
1351
|
+
|
|
1352
|
+
def _document_action_hash(
|
|
1353
|
+
workspace: Path,
|
|
1354
|
+
source: str,
|
|
1355
|
+
locale: str,
|
|
1356
|
+
engine: str = "legacy",
|
|
1357
|
+
) -> str:
|
|
1358
|
+
with _manifest_lock(workspace):
|
|
1359
|
+
return _document_action_hash_unlocked(workspace, source, locale, engine)
|
|
1360
|
+
|
|
1361
|
+
|
|
1362
|
+
def _document_action_hash_unlocked(
|
|
1363
|
+
workspace: Path,
|
|
1364
|
+
source: str,
|
|
1365
|
+
locale: str,
|
|
1366
|
+
engine: str = "legacy",
|
|
1367
|
+
) -> str:
|
|
1368
|
+
internal = _managed_internal_directory(workspace)
|
|
1369
|
+
manifest = _read_manifest(workspace)
|
|
1370
|
+
source_hash = str(manifest.get("source", {}).get("sha256", ""))
|
|
1371
|
+
layout_hash = str((manifest.get("layout") or {}).get("fingerprint", ""))
|
|
1372
|
+
if not source_hash or (engine == "legacy" and not layout_hash):
|
|
1373
|
+
raise ValueError("document layout is not ready" if engine == "legacy" else "document source is not ready")
|
|
1374
|
+
selected_hash = source_hash
|
|
1375
|
+
if source == "translation":
|
|
1376
|
+
entries = manifest.get("translations")
|
|
1377
|
+
entry = entries.get(locale) if isinstance(entries, dict) else None
|
|
1378
|
+
relative = entry.get("path") if isinstance(entry, dict) else None
|
|
1379
|
+
translation = workspace / relative if isinstance(relative, str) else internal / "translations" / f"{locale}.json"
|
|
1380
|
+
if not translation.is_file() or translation.is_symlink() or not _is_within(
|
|
1381
|
+
translation.resolve(strict=True),
|
|
1382
|
+
(workspace.resolve(strict=True),),
|
|
1383
|
+
):
|
|
1384
|
+
raise ValueError("selected translation is not available")
|
|
1385
|
+
selected_hash = _sha256(translation)
|
|
1386
|
+
engine_version = ""
|
|
1387
|
+
if engine == "precise":
|
|
1388
|
+
from crabcode_gateway.document_engine import BABELDOC_VERSION
|
|
1389
|
+
|
|
1390
|
+
engine_version = BABELDOC_VERSION
|
|
1391
|
+
return hashlib.sha256(
|
|
1392
|
+
f"{source_hash}:{layout_hash if engine == 'legacy' else ''}:{source}:{selected_hash}:{engine}:{engine_version}".encode("utf-8")
|
|
1393
|
+
).hexdigest()
|
|
1394
|
+
|
|
1395
|
+
|
|
1396
|
+
def _finalize_precise_translation_unlocked(
|
|
1397
|
+
workspace: Path,
|
|
1398
|
+
internal: Path,
|
|
1399
|
+
job: Path,
|
|
1400
|
+
manifest: dict[str, Any],
|
|
1401
|
+
operation_id: str,
|
|
1402
|
+
locale: str,
|
|
1403
|
+
now: str,
|
|
1404
|
+
) -> tuple[int, int]:
|
|
1405
|
+
from crabcode_gateway.document_engine import BABELDOC_VERSION
|
|
1406
|
+
|
|
1407
|
+
staged_pdf = job / "translated.pdf"
|
|
1408
|
+
staged_content = job / "content.json"
|
|
1409
|
+
staged_diagnostics = job / "diagnostics.json"
|
|
1410
|
+
for path in (staged_pdf, staged_content, staged_diagnostics):
|
|
1411
|
+
if path.is_symlink() or not path.is_file():
|
|
1412
|
+
raise ValueError(f"precise translation is missing {path.name}")
|
|
1413
|
+
if staged_pdf.stat().st_size <= 8 or staged_pdf.stat().st_size > MAX_DOCUMENT_BYTES * 4:
|
|
1414
|
+
raise ValueError("precise translated PDF has an invalid size")
|
|
1415
|
+
if staged_content.stat().st_size > 50 * 1024 * 1024 or staged_diagnostics.stat().st_size > 5 * 1024 * 1024:
|
|
1416
|
+
raise ValueError("precise translation metadata is too large")
|
|
1417
|
+
with staged_pdf.open("rb") as handle:
|
|
1418
|
+
if b"%PDF-" not in handle.read(1024):
|
|
1419
|
+
raise ValueError("precise translation did not produce a valid PDF")
|
|
1420
|
+
diagnostics = _read_json_file(staged_diagnostics)
|
|
1421
|
+
if diagnostics is None:
|
|
1422
|
+
raise ValueError("precise translation diagnostics are invalid")
|
|
1423
|
+
content = _read_json_file(staged_content)
|
|
1424
|
+
content_pages = content.get("pages") if isinstance(content, dict) else None
|
|
1425
|
+
if (
|
|
1426
|
+
content is None
|
|
1427
|
+
or content.get("schema_version") != 1
|
|
1428
|
+
or content.get("engine") != "precise"
|
|
1429
|
+
or not isinstance(content_pages, list)
|
|
1430
|
+
):
|
|
1431
|
+
raise ValueError("precise translation content is invalid")
|
|
1432
|
+
source_sha256 = str(manifest.get("source", {}).get("sha256") or "")
|
|
1433
|
+
if diagnostics.get("source_sha256") != source_sha256:
|
|
1434
|
+
raise ValueError("precise translation was generated for a different source document")
|
|
1435
|
+
if diagnostics.get("engine_version") != BABELDOC_VERSION:
|
|
1436
|
+
raise ValueError("precise translation engine version does not match")
|
|
1437
|
+
|
|
1438
|
+
source_pdf = _document_pdf_path(workspace)
|
|
1439
|
+
try:
|
|
1440
|
+
source_reader = PdfReader(str(source_pdf))
|
|
1441
|
+
translated_reader = PdfReader(str(staged_pdf))
|
|
1442
|
+
except Exception as exc:
|
|
1443
|
+
raise ValueError("precise translated PDF cannot be opened") from exc
|
|
1444
|
+
if len(source_reader.pages) != len(translated_reader.pages):
|
|
1445
|
+
raise ValueError("precise translated PDF page count changed")
|
|
1446
|
+
if len(content_pages) != len(translated_reader.pages):
|
|
1447
|
+
raise ValueError("precise translation content page count changed")
|
|
1448
|
+
source_has_text = any((page.extract_text() or "").strip() for page in source_reader.pages)
|
|
1449
|
+
translated_has_text = any((page.extract_text() or "").strip() for page in translated_reader.pages)
|
|
1450
|
+
if source_has_text and not translated_has_text:
|
|
1451
|
+
raise ValueError("precise translated PDF has no selectable text")
|
|
1452
|
+
for source_page, translated_page in zip(source_reader.pages, translated_reader.pages, strict=True):
|
|
1453
|
+
for source_box, translated_box in (
|
|
1454
|
+
(source_page.mediabox, translated_page.mediabox),
|
|
1455
|
+
(source_page.cropbox, translated_page.cropbox),
|
|
1456
|
+
):
|
|
1457
|
+
source_size = (float(source_box.width), float(source_box.height))
|
|
1458
|
+
translated_size = (float(translated_box.width), float(translated_box.height))
|
|
1459
|
+
if any(abs(left - right) > 0.5 for left, right in zip(source_size, translated_size, strict=True)):
|
|
1460
|
+
raise ValueError("precise translated PDF page dimensions changed")
|
|
1461
|
+
if int(source_page.get("/Rotate", 0) or 0) % 360 != int(translated_page.get("/Rotate", 0) or 0) % 360:
|
|
1462
|
+
raise ValueError("precise translated PDF page rotation changed")
|
|
1463
|
+
|
|
1464
|
+
translations_dir = internal / "translations"
|
|
1465
|
+
if translations_dir.is_symlink():
|
|
1466
|
+
raise ValueError("managed translations directory is invalid")
|
|
1467
|
+
translations_dir.mkdir(parents=True, exist_ok=True)
|
|
1468
|
+
target = translations_dir / locale
|
|
1469
|
+
if target.is_symlink() or (target.exists() and not target.is_dir()):
|
|
1470
|
+
raise ValueError("precise translation output directory is invalid")
|
|
1471
|
+
stage = translations_dir / f".{locale}-stage-{uuid.uuid4().hex}"
|
|
1472
|
+
backup = translations_dir / f".{locale}-backup-{uuid.uuid4().hex}"
|
|
1473
|
+
stage.mkdir()
|
|
1474
|
+
replaced = False
|
|
1475
|
+
try:
|
|
1476
|
+
shutil.copy2(staged_pdf, stage / "translated.pdf")
|
|
1477
|
+
shutil.copy2(staged_content, stage / "content.json")
|
|
1478
|
+
shutil.copy2(staged_diagnostics, stage / "diagnostics.json")
|
|
1479
|
+
digest = _sha256(stage / "translated.pdf")
|
|
1480
|
+
warnings = diagnostics.get("warnings") if isinstance(diagnostics.get("warnings"), list) else []
|
|
1481
|
+
metadata = {
|
|
1482
|
+
"schema_version": 1,
|
|
1483
|
+
"engine": "precise",
|
|
1484
|
+
"locale": locale,
|
|
1485
|
+
"source_sha256": source_sha256,
|
|
1486
|
+
"pdf_sha256": digest,
|
|
1487
|
+
"page_count": len(translated_reader.pages),
|
|
1488
|
+
"engine_version": BABELDOC_VERSION,
|
|
1489
|
+
"warnings": [str(item)[:500] for item in warnings],
|
|
1490
|
+
"updated_at": now,
|
|
1491
|
+
}
|
|
1492
|
+
_json_write(stage / "metadata.json", metadata)
|
|
1493
|
+
if target.exists():
|
|
1494
|
+
replace_with_retry(target, backup)
|
|
1495
|
+
replaced = True
|
|
1496
|
+
replace_with_retry(stage, target)
|
|
1497
|
+
translations = manifest.get("translations")
|
|
1498
|
+
if not isinstance(translations, dict):
|
|
1499
|
+
translations = {}
|
|
1500
|
+
manifest["translations"] = translations
|
|
1501
|
+
translations[locale] = {
|
|
1502
|
+
"engine": "precise",
|
|
1503
|
+
"path": (target / "translated.pdf").relative_to(workspace).as_posix(),
|
|
1504
|
+
"content_path": (target / "content.json").relative_to(workspace).as_posix(),
|
|
1505
|
+
"diagnostics_path": (target / "diagnostics.json").relative_to(workspace).as_posix(),
|
|
1506
|
+
"source_sha256": source_sha256,
|
|
1507
|
+
"pdf_sha256": digest,
|
|
1508
|
+
"page_count": len(translated_reader.pages),
|
|
1509
|
+
"engine_version": BABELDOC_VERSION,
|
|
1510
|
+
"warnings": metadata["warnings"],
|
|
1511
|
+
"updated_at": now,
|
|
1512
|
+
}
|
|
1513
|
+
manifest["updated_at"] = now
|
|
1514
|
+
try:
|
|
1515
|
+
_json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
|
|
1516
|
+
except Exception:
|
|
1517
|
+
shutil.rmtree(target, ignore_errors=True)
|
|
1518
|
+
if replaced and backup.exists():
|
|
1519
|
+
replace_with_retry(backup, target)
|
|
1520
|
+
raise
|
|
1521
|
+
shutil.rmtree(backup, ignore_errors=True)
|
|
1522
|
+
except Exception:
|
|
1523
|
+
if replaced and backup.exists() and not target.exists():
|
|
1524
|
+
replace_with_retry(backup, target)
|
|
1525
|
+
raise
|
|
1526
|
+
finally:
|
|
1527
|
+
shutil.rmtree(stage, ignore_errors=True)
|
|
1528
|
+
_cleanup_document_job(workspace, operation_id)
|
|
1529
|
+
return 100, 100
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def _finalize_document_job_unlocked(
|
|
1533
|
+
workspace: Path,
|
|
1534
|
+
operation_id: str,
|
|
1535
|
+
action: str,
|
|
1536
|
+
locale: str,
|
|
1537
|
+
source: str,
|
|
1538
|
+
expected_document_hash: str = "",
|
|
1539
|
+
language: str = "",
|
|
1540
|
+
engine: str = "legacy",
|
|
1541
|
+
) -> tuple[int, int]:
|
|
1542
|
+
"""Validate an Agent's staged artifact and publish it atomically."""
|
|
1543
|
+
if not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", locale):
|
|
1544
|
+
raise ValueError("invalid document locale")
|
|
1545
|
+
if language and not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", language):
|
|
1546
|
+
raise ValueError("invalid Blog language")
|
|
1547
|
+
if engine not in {"legacy", "precise"}:
|
|
1548
|
+
raise ValueError("invalid document translation engine")
|
|
1549
|
+
if expected_document_hash and _document_action_hash(workspace, source, locale, engine) != expected_document_hash:
|
|
1550
|
+
raise ValueError("source document changed while the operation was running")
|
|
1551
|
+
internal = _managed_internal_directory(workspace)
|
|
1552
|
+
manifest = _read_manifest(workspace)
|
|
1553
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
1554
|
+
if action == "generate_blog":
|
|
1555
|
+
# Finalization may be retried after the first call has atomically moved
|
|
1556
|
+
# blog.md and removed the staging directory. Treat that exact publish
|
|
1557
|
+
# as idempotently complete instead of reporting a missing artifact.
|
|
1558
|
+
published = manifest.get("blog")
|
|
1559
|
+
expected_language = locale if language == "source" and source == "translation" else language
|
|
1560
|
+
expected_language = expected_language or (locale if source == "translation" else "source")
|
|
1561
|
+
output = workspace / "blog.md"
|
|
1562
|
+
if (
|
|
1563
|
+
isinstance(published, dict)
|
|
1564
|
+
and published.get("operation_id") == operation_id
|
|
1565
|
+
and published.get("source") == source
|
|
1566
|
+
and output.is_file()
|
|
1567
|
+
and not output.is_symlink()
|
|
1568
|
+
):
|
|
1569
|
+
if published.get("language") != expected_language:
|
|
1570
|
+
published["language"] = expected_language
|
|
1571
|
+
manifest["updated_at"] = now
|
|
1572
|
+
_json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
|
|
1573
|
+
return 1, 1
|
|
1574
|
+
job = _document_job_directory(workspace, operation_id)
|
|
1575
|
+
if job.is_symlink():
|
|
1576
|
+
raise ValueError("document job directory cannot be a symlink")
|
|
1577
|
+
if action == "translate" and engine == "precise":
|
|
1578
|
+
return _finalize_precise_translation_unlocked(
|
|
1579
|
+
workspace,
|
|
1580
|
+
internal,
|
|
1581
|
+
job,
|
|
1582
|
+
manifest,
|
|
1583
|
+
operation_id,
|
|
1584
|
+
locale,
|
|
1585
|
+
now,
|
|
1586
|
+
)
|
|
1587
|
+
blog_backup: bytes | None = None
|
|
1588
|
+
blog_existed = False
|
|
1589
|
+
translation_output: Path | None = None
|
|
1590
|
+
translation_backup: bytes | None = None
|
|
1591
|
+
translation_existed = False
|
|
1592
|
+
if action == "translate":
|
|
1593
|
+
total = _document_job_total(workspace, action, engine)
|
|
1594
|
+
try:
|
|
1595
|
+
layout_path = internal / "layout.json"
|
|
1596
|
+
staged_path = job / "translation.json"
|
|
1597
|
+
if layout_path.is_symlink() or staged_path.is_symlink():
|
|
1598
|
+
raise ValueError("document translation inputs cannot be symlinks")
|
|
1599
|
+
staged = json.loads(staged_path.read_text(encoding="utf-8"))
|
|
1600
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
1601
|
+
raise ValueError("Agent did not produce valid translation JSON") from exc
|
|
1602
|
+
expected_ids = [
|
|
1603
|
+
block_id
|
|
1604
|
+
for page in _translation_source_pages(workspace)
|
|
1605
|
+
for block_id, _ in page
|
|
1606
|
+
]
|
|
1607
|
+
blocks = staged.get("blocks") if isinstance(staged, dict) else None
|
|
1608
|
+
if not isinstance(blocks, list):
|
|
1609
|
+
raise ValueError("translation JSON is missing blocks")
|
|
1610
|
+
if staged.get("locale") not in {None, locale}:
|
|
1611
|
+
raise ValueError("translation job locale does not match")
|
|
1612
|
+
translated: dict[str, str] = {}
|
|
1613
|
+
for block in blocks:
|
|
1614
|
+
if not isinstance(block, dict) or not isinstance(block.get("id"), str) or not isinstance(block.get("translated_text"), str):
|
|
1615
|
+
raise ValueError("translation blocks must contain id and translated_text")
|
|
1616
|
+
block_id = block["id"]
|
|
1617
|
+
if not block["translated_text"].strip():
|
|
1618
|
+
raise ValueError(f"translation is empty for block id: {block_id}")
|
|
1619
|
+
if block_id in translated:
|
|
1620
|
+
raise ValueError(f"translation contains duplicate block id: {block_id}")
|
|
1621
|
+
translated[block_id] = block["translated_text"]
|
|
1622
|
+
if len(expected_ids) != total or set(translated) != set(expected_ids):
|
|
1623
|
+
missing = len(set(expected_ids) - set(translated))
|
|
1624
|
+
extra = len(set(translated) - set(expected_ids))
|
|
1625
|
+
raise ValueError(f"translation block ids do not match layout (missing={missing}, extra={extra})")
|
|
1626
|
+
source_sha256 = str(manifest.get("source", {}).get("sha256", ""))
|
|
1627
|
+
if staged.get("source_sha256") != source_sha256:
|
|
1628
|
+
raise ValueError("translation was generated for a different source document")
|
|
1629
|
+
layout_fingerprint = (manifest.get("layout") or {}).get("fingerprint", "")
|
|
1630
|
+
if staged.get("layout_fingerprint") != layout_fingerprint:
|
|
1631
|
+
raise ValueError("translation was generated for a different document layout")
|
|
1632
|
+
canonical = {
|
|
1633
|
+
"engine": "legacy",
|
|
1634
|
+
"locale": locale,
|
|
1635
|
+
"source_sha256": source_sha256,
|
|
1636
|
+
"layout_fingerprint": layout_fingerprint,
|
|
1637
|
+
"blocks": [{"id": block_id, "translated_text": translated[block_id]} for block_id in expected_ids],
|
|
1638
|
+
}
|
|
1639
|
+
translations_dir = internal / "translations"
|
|
1640
|
+
if translations_dir.is_symlink():
|
|
1641
|
+
raise ValueError("managed translations directory is invalid")
|
|
1642
|
+
output = translations_dir / f"{locale}.json"
|
|
1643
|
+
if output.is_symlink():
|
|
1644
|
+
raise ValueError("translation output path is invalid")
|
|
1645
|
+
translation_output = output
|
|
1646
|
+
if output.exists():
|
|
1647
|
+
translation_backup = output.read_bytes()
|
|
1648
|
+
translation_existed = True
|
|
1649
|
+
_json_write(output, canonical)
|
|
1650
|
+
translations = manifest.get("translations")
|
|
1651
|
+
if not isinstance(translations, dict):
|
|
1652
|
+
translations = {}
|
|
1653
|
+
manifest["translations"] = translations
|
|
1654
|
+
translations[locale] = {
|
|
1655
|
+
"engine": "legacy",
|
|
1656
|
+
"path": output.relative_to(workspace).as_posix(),
|
|
1657
|
+
"source_sha256": source_sha256,
|
|
1658
|
+
"blocks": total,
|
|
1659
|
+
"updated_at": now,
|
|
1660
|
+
}
|
|
1661
|
+
current = total
|
|
1662
|
+
elif action == "generate_blog":
|
|
1663
|
+
staged_blog = job / "blog.md"
|
|
1664
|
+
try:
|
|
1665
|
+
if staged_blog.is_symlink() or staged_blog.stat().st_size > 10_000_000:
|
|
1666
|
+
raise ValueError("generated Blog is invalid or too large")
|
|
1667
|
+
markdown = staged_blog.read_text(encoding="utf-8")
|
|
1668
|
+
except OSError as exc:
|
|
1669
|
+
raise ValueError("Agent did not produce blog.md") from exc
|
|
1670
|
+
if not markdown.strip():
|
|
1671
|
+
raise ValueError("generated Blog is empty")
|
|
1672
|
+
output = workspace / "blog.md"
|
|
1673
|
+
if output.is_symlink():
|
|
1674
|
+
raise ValueError("Blog path is invalid")
|
|
1675
|
+
if output.exists():
|
|
1676
|
+
blog_backup = output.read_bytes()
|
|
1677
|
+
blog_existed = True
|
|
1678
|
+
replace_with_retry(staged_blog, output)
|
|
1679
|
+
revision = _blog_revision(markdown)
|
|
1680
|
+
blog_language = locale if language == "source" and source == "translation" else language
|
|
1681
|
+
manifest["blog"] = {
|
|
1682
|
+
"path": "blog.md",
|
|
1683
|
+
"revision": revision,
|
|
1684
|
+
"language": blog_language or (locale if source == "translation" else "source"),
|
|
1685
|
+
"source": source,
|
|
1686
|
+
"operation_id": operation_id,
|
|
1687
|
+
"updated_at": now,
|
|
1688
|
+
}
|
|
1689
|
+
current = total = 1
|
|
1690
|
+
else:
|
|
1691
|
+
raise ValueError("unknown document action")
|
|
1692
|
+
manifest["updated_at"] = now
|
|
1693
|
+
try:
|
|
1694
|
+
_json_write(workspace / MANIFEST_RELATIVE_PATH, manifest)
|
|
1695
|
+
except Exception:
|
|
1696
|
+
if action == "generate_blog":
|
|
1697
|
+
if blog_existed and blog_backup is not None:
|
|
1698
|
+
_atomic_write_bytes(workspace / "blog.md", blog_backup)
|
|
1699
|
+
else:
|
|
1700
|
+
(workspace / "blog.md").unlink(missing_ok=True)
|
|
1701
|
+
if action == "translate" and translation_output is not None:
|
|
1702
|
+
if translation_existed and translation_backup is not None:
|
|
1703
|
+
_atomic_write_bytes(translation_output, translation_backup)
|
|
1704
|
+
else:
|
|
1705
|
+
translation_output.unlink(missing_ok=True)
|
|
1706
|
+
raise
|
|
1707
|
+
_cleanup_document_job(workspace, operation_id)
|
|
1708
|
+
return current, total
|
|
1709
|
+
|
|
1710
|
+
|
|
1711
|
+
def _finalize_document_job(
|
|
1712
|
+
workspace: Path,
|
|
1713
|
+
operation_id: str,
|
|
1714
|
+
action: str,
|
|
1715
|
+
locale: str,
|
|
1716
|
+
source: str,
|
|
1717
|
+
expected_document_hash: str = "",
|
|
1718
|
+
language: str = "",
|
|
1719
|
+
engine: str = "legacy",
|
|
1720
|
+
) -> tuple[int, int]:
|
|
1721
|
+
with _manifest_lock(workspace):
|
|
1722
|
+
return _finalize_document_job_unlocked(
|
|
1723
|
+
workspace,
|
|
1724
|
+
operation_id,
|
|
1725
|
+
action,
|
|
1726
|
+
locale,
|
|
1727
|
+
source,
|
|
1728
|
+
expected_document_hash,
|
|
1729
|
+
language,
|
|
1730
|
+
engine,
|
|
1731
|
+
)
|
|
1732
|
+
|
|
1733
|
+
|
|
1734
|
+
@router.get("/blog")
|
|
1735
|
+
async def read_document_blog(workspace: str, request: Request, response: Response) -> dict[str, Any]:
|
|
1736
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
1737
|
+
with _manifest_lock(root):
|
|
1738
|
+
path = root / "blog.md"
|
|
1739
|
+
if path.is_symlink():
|
|
1740
|
+
raise HTTPException(status_code=403, detail="Blog path is invalid")
|
|
1741
|
+
if path.exists() and not path.is_file():
|
|
1742
|
+
raise HTTPException(status_code=403, detail="Blog path is invalid")
|
|
1743
|
+
try:
|
|
1744
|
+
markdown = path.read_text(encoding="utf-8")
|
|
1745
|
+
except FileNotFoundError as exc:
|
|
1746
|
+
raise HTTPException(status_code=404, detail="Blog not found") from exc
|
|
1747
|
+
revision = _blog_revision(markdown)
|
|
1748
|
+
response.headers["ETag"] = f'"{revision}"'
|
|
1749
|
+
manifest = _read_manifest(root)
|
|
1750
|
+
return {"markdown": markdown, "revision": revision, "language": (manifest.get("blog") or {}).get("language", "")}
|
|
1751
|
+
|
|
1752
|
+
|
|
1753
|
+
@router.put("/blog")
|
|
1754
|
+
async def write_document_blog(req: DocumentBlogWriteRequest, workspace: str, request: Request) -> dict[str, Any]:
|
|
1755
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
1756
|
+
with _manifest_lock(root):
|
|
1757
|
+
path = root / "blog.md"
|
|
1758
|
+
if path.is_symlink():
|
|
1759
|
+
raise HTTPException(status_code=403, detail="Blog path is invalid")
|
|
1760
|
+
current = path.read_text(encoding="utf-8") if path.exists() else ""
|
|
1761
|
+
current_revision = _blog_revision(current) if path.exists() else None
|
|
1762
|
+
if req.revision != current_revision:
|
|
1763
|
+
raise HTTPException(status_code=409, detail="Blog changed since it was opened")
|
|
1764
|
+
temporary = root / f".blog-{uuid.uuid4().hex}.md"
|
|
1765
|
+
try:
|
|
1766
|
+
temporary.write_text(req.markdown, encoding="utf-8")
|
|
1767
|
+
replace_with_retry(temporary, path)
|
|
1768
|
+
finally:
|
|
1769
|
+
temporary.unlink(missing_ok=True)
|
|
1770
|
+
revision = _blog_revision(req.markdown)
|
|
1771
|
+
manifest = _read_manifest(root)
|
|
1772
|
+
manifest["blog"] = {"path": "blog.md", "revision": revision, "language": req.language}
|
|
1773
|
+
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
1774
|
+
_json_write(root / MANIFEST_RELATIVE_PATH, manifest)
|
|
1775
|
+
return {"markdown": req.markdown, "revision": revision, "language": req.language}
|
|
1776
|
+
|
|
1777
|
+
|
|
1778
|
+
@router.delete("/blog")
|
|
1779
|
+
async def delete_document_blog(
|
|
1780
|
+
workspace: str,
|
|
1781
|
+
request: Request,
|
|
1782
|
+
) -> dict[str, Any]:
|
|
1783
|
+
root = _validated_workspace(workspace, _workspace_roots(request))
|
|
1784
|
+
with _manifest_lock(root):
|
|
1785
|
+
path = root / "blog.md"
|
|
1786
|
+
if path.is_symlink():
|
|
1787
|
+
raise HTTPException(status_code=403, detail="Blog path is invalid")
|
|
1788
|
+
if path.exists() and not path.is_file():
|
|
1789
|
+
raise HTTPException(status_code=403, detail="Blog path is invalid")
|
|
1790
|
+
manifest = _read_manifest(root)
|
|
1791
|
+
entry = manifest.get("blog")
|
|
1792
|
+
blog_entry = entry if isinstance(entry, dict) else {}
|
|
1793
|
+
assets = root / "blog-assets"
|
|
1794
|
+
if assets.is_symlink() or (assets.exists() and not assets.is_dir()):
|
|
1795
|
+
raise HTTPException(status_code=403, detail="Blog assets path is invalid")
|
|
1796
|
+
removed = path.exists()
|
|
1797
|
+
if removed:
|
|
1798
|
+
path.unlink()
|
|
1799
|
+
removed_assets = assets.is_dir()
|
|
1800
|
+
if removed_assets:
|
|
1801
|
+
shutil.rmtree(assets)
|
|
1802
|
+
if entry is not None or removed or removed_assets:
|
|
1803
|
+
manifest["blog"] = None
|
|
1804
|
+
manifest["updated_at"] = datetime.now(timezone.utc).isoformat()
|
|
1805
|
+
_json_write(root / MANIFEST_RELATIVE_PATH, manifest)
|
|
1806
|
+
return {
|
|
1807
|
+
"removed": removed or removed_assets,
|
|
1808
|
+
"removed_assets": removed_assets,
|
|
1809
|
+
"language": blog_entry.get("language", ""),
|
|
1810
|
+
}
|