jscdocs 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jscdocs/__init__.py +8 -0
- jscdocs/build.py +65 -0
- jscdocs/cli.py +190 -0
- jscdocs/core/__init__.py +1 -0
- jscdocs/core/agent_cli.py +131 -0
- jscdocs/core/agent_config.py +370 -0
- jscdocs/core/config.py +31 -0
- jscdocs/core/crawl.py +170 -0
- jscdocs/core/evaluate.py +97 -0
- jscdocs/core/extract.py +403 -0
- jscdocs/core/fetch.py +100 -0
- jscdocs/core/init.py +382 -0
- jscdocs/core/inventory.py +31 -0
- jscdocs/core/lock.py +125 -0
- jscdocs/core/published.py +125 -0
- jscdocs/core/refresh.py +107 -0
- jscdocs/core/search.py +294 -0
- jscdocs/core/sites.py +55 -0
- jscdocs/core/store.py +233 -0
- jscdocs/core/sync.py +47 -0
- jscdocs/mcp/__init__.py +1 -0
- jscdocs/mcp/server.py +86 -0
- jscdocs/mcp/tools.py +120 -0
- jscdocs-0.2.0.dist-info/METADATA +127 -0
- jscdocs-0.2.0.dist-info/RECORD +28 -0
- jscdocs-0.2.0.dist-info/WHEEL +4 -0
- jscdocs-0.2.0.dist-info/entry_points.txt +3 -0
- jscdocs-0.2.0.dist-info/licenses/LICENSE +21 -0
jscdocs/core/init.py
ADDED
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
"""Plan and apply `jscdocs init`: register the jscdocs MCP server with a coding agent (Claude Code, Codex, pi or
|
|
2
|
+
OpenCode), through the agent's own command when it can write the whole entry, else by editing its configuration file.
|
|
3
|
+
Init never starts the server or fetches anything."""
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import shutil
|
|
8
|
+
import stat
|
|
9
|
+
import sys
|
|
10
|
+
import tempfile
|
|
11
|
+
from dataclasses import dataclass, replace
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
from .. import __version__
|
|
15
|
+
from . import agent_cli, agent_config
|
|
16
|
+
from .config import index_path
|
|
17
|
+
from .agent_config import NAME, Agent, InitError, ServerSpec, absolute, strict_json
|
|
18
|
+
|
|
19
|
+
FORMAT = 1
|
|
20
|
+
SERVER = "jscdocs-mcp"
|
|
21
|
+
# Codex starts servers with only a few variables of its own environment, so these are passed on by name: the server
|
|
22
|
+
# then uses the index, the published copy and the refresh interval that the jscdocs CLI uses.
|
|
23
|
+
FORWARD_ENV = ("JSCDOCS_INDEX", "JSCDOCS_INDEX_URL", "JSCDOCS_REFRESH_HOURS", "XDG_CACHE_HOME")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(frozen=True)
|
|
27
|
+
class InitRequest:
|
|
28
|
+
agent: Agent
|
|
29
|
+
index: Path | None = None # the server's --index; default: $JSCDOCS_INDEX or the cache, as for the CLI
|
|
30
|
+
refresh: bool = True # False adds --no-refresh
|
|
31
|
+
force: bool = False
|
|
32
|
+
project_root: Path | None = None
|
|
33
|
+
|
|
34
|
+
@property
|
|
35
|
+
def scope(self) -> str:
|
|
36
|
+
return "project" if self.project_root is not None else "user"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Snapshot:
|
|
41
|
+
path: Path
|
|
42
|
+
target: Path
|
|
43
|
+
data: bytes | None
|
|
44
|
+
mode: int
|
|
45
|
+
private: bool = True # project files are shared: new ones and their directories keep the umask's modes
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True)
|
|
49
|
+
class FileChange:
|
|
50
|
+
before: Snapshot
|
|
51
|
+
after: bytes
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True)
|
|
55
|
+
class Action:
|
|
56
|
+
kind: str
|
|
57
|
+
name: str
|
|
58
|
+
path: Path
|
|
59
|
+
action: str
|
|
60
|
+
|
|
61
|
+
def as_dict(self) -> dict:
|
|
62
|
+
return {"kind": self.kind, "name": self.name, "path": str(self.path), "action": self.action}
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass(frozen=True)
|
|
66
|
+
class InitPlan:
|
|
67
|
+
request: InitRequest
|
|
68
|
+
servers: tuple[ServerSpec, ...]
|
|
69
|
+
config: FileChange
|
|
70
|
+
manifest: FileChange
|
|
71
|
+
actions: tuple[Action, ...]
|
|
72
|
+
conflicts: tuple[str, ...]
|
|
73
|
+
notes: tuple[str, ...]
|
|
74
|
+
selected: agent_config.ServerChange
|
|
75
|
+
registration: agent_cli.RegistrationPlan
|
|
76
|
+
|
|
77
|
+
def as_dict(self) -> dict:
|
|
78
|
+
return {"agent": self.request.agent, "scope": self.request.scope, "dry_run": True,
|
|
79
|
+
"project_root": str(self.request.project_root) if self.request.project_root is not None else None,
|
|
80
|
+
"manifest": str(self.manifest.before.path),
|
|
81
|
+
"ok": not self.conflicts, "config": str(self.config.before.path),
|
|
82
|
+
"registration": self.registration.as_dict(),
|
|
83
|
+
"servers": [{"name": s.name, "command": s.command, "args": list(s.args),
|
|
84
|
+
"forward_env": list(s.forward_env)} for s in self.servers],
|
|
85
|
+
"actions": [a.as_dict() for a in self.actions], "conflicts": list(self.conflicts),
|
|
86
|
+
"notes": list(self.notes)}
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class InitResult:
|
|
91
|
+
plan: InitPlan
|
|
92
|
+
actions: tuple[Action, ...]
|
|
93
|
+
backups: tuple[Path, ...] = ()
|
|
94
|
+
error: str | None = None
|
|
95
|
+
interrupted: bool = False
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def ok(self) -> bool:
|
|
99
|
+
return not self.plan.conflicts and self.error is None
|
|
100
|
+
|
|
101
|
+
def as_dict(self) -> dict:
|
|
102
|
+
return self.plan.as_dict() | {"dry_run": False, "ok": self.ok,
|
|
103
|
+
"actions": [a.as_dict() for a in self.actions],
|
|
104
|
+
"backups": [str(p) for p in self.backups], "error": self.error,
|
|
105
|
+
"interrupted": self.interrupted}
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _parents(path: Path) -> None:
|
|
109
|
+
for parent in path.parents:
|
|
110
|
+
if parent.is_symlink() and not parent.exists():
|
|
111
|
+
raise InitError(f"{parent}: dangling directory symlink")
|
|
112
|
+
if parent.exists() and not parent.is_dir():
|
|
113
|
+
raise InitError(f"{parent}: expected a directory")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _file_mode(private: bool) -> int:
|
|
117
|
+
"""Mode for a new file: owner-only, or the umask's default for files others share."""
|
|
118
|
+
if private:
|
|
119
|
+
return 0o600
|
|
120
|
+
mask = os.umask(0o077)
|
|
121
|
+
os.umask(mask)
|
|
122
|
+
return 0o666 & ~mask
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def snapshot(path: Path, *, private: bool = True) -> Snapshot:
|
|
126
|
+
path = absolute(path)
|
|
127
|
+
try:
|
|
128
|
+
_parents(path)
|
|
129
|
+
target = path.resolve()
|
|
130
|
+
if path.is_symlink() and not path.exists():
|
|
131
|
+
raise InitError(f"{path}: dangling configuration symlink")
|
|
132
|
+
if target.exists() and not target.is_file():
|
|
133
|
+
raise InitError(f"{path}: expected a regular file")
|
|
134
|
+
data = target.read_bytes() if target.exists() else None
|
|
135
|
+
mode = stat.S_IMODE(target.stat().st_mode) if data is not None else _file_mode(private)
|
|
136
|
+
return Snapshot(path, target, data, mode, private)
|
|
137
|
+
except (OSError, RuntimeError):
|
|
138
|
+
raise InitError(f"{path}: cannot read configuration") from None
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def launcher() -> tuple[str, tuple[str, ...]]:
|
|
142
|
+
"""This installation's server: the jscdocs-mcp next to its Python, else the one on PATH, else its Python with -m."""
|
|
143
|
+
sibling = Path(sys.executable).absolute().parent / SERVER
|
|
144
|
+
if sibling.is_file() and os.access(sibling, os.X_OK):
|
|
145
|
+
return str(sibling), ()
|
|
146
|
+
if found := shutil.which(SERVER):
|
|
147
|
+
return str(absolute(found)), ()
|
|
148
|
+
# -P: `-m` would put the client's working directory (a project) first on sys.path, where a project's own mcp/
|
|
149
|
+
# would shadow jscdocs's dependencies.
|
|
150
|
+
return str(Path(sys.executable).absolute()), ("-P", "-m", "jscdocs.mcp.server")
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _manifest(data: bytes | None, path: Path, config_key: str) -> dict:
|
|
154
|
+
try:
|
|
155
|
+
document = strict_json(data.decode()) if data is not None else {"format": FORMAT, "configs": {}}
|
|
156
|
+
if type(document.get("format")) is not int or document["format"] != FORMAT or not isinstance(document.get("configs"), dict):
|
|
157
|
+
raise ValueError
|
|
158
|
+
entries = document["configs"].get(config_key, {})
|
|
159
|
+
if not isinstance(entries, dict):
|
|
160
|
+
raise ValueError
|
|
161
|
+
fields = entries.get(NAME, {})
|
|
162
|
+
if not isinstance(fields, dict) or set(fields) - {"command", "args", "env_vars", "type", "enabled", "url"}:
|
|
163
|
+
raise ValueError
|
|
164
|
+
return document
|
|
165
|
+
except (ValueError, UnicodeError):
|
|
166
|
+
raise InitError(f"{path}: invalid or unsupported initializer manifest") from None
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def normalize_request(request: InitRequest) -> InitRequest:
|
|
170
|
+
if request.project_root is None:
|
|
171
|
+
return request
|
|
172
|
+
try:
|
|
173
|
+
root = request.project_root.expanduser().resolve(strict=True)
|
|
174
|
+
except (OSError, RuntimeError):
|
|
175
|
+
raise ValueError("project directory must exist") from None
|
|
176
|
+
if not root.is_dir():
|
|
177
|
+
raise ValueError("project directory must be a directory")
|
|
178
|
+
return replace(request, project_root=root)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def manifest_path(request: InitRequest) -> Path:
|
|
182
|
+
"""Where init records the entry it wrote, to tell its own entry from one written by hand."""
|
|
183
|
+
if request.project_root is not None:
|
|
184
|
+
base = request.project_root / ".jscdocs"
|
|
185
|
+
else:
|
|
186
|
+
base = Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local" / "state").expanduser() / "jscdocs"
|
|
187
|
+
return base / "init" / f"{request.agent}.json"
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def require_project_path(path: Path, root: Path) -> Path:
|
|
191
|
+
_parents(path)
|
|
192
|
+
try:
|
|
193
|
+
effective = path.resolve()
|
|
194
|
+
effective.relative_to(root)
|
|
195
|
+
except (OSError, RuntimeError, ValueError):
|
|
196
|
+
raise InitError(f"{path}: project installation destination escapes the project") from None
|
|
197
|
+
if path.is_symlink() and not path.exists():
|
|
198
|
+
raise InitError(f"{path}: dangling project destination symlink")
|
|
199
|
+
if effective.exists() and not (effective.is_file() or effective.is_dir()):
|
|
200
|
+
raise InitError(f"{path}: unsupported file type")
|
|
201
|
+
return effective
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _shared_notes() -> list[str]:
|
|
205
|
+
"""Warnings for a project entry, which runs the jscdocs-mcp on PATH."""
|
|
206
|
+
own = launcher()[0]
|
|
207
|
+
found = shutil.which(SERVER)
|
|
208
|
+
if found is None:
|
|
209
|
+
return [f"{SERVER} is not on PATH, so the client cannot start the server: "
|
|
210
|
+
"start the client with jscdocs's environment active."]
|
|
211
|
+
if Path(found).resolve() != Path(own).resolve():
|
|
212
|
+
return [f"The client will start the {SERVER} on PATH ({found}), not this installation's ({own})."]
|
|
213
|
+
return []
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def plan_init(request: InitRequest) -> InitPlan:
|
|
217
|
+
"""What init would change, read-only: it creates no file and starts no process. Raises ValueError for an invalid
|
|
218
|
+
request and InitError for a configuration it cannot read or must not edit."""
|
|
219
|
+
request = normalize_request(request)
|
|
220
|
+
if request.agent not in agent_config.AGENTS:
|
|
221
|
+
raise ValueError(f"unknown agent: {request.agent}")
|
|
222
|
+
shared = request.project_root is not None
|
|
223
|
+
if shared and request.index is not None:
|
|
224
|
+
raise ValueError("--project writes a file others share, so it takes no --index: "
|
|
225
|
+
"each user's JSCDOCS_INDEX, or the default, applies")
|
|
226
|
+
index = absolute(request.index) if request.index is not None else None
|
|
227
|
+
if index is not None and "\0" in str(index):
|
|
228
|
+
raise ValueError("--index must not contain NUL")
|
|
229
|
+
reload = ("OpenCode reloads registrations by itself (/reload forces it)." if request.agent == "opencode"
|
|
230
|
+
else "Restart the client (or /reload in pi) to load the registration.")
|
|
231
|
+
notes = [reload, "User/managed settings, project overrides, and client trust settings may affect activation."]
|
|
232
|
+
if not (index or index_path()).exists():
|
|
233
|
+
notes.append("There is no index yet: run jscdocs sync to download it, or let the server download it when it "
|
|
234
|
+
"starts.")
|
|
235
|
+
client = agent_cli.find_client(request.agent)
|
|
236
|
+
adapter = agent_config.discover(request.agent, project_root=request.project_root)
|
|
237
|
+
metadata = manifest_path(request)
|
|
238
|
+
if request.project_root is not None:
|
|
239
|
+
for path in (adapter.path, metadata):
|
|
240
|
+
require_project_path(path, request.project_root)
|
|
241
|
+
config = snapshot(adapter.path, private=not shared)
|
|
242
|
+
manifest = snapshot(metadata, private=not shared)
|
|
243
|
+
document = adapter.parse(config.data)
|
|
244
|
+
for other in adapter.registered_elsewhere(NAME):
|
|
245
|
+
raise InitError(f"{other}: also registers {NAME}, and {request.agent} reads it together with "
|
|
246
|
+
f"{adapter.path}; keep the entry in one file, then rerun init")
|
|
247
|
+
# A project's manifest is keyed relative to the project, so a moved or copied project still owns its entry.
|
|
248
|
+
key = str(config.target.relative_to(request.project_root)) if shared else str(config.target)
|
|
249
|
+
history = _manifest(manifest.data, manifest.path, key)
|
|
250
|
+
owned = history["configs"].setdefault(key, {})
|
|
251
|
+
servers, actions, conflicts = [], [], []
|
|
252
|
+
config_dirty = False
|
|
253
|
+
# A project entry is the same for everyone: PATH finds the server, and each user's environment the index.
|
|
254
|
+
command, prefix = (SERVER, ()) if shared else launcher()
|
|
255
|
+
args = list(prefix)
|
|
256
|
+
if index is not None:
|
|
257
|
+
args.extend(["--index", str(index)])
|
|
258
|
+
if not request.refresh:
|
|
259
|
+
args.append("--no-refresh")
|
|
260
|
+
if shared:
|
|
261
|
+
notes.extend(_shared_notes())
|
|
262
|
+
spec = ServerSpec(NAME, command, tuple(args), FORWARD_ENV if request.agent == "codex" else ())
|
|
263
|
+
servers.append(spec)
|
|
264
|
+
selected = agent_config.reconcile(adapter, spec, adapter.server(document, spec.name), owned.get(spec.name),
|
|
265
|
+
force=request.force)
|
|
266
|
+
if selected.conflict:
|
|
267
|
+
conflicts.append(selected.conflict)
|
|
268
|
+
else:
|
|
269
|
+
owned[spec.name] = selected.desired
|
|
270
|
+
if selected.action in ("added", "updated"):
|
|
271
|
+
adapter.put(document, spec.name, selected.replacement)
|
|
272
|
+
config_dirty = True
|
|
273
|
+
actions.append(Action("server", spec.name, adapter.path, selected.action))
|
|
274
|
+
registration = agent_cli.plan_registration(adapter, client, selected, request.project_root)
|
|
275
|
+
if registration.backend == "file" and config_dirty and not adapter.lossless(config.data):
|
|
276
|
+
conflicts.append(f"{spec.name}: {adapter.path} has comments or trailing commas that writing the file directly "
|
|
277
|
+
f"would drop; preserved (add the entry with the {request.agent} CLI, or remove them)")
|
|
278
|
+
if registration.reason:
|
|
279
|
+
notes.append(f"Direct-file fallback: {registration.reason}. Avoid concurrent client edits during setup.")
|
|
280
|
+
if any(command.operation == "remove" for command in registration.commands):
|
|
281
|
+
notes.append(f"Claude replacement removes {NAME} before adding it again; a failed add can leave it missing.")
|
|
282
|
+
history["jscdocs"] = __version__
|
|
283
|
+
manifest_after = (json.dumps(history, indent=2, sort_keys=True) + "\n").encode()
|
|
284
|
+
return InitPlan(request, tuple(servers), FileChange(config, adapter.render(document) if config_dirty else config.data or b""),
|
|
285
|
+
FileChange(manifest, manifest_after), tuple(actions), tuple(conflicts), tuple(notes), selected,
|
|
286
|
+
registration)
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _mkdir(path: Path, mode: int) -> None:
|
|
290
|
+
if not path.exists():
|
|
291
|
+
_mkdir(path.parent, mode)
|
|
292
|
+
path.mkdir(mode=mode, exist_ok=True)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def _backup(before: Snapshot) -> Path:
|
|
296
|
+
path = before.target.with_name(f"{before.target.name}.jscdocs-backup.bak")
|
|
297
|
+
_write(FileChange(Snapshot(path, path, None, 0o600), before.data))
|
|
298
|
+
return path
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def _write(change: FileChange) -> None:
|
|
302
|
+
before = change.before
|
|
303
|
+
_mkdir(before.target.parent, 0o700 if before.private else 0o777)
|
|
304
|
+
fd, name = tempfile.mkstemp(prefix=f".{before.target.name}.jscdocs-", dir=before.target.parent)
|
|
305
|
+
staging = Path(name)
|
|
306
|
+
try:
|
|
307
|
+
with os.fdopen(fd, "wb") as out:
|
|
308
|
+
os.fchmod(out.fileno(), before.mode)
|
|
309
|
+
out.write(change.after)
|
|
310
|
+
out.flush()
|
|
311
|
+
os.fsync(out.fileno())
|
|
312
|
+
os.replace(staging, before.target)
|
|
313
|
+
finally:
|
|
314
|
+
staging.unlink(missing_ok=True)
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def _unchanged(before: Snapshot) -> None:
|
|
318
|
+
if snapshot(before.path).data != before.data:
|
|
319
|
+
raise InitError(f"{before.path}: changed since init planned the change; rerun init")
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def apply_init(plan: InitPlan) -> InitResult:
|
|
323
|
+
"""Make the planned change. Failures after the checks come back in the result, with what was done and what not."""
|
|
324
|
+
if plan.conflicts:
|
|
325
|
+
return InitResult(plan, tuple(Action(a.kind, a.name, a.path, "preserved") for a in plan.actions), error="resolve conflicts before applying")
|
|
326
|
+
completed, backups = [], []
|
|
327
|
+
server_state = "preserved"
|
|
328
|
+
writes_config = plan.registration.backend == "file" and plan.config.after != (plan.config.before.data or b"")
|
|
329
|
+
writes_manifest = plan.manifest.after != plan.manifest.before.data
|
|
330
|
+
try:
|
|
331
|
+
if plan.request.project_root is not None:
|
|
332
|
+
root = plan.request.project_root
|
|
333
|
+
if not root.is_dir():
|
|
334
|
+
raise InitError(f"{root}: project directory must exist")
|
|
335
|
+
paths = [plan.config.before.path, plan.config.before.target,
|
|
336
|
+
plan.manifest.before.path, plan.manifest.before.target]
|
|
337
|
+
for path in paths:
|
|
338
|
+
require_project_path(path, root)
|
|
339
|
+
# Writing the planned bytes over a file changed since planning (by the client, or another init) would lose
|
|
340
|
+
# that change, and the backup would not hold it either.
|
|
341
|
+
for change, writes in ((plan.config, writes_config), (plan.manifest, writes_manifest)):
|
|
342
|
+
if writes:
|
|
343
|
+
_unchanged(change.before)
|
|
344
|
+
if plan.registration.backend == "cli":
|
|
345
|
+
for command in plan.registration.commands:
|
|
346
|
+
try:
|
|
347
|
+
agent_cli.run_client(plan.request.agent, command)
|
|
348
|
+
except agent_cli.ClientFailure as error:
|
|
349
|
+
if error.unknown:
|
|
350
|
+
server_state = "unknown"
|
|
351
|
+
if command.operation == "add" and any(c.operation == "remove" for c in plan.registration.commands):
|
|
352
|
+
try:
|
|
353
|
+
adapter = agent_config.discover(plan.request.agent, project_root=plan.request.project_root)
|
|
354
|
+
if adapter.server(adapter.parse(snapshot(adapter.path).data), NAME) is None:
|
|
355
|
+
server_state = "removed"
|
|
356
|
+
except InitError:
|
|
357
|
+
pass
|
|
358
|
+
raise
|
|
359
|
+
server_state = "removed" if command.operation == "remove" else "unknown"
|
|
360
|
+
if plan.registration.commands:
|
|
361
|
+
adapter = agent_config.discover(plan.request.agent, project_root=plan.request.project_root)
|
|
362
|
+
actual = adapter.server(adapter.parse(snapshot(adapter.path).data), NAME)
|
|
363
|
+
if actual is None or agent_config.normalized_entry(plan.request.agent, actual) != agent_config.normalized_entry(
|
|
364
|
+
plan.request.agent, plan.selected.replacement):
|
|
365
|
+
raise InitError(f"{plan.request.agent} mcp add {NAME} could not be verified; rerun after checking client compatibility")
|
|
366
|
+
elif writes_config:
|
|
367
|
+
if plan.config.before.data is not None:
|
|
368
|
+
backups.append(_backup(plan.config.before))
|
|
369
|
+
_write(plan.config)
|
|
370
|
+
completed.extend(a for a in plan.actions if a.kind == "server")
|
|
371
|
+
if writes_manifest:
|
|
372
|
+
_write(plan.manifest)
|
|
373
|
+
except (OSError, InitError) as error:
|
|
374
|
+
if isinstance(error, OSError):
|
|
375
|
+
message = error.strerror or "filesystem operation failed"
|
|
376
|
+
else:
|
|
377
|
+
message = str(error)
|
|
378
|
+
remaining = [Action(a.kind, a.name, a.path, server_state if a.kind == "server" else "preserved") for a in plan.actions
|
|
379
|
+
if not any((a.kind, a.name) == (c.kind, c.name) for c in completed)]
|
|
380
|
+
return InitResult(plan, tuple(completed + remaining), tuple(backups), f"setup incomplete: {message}; rerun init to reconcile",
|
|
381
|
+
interrupted=isinstance(error, agent_cli.ClientFailure) and error.interrupted)
|
|
382
|
+
return InitResult(plan, tuple(completed), tuple(backups))
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Read a site's page list from its Sphinx `objects.inv` (inventory version 2).
|
|
2
|
+
|
|
3
|
+
The file is four header lines, then zlib-compressed lines of the form `name domain:role priority uri dispname`,
|
|
4
|
+
where `name` may contain spaces. The `std:doc` lines name the site's pages. The other lines, such as section
|
|
5
|
+
labels, are not used: the sections and their anchors come from the pages themselves.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import re
|
|
9
|
+
import zlib
|
|
10
|
+
|
|
11
|
+
_LINE = re.compile(r"(.+?)\s+(\S+:\S+)\s+(-?\d+)\s+(\S*)\s+(.*)")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class InventoryError(ValueError):
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def pages(data: bytes) -> list[str]:
|
|
19
|
+
"""The document names in the bytes of an objects.inv, such as `batchsystem`. Each page is at `<name>.html`.
|
|
20
|
+
|
|
21
|
+
Raises InventoryError when the bytes are not a version 2 inventory.
|
|
22
|
+
"""
|
|
23
|
+
head = data.split(b"\n", 4)
|
|
24
|
+
if len(head) < 5 or head[0].rstrip() != b"# Sphinx inventory version 2":
|
|
25
|
+
raise InventoryError("not a Sphinx inventory version 2")
|
|
26
|
+
try:
|
|
27
|
+
body = zlib.decompress(head[4]).decode("utf-8")
|
|
28
|
+
except (zlib.error, UnicodeDecodeError) as e:
|
|
29
|
+
raise InventoryError(f"cannot decompress the inventory: {e}") from e
|
|
30
|
+
lines = (_LINE.fullmatch(line.rstrip()) for line in body.splitlines())
|
|
31
|
+
return [m[1] for m in lines if m and m[2] == "std:doc"]
|
jscdocs/core/lock.py
ADDED
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""A lock file that lets one process at a time replace an index.
|
|
2
|
+
|
|
3
|
+
Every Claude session starts its own server, and login nodes may share one index, so several processes can decide to
|
|
4
|
+
sync at once. The lock is `<index>.lock`, created with O_EXCL (atomic on local file systems and on GPFS) and holding
|
|
5
|
+
the owner's host, PID and start time. A lock is stale, and taken over, when its owner on this host is gone, or when
|
|
6
|
+
it is older than `max_age` (an owner on another host cannot be checked, and no writer takes hours).
|
|
7
|
+
|
|
8
|
+
The age rule can take the lock from an owner that still runs, such as one whose machine slept in the middle of a
|
|
9
|
+
download. That owner leaves the new lock alone when it is done, and every writer has a temporary file of its own
|
|
10
|
+
(store.py), so two at once download the index twice but cannot damage it.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
import contextlib
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import socket
|
|
17
|
+
import time
|
|
18
|
+
from dataclasses import asdict, dataclass
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
|
|
21
|
+
MAX_AGE = 3600.0 # seconds; a download takes seconds, and a crawl of all sites about three minutes
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Busy(RuntimeError):
|
|
25
|
+
def __init__(self, path: Path, holder: str):
|
|
26
|
+
super().__init__(f"another sync is running ({holder}); lock file {path}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Holder:
|
|
31
|
+
host: str
|
|
32
|
+
pid: int
|
|
33
|
+
started: float
|
|
34
|
+
|
|
35
|
+
def describe(self) -> str:
|
|
36
|
+
return f"pid {self.pid} on {self.host}, started {time.strftime('%H:%M', time.localtime(self.started))}"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def path_for(index: Path) -> Path:
|
|
40
|
+
return index.with_name(index.name + ".lock")
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def holder(lock: Path) -> Holder | None:
|
|
44
|
+
"""Who holds the lock, or None when there is no lock file. An unreadable file has an unknown holder."""
|
|
45
|
+
try:
|
|
46
|
+
data = json.loads(lock.read_text())
|
|
47
|
+
return Holder(str(data["host"]), int(data["pid"]), float(data["started"]))
|
|
48
|
+
except FileNotFoundError:
|
|
49
|
+
return None
|
|
50
|
+
except (ValueError, KeyError, TypeError, OSError):
|
|
51
|
+
return Holder("?", 0, _mtime(lock))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _mtime(lock: Path) -> float:
|
|
55
|
+
try:
|
|
56
|
+
return lock.stat().st_mtime
|
|
57
|
+
except OSError:
|
|
58
|
+
return 0.0
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _alive(pid: int) -> bool:
|
|
62
|
+
try:
|
|
63
|
+
os.kill(pid, 0)
|
|
64
|
+
except ProcessLookupError:
|
|
65
|
+
return False
|
|
66
|
+
except PermissionError:
|
|
67
|
+
pass # it runs, as another user
|
|
68
|
+
return True
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def stale(h: Holder, max_age: float = MAX_AGE) -> bool:
|
|
72
|
+
"""Whether a lock is left over, so that the next sync takes it: its owner on this host is gone, or it is old."""
|
|
73
|
+
if time.time() - h.started > max_age:
|
|
74
|
+
return True
|
|
75
|
+
return h.host == socket.gethostname() and h.pid > 0 and not _alive(h.pid)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Lock:
|
|
79
|
+
"""`with Lock(index):` holds the index's sync lock, or raises Busy."""
|
|
80
|
+
|
|
81
|
+
def __init__(self, index: Path, max_age: float = MAX_AGE):
|
|
82
|
+
self.path = path_for(index)
|
|
83
|
+
self.max_age = max_age
|
|
84
|
+
self._me: Holder | None = None # what this lock wrote into the file while it holds it
|
|
85
|
+
|
|
86
|
+
def acquire(self) -> None:
|
|
87
|
+
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
88
|
+
me = Holder(socket.gethostname(), os.getpid(), time.time())
|
|
89
|
+
for _ in range(2): # a second try after removing a stale lock
|
|
90
|
+
try:
|
|
91
|
+
fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
92
|
+
except FileExistsError:
|
|
93
|
+
h = holder(self.path)
|
|
94
|
+
if h is None:
|
|
95
|
+
continue # released in between
|
|
96
|
+
if not stale(h, self.max_age):
|
|
97
|
+
raise Busy(self.path, h.describe()) from None
|
|
98
|
+
self.path.unlink(missing_ok=True)
|
|
99
|
+
continue
|
|
100
|
+
try:
|
|
101
|
+
with os.fdopen(fd, "w") as f:
|
|
102
|
+
json.dump(asdict(me), f)
|
|
103
|
+
except BaseException:
|
|
104
|
+
# as on a full disk: an empty lock file would count as held, and refuse every sync until it is old
|
|
105
|
+
with contextlib.suppress(OSError):
|
|
106
|
+
self.path.unlink()
|
|
107
|
+
raise
|
|
108
|
+
self._me = me
|
|
109
|
+
return
|
|
110
|
+
h = holder(self.path)
|
|
111
|
+
raise Busy(self.path, h.describe() if h else "unknown")
|
|
112
|
+
|
|
113
|
+
def release(self) -> None:
|
|
114
|
+
"""Remove the lock file, unless another process took the lock over as stale: the file is theirs then."""
|
|
115
|
+
if self._me is not None:
|
|
116
|
+
if holder(self.path) == self._me:
|
|
117
|
+
self.path.unlink(missing_ok=True)
|
|
118
|
+
self._me = None
|
|
119
|
+
|
|
120
|
+
def __enter__(self) -> "Lock":
|
|
121
|
+
self.acquire()
|
|
122
|
+
return self
|
|
123
|
+
|
|
124
|
+
def __exit__(self, *_) -> None:
|
|
125
|
+
self.release()
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""The published index: the index that this project's CI builds and uploads, and that every installation downloads.
|
|
2
|
+
No installation fetches the documentation sites itself.
|
|
3
|
+
|
|
4
|
+
A published index is two files in a directory named after the schema version, `schema-<N>/`:
|
|
5
|
+
|
|
6
|
+
index.sqlite.gz the index file, gzipped
|
|
7
|
+
index.json the manifest: when the index was synced, and the size and SHA-256 of index.sqlite.gz
|
|
8
|
+
|
|
9
|
+
A jscdocs with another schema version asks for another directory, so it never downloads a file it cannot read. The
|
|
10
|
+
manifest is fetched first, so finding that there is nothing newer costs one short request.
|
|
11
|
+
|
|
12
|
+
The index holds the documentation's text, so it is under the documentation's licence and not under that of the
|
|
13
|
+
code. The manifest names the licence and the source, as the index's `meta` table does (crawl.py writes them).
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import gzip
|
|
17
|
+
import hashlib
|
|
18
|
+
import json
|
|
19
|
+
import zlib
|
|
20
|
+
from collections.abc import Callable
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from datetime import datetime
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
from . import sites, store
|
|
26
|
+
from .fetch import Fetcher, FetchError
|
|
27
|
+
|
|
28
|
+
MANIFEST = "index.json"
|
|
29
|
+
DATA = "index.sqlite.gz"
|
|
30
|
+
MAX_DOWNLOAD = 100 * 1024 * 1024 # of index.sqlite.gz; it is about 2 MB today
|
|
31
|
+
MAX_INDEX = 500 * 1024 * 1024 # of the unpacked index, 7.5 MB today: a damaged file must not fill a small $HOME
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Unavailable(Exception):
|
|
35
|
+
"""The published index could not be downloaded, or what came is not an index this jscdocs can use."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class Manifest:
|
|
40
|
+
synced_at: datetime
|
|
41
|
+
bytes: int
|
|
42
|
+
sha256: str
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def directory() -> str:
|
|
46
|
+
return f"schema-{store.SCHEMA_VERSION}"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _manifest(data: bytes, url: str) -> Manifest:
|
|
50
|
+
try:
|
|
51
|
+
m = json.loads(data)
|
|
52
|
+
synced_at = datetime.fromisoformat(m["synced_at"])
|
|
53
|
+
if synced_at.tzinfo is None or m["schema_version"] != store.SCHEMA_VERSION:
|
|
54
|
+
raise ValueError
|
|
55
|
+
return Manifest(synced_at, int(m["bytes"]), str(m["sha256"]))
|
|
56
|
+
except (ValueError, KeyError, TypeError):
|
|
57
|
+
raise Unavailable(f"{url} is not the manifest of an index this version of jscdocs can read") from None
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _get(fetcher: Fetcher, url: str, max_bytes: int | None = None) -> bytes:
|
|
61
|
+
try:
|
|
62
|
+
return fetcher.get(url, max_bytes)
|
|
63
|
+
except FetchError as e:
|
|
64
|
+
raise Unavailable(str(e)) from e
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _unpack(packed: bytes, url: str) -> bytes:
|
|
68
|
+
unpacker = zlib.decompressobj(wbits=31) # 31: with a gzip header and trailer
|
|
69
|
+
try:
|
|
70
|
+
data = unpacker.decompress(packed, MAX_INDEX + 1)
|
|
71
|
+
except zlib.error as e:
|
|
72
|
+
raise Unavailable(f"{url} cannot be unpacked ({e})") from e
|
|
73
|
+
if len(data) > MAX_INDEX or not unpacker.eof:
|
|
74
|
+
raise Unavailable(f"{url} unpacks to more than {MAX_INDEX} bytes")
|
|
75
|
+
return data
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def download(path: Path, base_url: str, fetcher: Fetcher, have: datetime | None,
|
|
79
|
+
log: Callable[[str], None] = lambda _: None) -> datetime | None:
|
|
80
|
+
"""Replace the index at `path` with the published one, unless the index, synced at `have`, is as new.
|
|
81
|
+
|
|
82
|
+
Returns the sync time of the index it put in place, or None when it left the index alone. Raises Unavailable,
|
|
83
|
+
also when the download does not match its manifest or is not an index; `path` is untouched then. The caller
|
|
84
|
+
holds the index's lock.
|
|
85
|
+
"""
|
|
86
|
+
base = f"{base_url}/{directory()}/"
|
|
87
|
+
manifest = _manifest(_get(fetcher, base + MANIFEST), base + MANIFEST)
|
|
88
|
+
if have is not None and manifest.synced_at <= have:
|
|
89
|
+
return None
|
|
90
|
+
log(f"downloading the published index of {manifest.synced_at:%Y-%m-%d} ({manifest.bytes / 1e6:.1f} MB)")
|
|
91
|
+
packed = _get(fetcher, base + DATA, MAX_DOWNLOAD)
|
|
92
|
+
if len(packed) != manifest.bytes or hashlib.sha256(packed).hexdigest() != manifest.sha256:
|
|
93
|
+
raise Unavailable(f"{base + DATA} does not match its manifest")
|
|
94
|
+
try:
|
|
95
|
+
store.install(path, _unpack(packed, base + DATA))
|
|
96
|
+
except store.IndexMissing as e:
|
|
97
|
+
raise Unavailable(f"{base + DATA} is {e}") from None
|
|
98
|
+
return manifest.synced_at
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def export(index: Path, out: Path) -> Path:
|
|
102
|
+
"""Write the index at `index` as a published index into `out/schema-<N>/`, and return that directory.
|
|
103
|
+
|
|
104
|
+
Raises store.IndexMissing when there is no index to export.
|
|
105
|
+
"""
|
|
106
|
+
db = store.connect(index)
|
|
107
|
+
try:
|
|
108
|
+
meta = dict(db.execute("SELECT key, value FROM meta").fetchall())
|
|
109
|
+
finally:
|
|
110
|
+
db.close()
|
|
111
|
+
packed = gzip.compress(index.read_bytes(), mtime=0) # no time stamp: the same index gives the same file
|
|
112
|
+
manifest = {
|
|
113
|
+
"schema_version": store.SCHEMA_VERSION,
|
|
114
|
+
"synced_at": meta["synced_at"],
|
|
115
|
+
"jscdocs_version": meta.get("jscdocs_version", ""),
|
|
116
|
+
"bytes": len(packed),
|
|
117
|
+
"sha256": hashlib.sha256(packed).hexdigest(),
|
|
118
|
+
"license": sites.LICENSE,
|
|
119
|
+
"attribution": sites.ATTRIBUTION,
|
|
120
|
+
}
|
|
121
|
+
target = out / directory()
|
|
122
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
123
|
+
(target / DATA).write_bytes(packed)
|
|
124
|
+
(target / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n")
|
|
125
|
+
return target
|