jscdocs 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
jscdocs/core/init.py ADDED
@@ -0,0 +1,382 @@
1
+ """Plan and apply `jscdocs init`: register the jscdocs MCP server with a coding agent (Claude Code, Codex, pi or
2
+ OpenCode), through the agent's own command when it can write the whole entry, else by editing its configuration file.
3
+ Init never starts the server or fetches anything."""
4
+
5
+ import json
6
+ import os
7
+ import shutil
8
+ import stat
9
+ import sys
10
+ import tempfile
11
+ from dataclasses import dataclass, replace
12
+ from pathlib import Path
13
+
14
+ from .. import __version__
15
+ from . import agent_cli, agent_config
16
+ from .config import index_path
17
+ from .agent_config import NAME, Agent, InitError, ServerSpec, absolute, strict_json
18
+
19
+ FORMAT = 1
20
+ SERVER = "jscdocs-mcp"
21
+ # Codex starts servers with only a few variables of its own environment, so these are passed on by name: the server
22
+ # then uses the index, the published copy and the refresh interval that the jscdocs CLI uses.
23
+ FORWARD_ENV = ("JSCDOCS_INDEX", "JSCDOCS_INDEX_URL", "JSCDOCS_REFRESH_HOURS", "XDG_CACHE_HOME")
24
+
25
+
26
+ @dataclass(frozen=True)
27
+ class InitRequest:
28
+ agent: Agent
29
+ index: Path | None = None # the server's --index; default: $JSCDOCS_INDEX or the cache, as for the CLI
30
+ refresh: bool = True # False adds --no-refresh
31
+ force: bool = False
32
+ project_root: Path | None = None
33
+
34
+ @property
35
+ def scope(self) -> str:
36
+ return "project" if self.project_root is not None else "user"
37
+
38
+
39
+ @dataclass(frozen=True)
40
+ class Snapshot:
41
+ path: Path
42
+ target: Path
43
+ data: bytes | None
44
+ mode: int
45
+ private: bool = True # project files are shared: new ones and their directories keep the umask's modes
46
+
47
+
48
+ @dataclass(frozen=True)
49
+ class FileChange:
50
+ before: Snapshot
51
+ after: bytes
52
+
53
+
54
+ @dataclass(frozen=True)
55
+ class Action:
56
+ kind: str
57
+ name: str
58
+ path: Path
59
+ action: str
60
+
61
+ def as_dict(self) -> dict:
62
+ return {"kind": self.kind, "name": self.name, "path": str(self.path), "action": self.action}
63
+
64
+
65
+ @dataclass(frozen=True)
66
+ class InitPlan:
67
+ request: InitRequest
68
+ servers: tuple[ServerSpec, ...]
69
+ config: FileChange
70
+ manifest: FileChange
71
+ actions: tuple[Action, ...]
72
+ conflicts: tuple[str, ...]
73
+ notes: tuple[str, ...]
74
+ selected: agent_config.ServerChange
75
+ registration: agent_cli.RegistrationPlan
76
+
77
+ def as_dict(self) -> dict:
78
+ return {"agent": self.request.agent, "scope": self.request.scope, "dry_run": True,
79
+ "project_root": str(self.request.project_root) if self.request.project_root is not None else None,
80
+ "manifest": str(self.manifest.before.path),
81
+ "ok": not self.conflicts, "config": str(self.config.before.path),
82
+ "registration": self.registration.as_dict(),
83
+ "servers": [{"name": s.name, "command": s.command, "args": list(s.args),
84
+ "forward_env": list(s.forward_env)} for s in self.servers],
85
+ "actions": [a.as_dict() for a in self.actions], "conflicts": list(self.conflicts),
86
+ "notes": list(self.notes)}
87
+
88
+
89
+ @dataclass(frozen=True)
90
+ class InitResult:
91
+ plan: InitPlan
92
+ actions: tuple[Action, ...]
93
+ backups: tuple[Path, ...] = ()
94
+ error: str | None = None
95
+ interrupted: bool = False
96
+
97
+ @property
98
+ def ok(self) -> bool:
99
+ return not self.plan.conflicts and self.error is None
100
+
101
+ def as_dict(self) -> dict:
102
+ return self.plan.as_dict() | {"dry_run": False, "ok": self.ok,
103
+ "actions": [a.as_dict() for a in self.actions],
104
+ "backups": [str(p) for p in self.backups], "error": self.error,
105
+ "interrupted": self.interrupted}
106
+
107
+
108
+ def _parents(path: Path) -> None:
109
+ for parent in path.parents:
110
+ if parent.is_symlink() and not parent.exists():
111
+ raise InitError(f"{parent}: dangling directory symlink")
112
+ if parent.exists() and not parent.is_dir():
113
+ raise InitError(f"{parent}: expected a directory")
114
+
115
+
116
+ def _file_mode(private: bool) -> int:
117
+ """Mode for a new file: owner-only, or the umask's default for files others share."""
118
+ if private:
119
+ return 0o600
120
+ mask = os.umask(0o077)
121
+ os.umask(mask)
122
+ return 0o666 & ~mask
123
+
124
+
125
+ def snapshot(path: Path, *, private: bool = True) -> Snapshot:
126
+ path = absolute(path)
127
+ try:
128
+ _parents(path)
129
+ target = path.resolve()
130
+ if path.is_symlink() and not path.exists():
131
+ raise InitError(f"{path}: dangling configuration symlink")
132
+ if target.exists() and not target.is_file():
133
+ raise InitError(f"{path}: expected a regular file")
134
+ data = target.read_bytes() if target.exists() else None
135
+ mode = stat.S_IMODE(target.stat().st_mode) if data is not None else _file_mode(private)
136
+ return Snapshot(path, target, data, mode, private)
137
+ except (OSError, RuntimeError):
138
+ raise InitError(f"{path}: cannot read configuration") from None
139
+
140
+
141
+ def launcher() -> tuple[str, tuple[str, ...]]:
142
+ """This installation's server: the jscdocs-mcp next to its Python, else the one on PATH, else its Python with -m."""
143
+ sibling = Path(sys.executable).absolute().parent / SERVER
144
+ if sibling.is_file() and os.access(sibling, os.X_OK):
145
+ return str(sibling), ()
146
+ if found := shutil.which(SERVER):
147
+ return str(absolute(found)), ()
148
+ # -P: `-m` would put the client's working directory (a project) first on sys.path, where a project's own mcp/
149
+ # would shadow jscdocs's dependencies.
150
+ return str(Path(sys.executable).absolute()), ("-P", "-m", "jscdocs.mcp.server")
151
+
152
+
153
+ def _manifest(data: bytes | None, path: Path, config_key: str) -> dict:
154
+ try:
155
+ document = strict_json(data.decode()) if data is not None else {"format": FORMAT, "configs": {}}
156
+ if type(document.get("format")) is not int or document["format"] != FORMAT or not isinstance(document.get("configs"), dict):
157
+ raise ValueError
158
+ entries = document["configs"].get(config_key, {})
159
+ if not isinstance(entries, dict):
160
+ raise ValueError
161
+ fields = entries.get(NAME, {})
162
+ if not isinstance(fields, dict) or set(fields) - {"command", "args", "env_vars", "type", "enabled", "url"}:
163
+ raise ValueError
164
+ return document
165
+ except (ValueError, UnicodeError):
166
+ raise InitError(f"{path}: invalid or unsupported initializer manifest") from None
167
+
168
+
169
+ def normalize_request(request: InitRequest) -> InitRequest:
170
+ if request.project_root is None:
171
+ return request
172
+ try:
173
+ root = request.project_root.expanduser().resolve(strict=True)
174
+ except (OSError, RuntimeError):
175
+ raise ValueError("project directory must exist") from None
176
+ if not root.is_dir():
177
+ raise ValueError("project directory must be a directory")
178
+ return replace(request, project_root=root)
179
+
180
+
181
+ def manifest_path(request: InitRequest) -> Path:
182
+ """Where init records the entry it wrote, to tell its own entry from one written by hand."""
183
+ if request.project_root is not None:
184
+ base = request.project_root / ".jscdocs"
185
+ else:
186
+ base = Path(os.environ.get("XDG_STATE_HOME") or Path.home() / ".local" / "state").expanduser() / "jscdocs"
187
+ return base / "init" / f"{request.agent}.json"
188
+
189
+
190
+ def require_project_path(path: Path, root: Path) -> Path:
191
+ _parents(path)
192
+ try:
193
+ effective = path.resolve()
194
+ effective.relative_to(root)
195
+ except (OSError, RuntimeError, ValueError):
196
+ raise InitError(f"{path}: project installation destination escapes the project") from None
197
+ if path.is_symlink() and not path.exists():
198
+ raise InitError(f"{path}: dangling project destination symlink")
199
+ if effective.exists() and not (effective.is_file() or effective.is_dir()):
200
+ raise InitError(f"{path}: unsupported file type")
201
+ return effective
202
+
203
+
204
+ def _shared_notes() -> list[str]:
205
+ """Warnings for a project entry, which runs the jscdocs-mcp on PATH."""
206
+ own = launcher()[0]
207
+ found = shutil.which(SERVER)
208
+ if found is None:
209
+ return [f"{SERVER} is not on PATH, so the client cannot start the server: "
210
+ "start the client with jscdocs's environment active."]
211
+ if Path(found).resolve() != Path(own).resolve():
212
+ return [f"The client will start the {SERVER} on PATH ({found}), not this installation's ({own})."]
213
+ return []
214
+
215
+
216
+ def plan_init(request: InitRequest) -> InitPlan:
217
+ """What init would change, read-only: it creates no file and starts no process. Raises ValueError for an invalid
218
+ request and InitError for a configuration it cannot read or must not edit."""
219
+ request = normalize_request(request)
220
+ if request.agent not in agent_config.AGENTS:
221
+ raise ValueError(f"unknown agent: {request.agent}")
222
+ shared = request.project_root is not None
223
+ if shared and request.index is not None:
224
+ raise ValueError("--project writes a file others share, so it takes no --index: "
225
+ "each user's JSCDOCS_INDEX, or the default, applies")
226
+ index = absolute(request.index) if request.index is not None else None
227
+ if index is not None and "\0" in str(index):
228
+ raise ValueError("--index must not contain NUL")
229
+ reload = ("OpenCode reloads registrations by itself (/reload forces it)." if request.agent == "opencode"
230
+ else "Restart the client (or /reload in pi) to load the registration.")
231
+ notes = [reload, "User/managed settings, project overrides, and client trust settings may affect activation."]
232
+ if not (index or index_path()).exists():
233
+ notes.append("There is no index yet: run jscdocs sync to download it, or let the server download it when it "
234
+ "starts.")
235
+ client = agent_cli.find_client(request.agent)
236
+ adapter = agent_config.discover(request.agent, project_root=request.project_root)
237
+ metadata = manifest_path(request)
238
+ if request.project_root is not None:
239
+ for path in (adapter.path, metadata):
240
+ require_project_path(path, request.project_root)
241
+ config = snapshot(adapter.path, private=not shared)
242
+ manifest = snapshot(metadata, private=not shared)
243
+ document = adapter.parse(config.data)
244
+ for other in adapter.registered_elsewhere(NAME):
245
+ raise InitError(f"{other}: also registers {NAME}, and {request.agent} reads it together with "
246
+ f"{adapter.path}; keep the entry in one file, then rerun init")
247
+ # A project's manifest is keyed relative to the project, so a moved or copied project still owns its entry.
248
+ key = str(config.target.relative_to(request.project_root)) if shared else str(config.target)
249
+ history = _manifest(manifest.data, manifest.path, key)
250
+ owned = history["configs"].setdefault(key, {})
251
+ servers, actions, conflicts = [], [], []
252
+ config_dirty = False
253
+ # A project entry is the same for everyone: PATH finds the server, and each user's environment the index.
254
+ command, prefix = (SERVER, ()) if shared else launcher()
255
+ args = list(prefix)
256
+ if index is not None:
257
+ args.extend(["--index", str(index)])
258
+ if not request.refresh:
259
+ args.append("--no-refresh")
260
+ if shared:
261
+ notes.extend(_shared_notes())
262
+ spec = ServerSpec(NAME, command, tuple(args), FORWARD_ENV if request.agent == "codex" else ())
263
+ servers.append(spec)
264
+ selected = agent_config.reconcile(adapter, spec, adapter.server(document, spec.name), owned.get(spec.name),
265
+ force=request.force)
266
+ if selected.conflict:
267
+ conflicts.append(selected.conflict)
268
+ else:
269
+ owned[spec.name] = selected.desired
270
+ if selected.action in ("added", "updated"):
271
+ adapter.put(document, spec.name, selected.replacement)
272
+ config_dirty = True
273
+ actions.append(Action("server", spec.name, adapter.path, selected.action))
274
+ registration = agent_cli.plan_registration(adapter, client, selected, request.project_root)
275
+ if registration.backend == "file" and config_dirty and not adapter.lossless(config.data):
276
+ conflicts.append(f"{spec.name}: {adapter.path} has comments or trailing commas that writing the file directly "
277
+ f"would drop; preserved (add the entry with the {request.agent} CLI, or remove them)")
278
+ if registration.reason:
279
+ notes.append(f"Direct-file fallback: {registration.reason}. Avoid concurrent client edits during setup.")
280
+ if any(command.operation == "remove" for command in registration.commands):
281
+ notes.append(f"Claude replacement removes {NAME} before adding it again; a failed add can leave it missing.")
282
+ history["jscdocs"] = __version__
283
+ manifest_after = (json.dumps(history, indent=2, sort_keys=True) + "\n").encode()
284
+ return InitPlan(request, tuple(servers), FileChange(config, adapter.render(document) if config_dirty else config.data or b""),
285
+ FileChange(manifest, manifest_after), tuple(actions), tuple(conflicts), tuple(notes), selected,
286
+ registration)
287
+
288
+
289
+ def _mkdir(path: Path, mode: int) -> None:
290
+ if not path.exists():
291
+ _mkdir(path.parent, mode)
292
+ path.mkdir(mode=mode, exist_ok=True)
293
+
294
+
295
+ def _backup(before: Snapshot) -> Path:
296
+ path = before.target.with_name(f"{before.target.name}.jscdocs-backup.bak")
297
+ _write(FileChange(Snapshot(path, path, None, 0o600), before.data))
298
+ return path
299
+
300
+
301
+ def _write(change: FileChange) -> None:
302
+ before = change.before
303
+ _mkdir(before.target.parent, 0o700 if before.private else 0o777)
304
+ fd, name = tempfile.mkstemp(prefix=f".{before.target.name}.jscdocs-", dir=before.target.parent)
305
+ staging = Path(name)
306
+ try:
307
+ with os.fdopen(fd, "wb") as out:
308
+ os.fchmod(out.fileno(), before.mode)
309
+ out.write(change.after)
310
+ out.flush()
311
+ os.fsync(out.fileno())
312
+ os.replace(staging, before.target)
313
+ finally:
314
+ staging.unlink(missing_ok=True)
315
+
316
+
317
+ def _unchanged(before: Snapshot) -> None:
318
+ if snapshot(before.path).data != before.data:
319
+ raise InitError(f"{before.path}: changed since init planned the change; rerun init")
320
+
321
+
322
+ def apply_init(plan: InitPlan) -> InitResult:
323
+ """Make the planned change. Failures after the checks come back in the result, with what was done and what not."""
324
+ if plan.conflicts:
325
+ return InitResult(plan, tuple(Action(a.kind, a.name, a.path, "preserved") for a in plan.actions), error="resolve conflicts before applying")
326
+ completed, backups = [], []
327
+ server_state = "preserved"
328
+ writes_config = plan.registration.backend == "file" and plan.config.after != (plan.config.before.data or b"")
329
+ writes_manifest = plan.manifest.after != plan.manifest.before.data
330
+ try:
331
+ if plan.request.project_root is not None:
332
+ root = plan.request.project_root
333
+ if not root.is_dir():
334
+ raise InitError(f"{root}: project directory must exist")
335
+ paths = [plan.config.before.path, plan.config.before.target,
336
+ plan.manifest.before.path, plan.manifest.before.target]
337
+ for path in paths:
338
+ require_project_path(path, root)
339
+ # Writing the planned bytes over a file changed since planning (by the client, or another init) would lose
340
+ # that change, and the backup would not hold it either.
341
+ for change, writes in ((plan.config, writes_config), (plan.manifest, writes_manifest)):
342
+ if writes:
343
+ _unchanged(change.before)
344
+ if plan.registration.backend == "cli":
345
+ for command in plan.registration.commands:
346
+ try:
347
+ agent_cli.run_client(plan.request.agent, command)
348
+ except agent_cli.ClientFailure as error:
349
+ if error.unknown:
350
+ server_state = "unknown"
351
+ if command.operation == "add" and any(c.operation == "remove" for c in plan.registration.commands):
352
+ try:
353
+ adapter = agent_config.discover(plan.request.agent, project_root=plan.request.project_root)
354
+ if adapter.server(adapter.parse(snapshot(adapter.path).data), NAME) is None:
355
+ server_state = "removed"
356
+ except InitError:
357
+ pass
358
+ raise
359
+ server_state = "removed" if command.operation == "remove" else "unknown"
360
+ if plan.registration.commands:
361
+ adapter = agent_config.discover(plan.request.agent, project_root=plan.request.project_root)
362
+ actual = adapter.server(adapter.parse(snapshot(adapter.path).data), NAME)
363
+ if actual is None or agent_config.normalized_entry(plan.request.agent, actual) != agent_config.normalized_entry(
364
+ plan.request.agent, plan.selected.replacement):
365
+ raise InitError(f"{plan.request.agent} mcp add {NAME} could not be verified; rerun after checking client compatibility")
366
+ elif writes_config:
367
+ if plan.config.before.data is not None:
368
+ backups.append(_backup(plan.config.before))
369
+ _write(plan.config)
370
+ completed.extend(a for a in plan.actions if a.kind == "server")
371
+ if writes_manifest:
372
+ _write(plan.manifest)
373
+ except (OSError, InitError) as error:
374
+ if isinstance(error, OSError):
375
+ message = error.strerror or "filesystem operation failed"
376
+ else:
377
+ message = str(error)
378
+ remaining = [Action(a.kind, a.name, a.path, server_state if a.kind == "server" else "preserved") for a in plan.actions
379
+ if not any((a.kind, a.name) == (c.kind, c.name) for c in completed)]
380
+ return InitResult(plan, tuple(completed + remaining), tuple(backups), f"setup incomplete: {message}; rerun init to reconcile",
381
+ interrupted=isinstance(error, agent_cli.ClientFailure) and error.interrupted)
382
+ return InitResult(plan, tuple(completed), tuple(backups))
@@ -0,0 +1,31 @@
1
+ """Read a site's page list from its Sphinx `objects.inv` (inventory version 2).
2
+
3
+ The file is four header lines, then zlib-compressed lines of the form `name domain:role priority uri dispname`,
4
+ where `name` may contain spaces. The `std:doc` lines name the site's pages. The other lines, such as section
5
+ labels, are not used: the sections and their anchors come from the pages themselves.
6
+ """
7
+
8
+ import re
9
+ import zlib
10
+
11
+ _LINE = re.compile(r"(.+?)\s+(\S+:\S+)\s+(-?\d+)\s+(\S*)\s+(.*)")
12
+
13
+
14
+ class InventoryError(ValueError):
15
+ pass
16
+
17
+
18
+ def pages(data: bytes) -> list[str]:
19
+ """The document names in the bytes of an objects.inv, such as `batchsystem`. Each page is at `<name>.html`.
20
+
21
+ Raises InventoryError when the bytes are not a version 2 inventory.
22
+ """
23
+ head = data.split(b"\n", 4)
24
+ if len(head) < 5 or head[0].rstrip() != b"# Sphinx inventory version 2":
25
+ raise InventoryError("not a Sphinx inventory version 2")
26
+ try:
27
+ body = zlib.decompress(head[4]).decode("utf-8")
28
+ except (zlib.error, UnicodeDecodeError) as e:
29
+ raise InventoryError(f"cannot decompress the inventory: {e}") from e
30
+ lines = (_LINE.fullmatch(line.rstrip()) for line in body.splitlines())
31
+ return [m[1] for m in lines if m and m[2] == "std:doc"]
jscdocs/core/lock.py ADDED
@@ -0,0 +1,125 @@
1
+ """A lock file that lets one process at a time replace an index.
2
+
3
+ Every Claude session starts its own server, and login nodes may share one index, so several processes can decide to
4
+ sync at once. The lock is `<index>.lock`, created with O_EXCL (atomic on local file systems and on GPFS) and holding
5
+ the owner's host, PID and start time. A lock is stale, and taken over, when its owner on this host is gone, or when
6
+ it is older than `max_age` (an owner on another host cannot be checked, and no writer takes hours).
7
+
8
+ The age rule can take the lock from an owner that still runs, such as one whose machine slept in the middle of a
9
+ download. That owner leaves the new lock alone when it is done, and every writer has a temporary file of its own
10
+ (store.py), so two at once download the index twice but cannot damage it.
11
+ """
12
+
13
+ import contextlib
14
+ import json
15
+ import os
16
+ import socket
17
+ import time
18
+ from dataclasses import asdict, dataclass
19
+ from pathlib import Path
20
+
21
+ MAX_AGE = 3600.0 # seconds; a download takes seconds, and a crawl of all sites about three minutes
22
+
23
+
24
+ class Busy(RuntimeError):
25
+ def __init__(self, path: Path, holder: str):
26
+ super().__init__(f"another sync is running ({holder}); lock file {path}")
27
+
28
+
29
+ @dataclass(frozen=True)
30
+ class Holder:
31
+ host: str
32
+ pid: int
33
+ started: float
34
+
35
+ def describe(self) -> str:
36
+ return f"pid {self.pid} on {self.host}, started {time.strftime('%H:%M', time.localtime(self.started))}"
37
+
38
+
39
+ def path_for(index: Path) -> Path:
40
+ return index.with_name(index.name + ".lock")
41
+
42
+
43
+ def holder(lock: Path) -> Holder | None:
44
+ """Who holds the lock, or None when there is no lock file. An unreadable file has an unknown holder."""
45
+ try:
46
+ data = json.loads(lock.read_text())
47
+ return Holder(str(data["host"]), int(data["pid"]), float(data["started"]))
48
+ except FileNotFoundError:
49
+ return None
50
+ except (ValueError, KeyError, TypeError, OSError):
51
+ return Holder("?", 0, _mtime(lock))
52
+
53
+
54
+ def _mtime(lock: Path) -> float:
55
+ try:
56
+ return lock.stat().st_mtime
57
+ except OSError:
58
+ return 0.0
59
+
60
+
61
+ def _alive(pid: int) -> bool:
62
+ try:
63
+ os.kill(pid, 0)
64
+ except ProcessLookupError:
65
+ return False
66
+ except PermissionError:
67
+ pass # it runs, as another user
68
+ return True
69
+
70
+
71
+ def stale(h: Holder, max_age: float = MAX_AGE) -> bool:
72
+ """Whether a lock is left over, so that the next sync takes it: its owner on this host is gone, or it is old."""
73
+ if time.time() - h.started > max_age:
74
+ return True
75
+ return h.host == socket.gethostname() and h.pid > 0 and not _alive(h.pid)
76
+
77
+
78
+ class Lock:
79
+ """`with Lock(index):` holds the index's sync lock, or raises Busy."""
80
+
81
+ def __init__(self, index: Path, max_age: float = MAX_AGE):
82
+ self.path = path_for(index)
83
+ self.max_age = max_age
84
+ self._me: Holder | None = None # what this lock wrote into the file while it holds it
85
+
86
+ def acquire(self) -> None:
87
+ self.path.parent.mkdir(parents=True, exist_ok=True)
88
+ me = Holder(socket.gethostname(), os.getpid(), time.time())
89
+ for _ in range(2): # a second try after removing a stale lock
90
+ try:
91
+ fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
92
+ except FileExistsError:
93
+ h = holder(self.path)
94
+ if h is None:
95
+ continue # released in between
96
+ if not stale(h, self.max_age):
97
+ raise Busy(self.path, h.describe()) from None
98
+ self.path.unlink(missing_ok=True)
99
+ continue
100
+ try:
101
+ with os.fdopen(fd, "w") as f:
102
+ json.dump(asdict(me), f)
103
+ except BaseException:
104
+ # as on a full disk: an empty lock file would count as held, and refuse every sync until it is old
105
+ with contextlib.suppress(OSError):
106
+ self.path.unlink()
107
+ raise
108
+ self._me = me
109
+ return
110
+ h = holder(self.path)
111
+ raise Busy(self.path, h.describe() if h else "unknown")
112
+
113
+ def release(self) -> None:
114
+ """Remove the lock file, unless another process took the lock over as stale: the file is theirs then."""
115
+ if self._me is not None:
116
+ if holder(self.path) == self._me:
117
+ self.path.unlink(missing_ok=True)
118
+ self._me = None
119
+
120
+ def __enter__(self) -> "Lock":
121
+ self.acquire()
122
+ return self
123
+
124
+ def __exit__(self, *_) -> None:
125
+ self.release()
@@ -0,0 +1,125 @@
1
+ """The published index: the index that this project's CI builds and uploads, and that every installation downloads.
2
+ No installation fetches the documentation sites itself.
3
+
4
+ A published index is two files in a directory named after the schema version, `schema-<N>/`:
5
+
6
+ index.sqlite.gz the index file, gzipped
7
+ index.json the manifest: when the index was synced, and the size and SHA-256 of index.sqlite.gz
8
+
9
+ A jscdocs with another schema version asks for another directory, so it never downloads a file it cannot read. The
10
+ manifest is fetched first, so finding that there is nothing newer costs one short request.
11
+
12
+ The index holds the documentation's text, so it is under the documentation's licence and not under that of the
13
+ code. The manifest names the licence and the source, as the index's `meta` table does (crawl.py writes them).
14
+ """
15
+
16
+ import gzip
17
+ import hashlib
18
+ import json
19
+ import zlib
20
+ from collections.abc import Callable
21
+ from dataclasses import dataclass
22
+ from datetime import datetime
23
+ from pathlib import Path
24
+
25
+ from . import sites, store
26
+ from .fetch import Fetcher, FetchError
27
+
28
+ MANIFEST = "index.json"
29
+ DATA = "index.sqlite.gz"
30
+ MAX_DOWNLOAD = 100 * 1024 * 1024 # of index.sqlite.gz; it is about 2 MB today
31
+ MAX_INDEX = 500 * 1024 * 1024 # of the unpacked index, 7.5 MB today: a damaged file must not fill a small $HOME
32
+
33
+
34
+ class Unavailable(Exception):
35
+ """The published index could not be downloaded, or what came is not an index this jscdocs can use."""
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class Manifest:
40
+ synced_at: datetime
41
+ bytes: int
42
+ sha256: str
43
+
44
+
45
+ def directory() -> str:
46
+ return f"schema-{store.SCHEMA_VERSION}"
47
+
48
+
49
+ def _manifest(data: bytes, url: str) -> Manifest:
50
+ try:
51
+ m = json.loads(data)
52
+ synced_at = datetime.fromisoformat(m["synced_at"])
53
+ if synced_at.tzinfo is None or m["schema_version"] != store.SCHEMA_VERSION:
54
+ raise ValueError
55
+ return Manifest(synced_at, int(m["bytes"]), str(m["sha256"]))
56
+ except (ValueError, KeyError, TypeError):
57
+ raise Unavailable(f"{url} is not the manifest of an index this version of jscdocs can read") from None
58
+
59
+
60
+ def _get(fetcher: Fetcher, url: str, max_bytes: int | None = None) -> bytes:
61
+ try:
62
+ return fetcher.get(url, max_bytes)
63
+ except FetchError as e:
64
+ raise Unavailable(str(e)) from e
65
+
66
+
67
+ def _unpack(packed: bytes, url: str) -> bytes:
68
+ unpacker = zlib.decompressobj(wbits=31) # 31: with a gzip header and trailer
69
+ try:
70
+ data = unpacker.decompress(packed, MAX_INDEX + 1)
71
+ except zlib.error as e:
72
+ raise Unavailable(f"{url} cannot be unpacked ({e})") from e
73
+ if len(data) > MAX_INDEX or not unpacker.eof:
74
+ raise Unavailable(f"{url} unpacks to more than {MAX_INDEX} bytes")
75
+ return data
76
+
77
+
78
+ def download(path: Path, base_url: str, fetcher: Fetcher, have: datetime | None,
79
+ log: Callable[[str], None] = lambda _: None) -> datetime | None:
80
+ """Replace the index at `path` with the published one, unless the index, synced at `have`, is as new.
81
+
82
+ Returns the sync time of the index it put in place, or None when it left the index alone. Raises Unavailable,
83
+ also when the download does not match its manifest or is not an index; `path` is untouched then. The caller
84
+ holds the index's lock.
85
+ """
86
+ base = f"{base_url}/{directory()}/"
87
+ manifest = _manifest(_get(fetcher, base + MANIFEST), base + MANIFEST)
88
+ if have is not None and manifest.synced_at <= have:
89
+ return None
90
+ log(f"downloading the published index of {manifest.synced_at:%Y-%m-%d} ({manifest.bytes / 1e6:.1f} MB)")
91
+ packed = _get(fetcher, base + DATA, MAX_DOWNLOAD)
92
+ if len(packed) != manifest.bytes or hashlib.sha256(packed).hexdigest() != manifest.sha256:
93
+ raise Unavailable(f"{base + DATA} does not match its manifest")
94
+ try:
95
+ store.install(path, _unpack(packed, base + DATA))
96
+ except store.IndexMissing as e:
97
+ raise Unavailable(f"{base + DATA} is {e}") from None
98
+ return manifest.synced_at
99
+
100
+
101
+ def export(index: Path, out: Path) -> Path:
102
+ """Write the index at `index` as a published index into `out/schema-<N>/`, and return that directory.
103
+
104
+ Raises store.IndexMissing when there is no index to export.
105
+ """
106
+ db = store.connect(index)
107
+ try:
108
+ meta = dict(db.execute("SELECT key, value FROM meta").fetchall())
109
+ finally:
110
+ db.close()
111
+ packed = gzip.compress(index.read_bytes(), mtime=0) # no time stamp: the same index gives the same file
112
+ manifest = {
113
+ "schema_version": store.SCHEMA_VERSION,
114
+ "synced_at": meta["synced_at"],
115
+ "jscdocs_version": meta.get("jscdocs_version", ""),
116
+ "bytes": len(packed),
117
+ "sha256": hashlib.sha256(packed).hexdigest(),
118
+ "license": sites.LICENSE,
119
+ "attribution": sites.ATTRIBUTION,
120
+ }
121
+ target = out / directory()
122
+ target.mkdir(parents=True, exist_ok=True)
123
+ (target / DATA).write_bytes(packed)
124
+ (target / MANIFEST).write_text(json.dumps(manifest, indent=2) + "\n")
125
+ return target