jscdocs 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jscdocs/__init__.py +8 -0
- jscdocs/build.py +65 -0
- jscdocs/cli.py +190 -0
- jscdocs/core/__init__.py +1 -0
- jscdocs/core/agent_cli.py +131 -0
- jscdocs/core/agent_config.py +370 -0
- jscdocs/core/config.py +31 -0
- jscdocs/core/crawl.py +170 -0
- jscdocs/core/evaluate.py +97 -0
- jscdocs/core/extract.py +403 -0
- jscdocs/core/fetch.py +100 -0
- jscdocs/core/init.py +382 -0
- jscdocs/core/inventory.py +31 -0
- jscdocs/core/lock.py +125 -0
- jscdocs/core/published.py +125 -0
- jscdocs/core/refresh.py +107 -0
- jscdocs/core/search.py +294 -0
- jscdocs/core/sites.py +55 -0
- jscdocs/core/store.py +233 -0
- jscdocs/core/sync.py +47 -0
- jscdocs/mcp/__init__.py +1 -0
- jscdocs/mcp/server.py +86 -0
- jscdocs/mcp/tools.py +120 -0
- jscdocs-0.2.0.dist-info/METADATA +127 -0
- jscdocs-0.2.0.dist-info/RECORD +28 -0
- jscdocs-0.2.0.dist-info/WHEEL +4 -0
- jscdocs-0.2.0.dist-info/entry_points.txt +3 -0
- jscdocs-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""The configuration files of the coding agents that `jscdocs init` registers the MCP server with."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import math
|
|
5
|
+
import os
|
|
6
|
+
import tomllib
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Literal
|
|
10
|
+
|
|
11
|
+
import tomlkit
|
|
12
|
+
|
|
13
|
+
Agent = Literal["codex", "claude", "pi", "opencode"]
|
|
14
|
+
AGENTS = ("codex", "claude", "pi", "opencode")
|
|
15
|
+
NAME = "jscdocs" # the server entry's name; Claude Code shows the tools as mcp__jscdocs__<tool>
|
|
16
|
+
OPENCODE_SCHEMA = "https://opencode.ai/config.json"
|
|
17
|
+
# OpenCode's own names for the fields init compares under the names the other clients use.
|
|
18
|
+
OPENCODE_TYPES = {"local": "stdio", "remote": "http"}
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class InitError(Exception):
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _scan(text: str, other) -> str:
|
|
26
|
+
"""Copy JSON strings as they are and pass everything between them through `other(text, i)`, which returns
|
|
27
|
+
the text to keep and the next position."""
|
|
28
|
+
out, i = [], 0
|
|
29
|
+
while i < len(text):
|
|
30
|
+
if text[i] == '"':
|
|
31
|
+
j = i + 1
|
|
32
|
+
while j < len(text) and text[j] != '"':
|
|
33
|
+
j += 2 if text[j] == "\\" else 1
|
|
34
|
+
out.append(text[i:j + 1])
|
|
35
|
+
i = j + 1
|
|
36
|
+
else:
|
|
37
|
+
kept, i = other(text, i)
|
|
38
|
+
out.append(kept)
|
|
39
|
+
return "".join(out)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def jsonc(text: str) -> str:
|
|
43
|
+
"""JSON from JSONC: comments become spaces and trailing commas are dropped."""
|
|
44
|
+
def comment(text, i):
|
|
45
|
+
if text.startswith("//", i):
|
|
46
|
+
end = text.find("\n", i)
|
|
47
|
+
return " ", len(text) if end < 0 else end
|
|
48
|
+
if text.startswith("/*", i):
|
|
49
|
+
end = text.find("*/", i + 2)
|
|
50
|
+
if end < 0:
|
|
51
|
+
raise ValueError("unterminated comment")
|
|
52
|
+
return " ", end + 2
|
|
53
|
+
return text[i], i + 1
|
|
54
|
+
|
|
55
|
+
def comma(text, i):
|
|
56
|
+
if text[i] == ",":
|
|
57
|
+
j = i + 1 # scan, not slice: a copy of the rest per comma is quadratic
|
|
58
|
+
while j < len(text) and text[j].isspace():
|
|
59
|
+
j += 1
|
|
60
|
+
if text[j:j + 1] in ("}", "]"):
|
|
61
|
+
return "", i + 1
|
|
62
|
+
return text[i], i + 1
|
|
63
|
+
|
|
64
|
+
return _scan(_scan(text, comment), comma)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def strict_json(text: str) -> dict:
|
|
68
|
+
def pairs(items):
|
|
69
|
+
found = {}
|
|
70
|
+
for key, value in items:
|
|
71
|
+
if key in found:
|
|
72
|
+
raise ValueError("duplicate JSON key")
|
|
73
|
+
found[key] = value
|
|
74
|
+
return found
|
|
75
|
+
|
|
76
|
+
def constant(_):
|
|
77
|
+
raise ValueError("nonfinite JSON number")
|
|
78
|
+
|
|
79
|
+
def number(text):
|
|
80
|
+
value = float(text)
|
|
81
|
+
if not math.isfinite(value):
|
|
82
|
+
raise ValueError("nonfinite JSON number")
|
|
83
|
+
return value
|
|
84
|
+
|
|
85
|
+
result = json.loads(text, object_pairs_hook=pairs, parse_constant=constant, parse_float=number)
|
|
86
|
+
if not isinstance(result, dict):
|
|
87
|
+
raise ValueError("expected a JSON object")
|
|
88
|
+
return result
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
@dataclass(frozen=True)
|
|
92
|
+
class ServerSpec:
|
|
93
|
+
name: str
|
|
94
|
+
command: str
|
|
95
|
+
args: tuple[str, ...]
|
|
96
|
+
forward_env: tuple[str, ...] = () # variables Codex passes on by name; it starts servers without the user's
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass(frozen=True)
|
|
100
|
+
class AgentConfig:
|
|
101
|
+
agent: Agent
|
|
102
|
+
path: Path
|
|
103
|
+
# The client's other config files in the same scope, which it merges with path (OpenCode only).
|
|
104
|
+
others: tuple[Path, ...] = ()
|
|
105
|
+
|
|
106
|
+
def registered_elsewhere(self, name: str) -> list[Path]:
|
|
107
|
+
"""The other config files that also hold the server. Unreadable or invalid ones are skipped: init does
|
|
108
|
+
not edit them."""
|
|
109
|
+
found = []
|
|
110
|
+
for path in self.others:
|
|
111
|
+
try:
|
|
112
|
+
document = self.parse(path.read_bytes()) if path.is_file() else {}
|
|
113
|
+
except (OSError, InitError):
|
|
114
|
+
continue
|
|
115
|
+
if name in self.servers(document):
|
|
116
|
+
found.append(path)
|
|
117
|
+
return found
|
|
118
|
+
|
|
119
|
+
@property
|
|
120
|
+
def key(self) -> str:
|
|
121
|
+
"""The top-level key that holds the servers; OpenCode nests them one level deeper, in mcp.servers."""
|
|
122
|
+
return {"codex": "mcp_servers", "opencode": "mcp"}.get(self.agent, "mcpServers")
|
|
123
|
+
|
|
124
|
+
def servers(self, document, *, create: bool = False) -> dict:
|
|
125
|
+
"""The client's server entries, in its own format."""
|
|
126
|
+
if create and self.key not in document:
|
|
127
|
+
document[self.key] = {}
|
|
128
|
+
servers = document.get(self.key, {})
|
|
129
|
+
if self.agent == "opencode":
|
|
130
|
+
if create and "servers" not in servers:
|
|
131
|
+
servers["servers"] = {}
|
|
132
|
+
servers = servers.get("servers", {})
|
|
133
|
+
return servers
|
|
134
|
+
|
|
135
|
+
def server(self, document, name: str) -> dict | None:
|
|
136
|
+
"""One server entry, under the field names init compares (command and args, env, enabled, type stdio)."""
|
|
137
|
+
entry = self.servers(document).get(name)
|
|
138
|
+
return None if entry is None else self.common(entry)
|
|
139
|
+
|
|
140
|
+
def put(self, document, name: str, entry: dict) -> None:
|
|
141
|
+
"""Write a server entry given under init's field names, updating an existing one in place."""
|
|
142
|
+
native = self.native(entry)
|
|
143
|
+
servers = self.servers(document, create=True)
|
|
144
|
+
if name not in servers:
|
|
145
|
+
servers[name] = native
|
|
146
|
+
return
|
|
147
|
+
current = servers[name]
|
|
148
|
+
for key in set(current) - set(native):
|
|
149
|
+
del current[key]
|
|
150
|
+
for key, value in native.items():
|
|
151
|
+
if key not in current or current[key] != value:
|
|
152
|
+
current[key] = value
|
|
153
|
+
|
|
154
|
+
def common(self, entry: dict) -> dict:
|
|
155
|
+
if self.agent != "opencode":
|
|
156
|
+
return dict(entry)
|
|
157
|
+
# init's own names, or V1's enabled: native() writes disabled, so an enabled key would silently change.
|
|
158
|
+
if {"args", "env", "enabled"} & set(entry):
|
|
159
|
+
raise ValueError("ambiguous server fields")
|
|
160
|
+
result = {}
|
|
161
|
+
for key, value in entry.items():
|
|
162
|
+
if key == "command":
|
|
163
|
+
if not isinstance(value, list) or not value or not all(isinstance(arg, str) for arg in value):
|
|
164
|
+
raise ValueError("invalid server command")
|
|
165
|
+
result["command"], result["args"] = value[0], value[1:]
|
|
166
|
+
elif key == "environment":
|
|
167
|
+
result["env"] = value
|
|
168
|
+
elif key == "disabled":
|
|
169
|
+
if not isinstance(value, bool):
|
|
170
|
+
raise ValueError("invalid disabled flag")
|
|
171
|
+
result["enabled"] = not value
|
|
172
|
+
elif key == "type" and isinstance(value, str):
|
|
173
|
+
result["type"] = OPENCODE_TYPES.get(value, value)
|
|
174
|
+
else:
|
|
175
|
+
result[key] = value
|
|
176
|
+
return result
|
|
177
|
+
|
|
178
|
+
def native(self, entry: dict) -> dict:
|
|
179
|
+
if self.agent != "opencode":
|
|
180
|
+
return dict(entry)
|
|
181
|
+
result = {}
|
|
182
|
+
if isinstance(entry.get("type"), str):
|
|
183
|
+
result["type"] = {common: native for native, common in OPENCODE_TYPES.items()}.get(entry["type"], entry["type"])
|
|
184
|
+
for key, value in entry.items():
|
|
185
|
+
if key == "command":
|
|
186
|
+
result["command"] = [value, *entry.get("args", [])]
|
|
187
|
+
elif key == "env":
|
|
188
|
+
result["environment"] = value
|
|
189
|
+
elif key == "enabled":
|
|
190
|
+
result["disabled"] = not value
|
|
191
|
+
elif key not in ("args", "type"):
|
|
192
|
+
result[key] = value
|
|
193
|
+
return result
|
|
194
|
+
|
|
195
|
+
def lossless(self, data: bytes | None) -> bool:
|
|
196
|
+
"""Whether rendering the parsed file keeps everything in it: not when JSONC comments or commas would go."""
|
|
197
|
+
if self.agent != "opencode" or data is None:
|
|
198
|
+
return True
|
|
199
|
+
try:
|
|
200
|
+
strict_json(data.decode("utf-8"))
|
|
201
|
+
except (ValueError, UnicodeError):
|
|
202
|
+
return False
|
|
203
|
+
return True
|
|
204
|
+
|
|
205
|
+
def parse(self, data: bytes | None):
|
|
206
|
+
try:
|
|
207
|
+
text = data.decode("utf-8") if data is not None else ""
|
|
208
|
+
if self.agent == "codex":
|
|
209
|
+
tomllib.loads(text)
|
|
210
|
+
document = tomlkit.parse(text)
|
|
211
|
+
elif self.agent == "opencode":
|
|
212
|
+
document = strict_json(jsonc(text)) if data is not None else {"$schema": OPENCODE_SCHEMA}
|
|
213
|
+
else:
|
|
214
|
+
document = strict_json(text) if data is not None else {}
|
|
215
|
+
if not isinstance(document.get(self.key, {}), dict):
|
|
216
|
+
raise ValueError("server container must be a mapping")
|
|
217
|
+
if self.agent == "opencode":
|
|
218
|
+
self._opencode_layout(document)
|
|
219
|
+
servers = self.servers(document)
|
|
220
|
+
if not isinstance(servers, dict):
|
|
221
|
+
raise ValueError("server container must be a mapping")
|
|
222
|
+
if any(not isinstance(entry, dict) for entry in servers.values()):
|
|
223
|
+
raise ValueError("server entries must be mappings")
|
|
224
|
+
for entry in [self.common(servers[NAME])] if NAME in servers else []:
|
|
225
|
+
for key in ("command", "cwd", "type", "url"):
|
|
226
|
+
if key in entry and not isinstance(entry[key], str):
|
|
227
|
+
raise ValueError("invalid server field")
|
|
228
|
+
if "args" in entry and (not isinstance(entry["args"], list) or
|
|
229
|
+
not all(isinstance(arg, str) for arg in entry["args"])):
|
|
230
|
+
raise ValueError("invalid server arguments")
|
|
231
|
+
if "env" in entry and (not isinstance(entry["env"], dict) or
|
|
232
|
+
not all(isinstance(v, str) for v in entry["env"].values())):
|
|
233
|
+
raise ValueError("invalid server environment")
|
|
234
|
+
if "enabled" in entry and not isinstance(entry["enabled"], bool):
|
|
235
|
+
raise ValueError("invalid enabled flag")
|
|
236
|
+
return document
|
|
237
|
+
except (ValueError, UnicodeError):
|
|
238
|
+
raise InitError(f"{self.path}: invalid {self.agent} configuration") from None
|
|
239
|
+
|
|
240
|
+
def _opencode_layout(self, document) -> None:
|
|
241
|
+
"""Refuse servers placed directly under mcp, OpenCode's V1 layout: an entry added to mcp.servers
|
|
242
|
+
next to them would mix the two layouts. A V1 entry may be a bare override such as {"enabled": false}."""
|
|
243
|
+
v1_fields = {"type", "command", "url", "enabled", "environment", "headers", "oauth"}
|
|
244
|
+
for name, value in document.get("mcp", {}).items():
|
|
245
|
+
if name not in ("servers", "timeout") and isinstance(value, dict) and v1_fields & set(value):
|
|
246
|
+
raise InitError(f"{self.path}: MCP servers directly under mcp (OpenCode's V1 layout); "
|
|
247
|
+
f"move them under mcp.servers, then rerun init")
|
|
248
|
+
|
|
249
|
+
def entry(self, spec: ServerSpec) -> dict:
|
|
250
|
+
result = {"command": spec.command, "args": list(spec.args), "url": None} # url None: a remote entry becomes stdio
|
|
251
|
+
if self.agent in ("claude", "pi", "opencode"):
|
|
252
|
+
result["type"] = "stdio"
|
|
253
|
+
if self.agent not in ("claude", "opencode"): # OpenCode has only disabled, which an enabled server omits
|
|
254
|
+
result["enabled"] = True
|
|
255
|
+
if self.agent == "codex":
|
|
256
|
+
result["env_vars"] = list(spec.forward_env) if spec.forward_env else None
|
|
257
|
+
return result
|
|
258
|
+
|
|
259
|
+
def render(self, document) -> bytes:
|
|
260
|
+
if self.agent == "codex":
|
|
261
|
+
text = tomlkit.dumps(document)
|
|
262
|
+
tomllib.loads(text)
|
|
263
|
+
else:
|
|
264
|
+
text = json.dumps(document, indent=2, ensure_ascii=False, allow_nan=False)
|
|
265
|
+
strict_json(text)
|
|
266
|
+
try:
|
|
267
|
+
return (text.rstrip("\n") + "\n").encode()
|
|
268
|
+
except UnicodeEncodeError: # a lone surrogate escape (e.g. a cut emoji) is valid JSON but not UTF-8
|
|
269
|
+
if self.agent == "codex":
|
|
270
|
+
raise
|
|
271
|
+
return (json.dumps(document, indent=2, ensure_ascii=True, allow_nan=False) + "\n").encode()
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
@dataclass(frozen=True)
|
|
275
|
+
class ServerChange:
|
|
276
|
+
existing: dict | None
|
|
277
|
+
desired: dict
|
|
278
|
+
replacement: dict
|
|
279
|
+
action: str
|
|
280
|
+
conflict: str | None = None
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def normalized_entry(agent: Agent, entry: dict) -> dict:
|
|
284
|
+
result = dict(entry)
|
|
285
|
+
nullable = ("cwd", "url", "env_vars") if agent == "codex" else ("cwd", "url")
|
|
286
|
+
for key in nullable:
|
|
287
|
+
if result.get(key) is None:
|
|
288
|
+
result.pop(key, None)
|
|
289
|
+
if result.get("command") and not result.get("url"):
|
|
290
|
+
result.setdefault("args", [])
|
|
291
|
+
if agent in ("claude", "pi", "opencode"):
|
|
292
|
+
result.setdefault("type", "stdio")
|
|
293
|
+
if agent == "pi":
|
|
294
|
+
result.setdefault("exposure", "codemode")
|
|
295
|
+
if agent in ("codex", "pi", "opencode"):
|
|
296
|
+
result.setdefault("enabled", True)
|
|
297
|
+
if result.get("env") == {}:
|
|
298
|
+
result.pop("env")
|
|
299
|
+
if agent == "codex" and result.get("env_vars") == []:
|
|
300
|
+
result.pop("env_vars")
|
|
301
|
+
return result
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def matches(agent: Agent, entry: dict, fields: dict) -> bool:
|
|
305
|
+
actual, expected = normalized_entry(agent, entry), normalized_entry(agent, fields)
|
|
306
|
+
return all(actual.get(key) == expected.get(key) for key in fields)
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def reconcile(adapter: AgentConfig, spec: ServerSpec, existing: dict | None,
|
|
310
|
+
previous: dict | None, *, force: bool) -> ServerChange:
|
|
311
|
+
desired = adapter.entry(spec)
|
|
312
|
+
replacement = dict(existing or {})
|
|
313
|
+
conflict = None
|
|
314
|
+
if existing is not None and existing.get("enabled") is False and not force:
|
|
315
|
+
action, conflict = "conflict", "server is disabled; preserved (use --force to enable it)"
|
|
316
|
+
elif existing is not None and existing.get("enabled") is not False and matches(adapter.agent, existing, desired):
|
|
317
|
+
action = "current" if previous is not None and matches(adapter.agent, previous, desired) else "adopted"
|
|
318
|
+
elif existing is not None and not force and (previous is None or not matches(adapter.agent, existing, previous)):
|
|
319
|
+
action, conflict = "conflict", "existing server differs; preserved (use --force to replace owned fields)"
|
|
320
|
+
else:
|
|
321
|
+
action = "added" if existing is None else "updated"
|
|
322
|
+
if force:
|
|
323
|
+
for key in ("headers", "http_headers", "env_http_headers", "bearer_token_env_var", "oauth", "auth"):
|
|
324
|
+
replacement.pop(key, None)
|
|
325
|
+
for key in set(desired) | set(previous or {}):
|
|
326
|
+
if desired.get(key) is None:
|
|
327
|
+
replacement.pop(key, None)
|
|
328
|
+
else:
|
|
329
|
+
replacement[key] = desired[key]
|
|
330
|
+
if force and replacement.get("enabled") is False:
|
|
331
|
+
replacement.pop("enabled")
|
|
332
|
+
return ServerChange(dict(existing) if existing is not None else None, desired, replacement, action,
|
|
333
|
+
f"{spec.name}: {conflict}" if conflict else None)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def absolute(path: str | Path) -> Path:
|
|
337
|
+
return Path(path).expanduser().absolute()
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _opencode(candidates: list[Path]) -> AgentConfig:
|
|
341
|
+
"""The first config file that exists (a dangling link counts, to be refused later), else the first name: the
|
|
342
|
+
file `opencode mcp add` writes, so init and the client agree. The rest are kept as others, since OpenCode
|
|
343
|
+
merges them and an entry there may override this one."""
|
|
344
|
+
path = next((path for path in candidates if path.is_symlink() or path.exists()), candidates[0])
|
|
345
|
+
return AgentConfig("opencode", path, tuple(other for other in candidates if other != path))
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def discover(agent: Agent, *, project_root: Path | None = None) -> AgentConfig:
|
|
349
|
+
"""The client's server file: the user's, or the project's when `project_root` is given."""
|
|
350
|
+
if project_root is not None and agent == "opencode":
|
|
351
|
+
names = ["opencode.json", "opencode.jsonc", ".opencode/opencode.json", ".opencode/opencode.jsonc"]
|
|
352
|
+
return _opencode([project_root / name for name in names])
|
|
353
|
+
if project_root is not None:
|
|
354
|
+
layouts = {"codex": ".codex/config.toml", "claude": ".mcp.json", "pi": ".pi/mcp.json"}
|
|
355
|
+
if agent not in layouts:
|
|
356
|
+
raise ValueError(f"unknown agent: {agent}")
|
|
357
|
+
return AgentConfig(agent, project_root / layouts[agent])
|
|
358
|
+
home = Path.home()
|
|
359
|
+
if agent == "codex":
|
|
360
|
+
return AgentConfig(agent, absolute(os.environ.get("CODEX_HOME", home / ".codex")) / "config.toml")
|
|
361
|
+
if agent == "claude":
|
|
362
|
+
override = os.environ.get("CLAUDE_CONFIG_DIR")
|
|
363
|
+
return AgentConfig(agent, absolute(override) / ".claude.json" if override else home / ".claude.json")
|
|
364
|
+
if agent == "pi":
|
|
365
|
+
return AgentConfig(agent, absolute(os.environ.get("PI_CODING_AGENT_DIR", home / ".pi" / "agent")) / "mcp.json")
|
|
366
|
+
if agent == "opencode":
|
|
367
|
+
xdg = os.environ.get("XDG_CONFIG_HOME")
|
|
368
|
+
base = (absolute(xdg) if xdg else home / ".config") / "opencode"
|
|
369
|
+
return _opencode([base / "opencode.json", base / "opencode.jsonc"])
|
|
370
|
+
raise ValueError(f"unknown agent: {agent}")
|
jscdocs/core/config.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Where the index lives, where its published copy is and how often it is refreshed, from environment variables."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
REFRESH_HOURS = 7 * 24.0 # the published index is rebuilt once a week, and the documentation changes slowly
|
|
7
|
+
STALE_DAYS = 14.0 # results from an older index carry a warning: at least one refresh was missed
|
|
8
|
+
# The generic package `jscdocs-index` of this project on JSC's GitLab, which CI uploads to (core/published.py).
|
|
9
|
+
# 9367 is the project's ID, which stays the same when the project is renamed or moved.
|
|
10
|
+
INDEX_URL = "https://gitlab.jsc.fz-juelich.de/api/v4/projects/9367/packages/generic/jscdocs-index"
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def index_path() -> Path:
|
|
14
|
+
"""`$JSCDOCS_INDEX`, else `$XDG_CACHE_HOME/jscdocs/index.sqlite`, else `~/.cache/jscdocs/index.sqlite`."""
|
|
15
|
+
if env := os.environ.get("JSCDOCS_INDEX"):
|
|
16
|
+
return Path(env).expanduser()
|
|
17
|
+
cache = os.environ.get("XDG_CACHE_HOME") or "~/.cache"
|
|
18
|
+
return Path(cache).expanduser() / "jscdocs" / "index.sqlite"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def published_url() -> str:
|
|
22
|
+
"""`$JSCDOCS_INDEX_URL`, else INDEX_URL: where the published index is downloaded from."""
|
|
23
|
+
return (os.environ.get("JSCDOCS_INDEX_URL", "").strip() or INDEX_URL).rstrip("/")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def refresh_hours() -> float:
|
|
27
|
+
"""`$JSCDOCS_REFRESH_HOURS`, the index age after which the server updates it; 0 turns that off."""
|
|
28
|
+
try:
|
|
29
|
+
return max(0.0, float(os.environ.get("JSCDOCS_REFRESH_HOURS", REFRESH_HOURS)))
|
|
30
|
+
except ValueError:
|
|
31
|
+
return REFRESH_HOURS
|
jscdocs/core/crawl.py
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Build the index from the documentation sites: this is how the published index is made.
|
|
2
|
+
|
|
3
|
+
Installations never do this. They download the index that CI builds with this module (sync.py, published.py), and
|
|
4
|
+
`python -m jscdocs.build` is the only front end that calls it.
|
|
5
|
+
|
|
6
|
+
For each site, a crawl reads objects.inv, fetches every page, extracts its sections and stores them. A site that
|
|
7
|
+
fails, or that returns far fewer pages than the previous index holds (a maintenance page, a broken build), keeps
|
|
8
|
+
its previous rows. A single page that fails is kept as the previous index has it, so that a page which could not
|
|
9
|
+
be fetched this time does not vanish from the index. Sites not asked for are carried over unchanged. When no site
|
|
10
|
+
at all could be updated, nothing is written: the index, or its absence, stays as it was and keeps its sync time,
|
|
11
|
+
so that a failed crawl is never taken for a fresh index.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
import sqlite3
|
|
15
|
+
from collections.abc import Callable, Iterable
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from datetime import UTC, datetime
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
|
|
20
|
+
from .. import __version__
|
|
21
|
+
from . import extract, inventory, lock, sites, store
|
|
22
|
+
from .fetch import Fetcher, FetchError
|
|
23
|
+
|
|
24
|
+
MIN_PAGE_SHARE = 0.5 # a site with fewer pages than this share of its previous count is not replaced
|
|
25
|
+
MAX_FAILED_SHARE = 0.2 # a site where more pages than this share fail to fetch or parse is not replaced
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
@dataclass
|
|
29
|
+
class SiteResult:
|
|
30
|
+
system: str
|
|
31
|
+
replaced: bool # new rows were stored; otherwise the previous ones were kept (if any)
|
|
32
|
+
pages: int = 0
|
|
33
|
+
sections: int = 0
|
|
34
|
+
failed_pages: tuple[str, ...] = () # names of the pages that could not be fetched or converted
|
|
35
|
+
kept_pages: tuple[str, ...] = () # those of them that the index holds as the previous index had them
|
|
36
|
+
error: str = ""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class _Fetched:
|
|
41
|
+
names: list[str] # every page of the site, in the order of its inventory
|
|
42
|
+
pages: dict[str, extract.ExtractedPage] # what was extracted, by page name
|
|
43
|
+
failed: list[str] # page names
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def now() -> str:
|
|
47
|
+
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _fetch_site(site: sites.Site, fetcher: Fetcher, log: Callable[[str], None]) -> _Fetched:
|
|
51
|
+
names = inventory.pages(fetcher.get(site.url("objects.inv")))
|
|
52
|
+
out = _Fetched(names, {}, [])
|
|
53
|
+
for i, name in enumerate(names, 1):
|
|
54
|
+
url = site.url(f"{name}.html")
|
|
55
|
+
try:
|
|
56
|
+
out.pages[name] = extract.extract(fetcher.get(url), url)
|
|
57
|
+
except (FetchError, extract.ExtractError) as e:
|
|
58
|
+
out.failed.append(name)
|
|
59
|
+
log(f"{site.name}: page failed: {e}")
|
|
60
|
+
if i % 10 == 0:
|
|
61
|
+
log(f"{site.name}: {i}/{len(names)} pages")
|
|
62
|
+
return out
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _store_site(w: store.Writer, site: sites.Site, fetched: _Fetched,
|
|
66
|
+
old: sqlite3.Connection | None) -> tuple[int, int, list[str]]:
|
|
67
|
+
"""Store a site's pages. A page that failed is carried over from `old`, the previous index, when that has it.
|
|
68
|
+
|
|
69
|
+
Returns the number of pages and of sections stored, and the names of the pages carried over.
|
|
70
|
+
"""
|
|
71
|
+
notes = extract.site_notes(list(fetched.pages.values()))
|
|
72
|
+
pages = sections = 0
|
|
73
|
+
kept = []
|
|
74
|
+
for name in fetched.names:
|
|
75
|
+
ex = fetched.pages.get(name)
|
|
76
|
+
if ex is not None:
|
|
77
|
+
w.add_page(site.name, name, ex.url, pages)
|
|
78
|
+
rows = [
|
|
79
|
+
store.SectionRow(site.name, name, s.anchor, s.title, " > ".join(s.path), s.depth, i, s.aliases,
|
|
80
|
+
s.markdown)
|
|
81
|
+
for i, s in enumerate(ex.sections)
|
|
82
|
+
]
|
|
83
|
+
w.add_sections(rows)
|
|
84
|
+
count = len(rows)
|
|
85
|
+
else:
|
|
86
|
+
count = w.copy_page(old, site.name, name, pages) if old is not None else None
|
|
87
|
+
if count is None:
|
|
88
|
+
continue # a page the previous index does not have either
|
|
89
|
+
kept.append(name)
|
|
90
|
+
pages += 1
|
|
91
|
+
sections += count
|
|
92
|
+
w.add_system(site.name, site.display, site.base_url, now(), pages, "\n\n".join(notes))
|
|
93
|
+
return pages, sections, kept
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _open_previous(path: Path) -> sqlite3.Connection | None:
|
|
97
|
+
"""The index a crawl carries rows over from, or None when there is none it can read."""
|
|
98
|
+
try:
|
|
99
|
+
return store.connect(path)
|
|
100
|
+
except store.IndexMissing:
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def crawl(path: Path, systems: Iterable[str] | None = None, fetcher: Fetcher | None = None,
|
|
105
|
+
log: Callable[[str], None] = lambda _: None) -> list[SiteResult]:
|
|
106
|
+
"""Fetch the given systems (all by default) and replace the index at `path`.
|
|
107
|
+
|
|
108
|
+
The index is replaced when at least one result has `replaced` set, and left untouched otherwise. Holds the
|
|
109
|
+
index's lock while it runs; raises lock.Busy when another process is writing the same index.
|
|
110
|
+
"""
|
|
111
|
+
with lock.Lock(path):
|
|
112
|
+
return _crawl(path, systems, fetcher, log)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def _crawl(path: Path, systems: Iterable[str] | None, fetcher: Fetcher | None,
|
|
116
|
+
log: Callable[[str], None]) -> list[SiteResult]:
|
|
117
|
+
wanted = [sites.get(s).name for s in systems] if systems else list(sites.SITES)
|
|
118
|
+
fetcher = fetcher or Fetcher()
|
|
119
|
+
old = _open_previous(path)
|
|
120
|
+
w = store.Writer(path)
|
|
121
|
+
results: list[SiteResult] = []
|
|
122
|
+
committed = False
|
|
123
|
+
try:
|
|
124
|
+
for name, site in sites.SITES.items():
|
|
125
|
+
if name not in wanted:
|
|
126
|
+
if old is not None:
|
|
127
|
+
w.copy_system(old, name)
|
|
128
|
+
continue
|
|
129
|
+
log(f"{name}: fetching {site.base_url}")
|
|
130
|
+
result = SiteResult(name, False)
|
|
131
|
+
try:
|
|
132
|
+
fetched = _fetch_site(site, fetcher, log)
|
|
133
|
+
except (FetchError, inventory.InventoryError) as e:
|
|
134
|
+
fetched, result.error = None, str(e)
|
|
135
|
+
if fetched is not None:
|
|
136
|
+
total = len(fetched.pages) + len(fetched.failed)
|
|
137
|
+
previous = old.execute("SELECT page_count FROM systems WHERE name = ?", (name,)).fetchone() if old else None
|
|
138
|
+
result.failed_pages = tuple(fetched.failed)
|
|
139
|
+
if total == 0 or len(fetched.failed) > MAX_FAILED_SHARE * total:
|
|
140
|
+
result.error = f"{len(fetched.failed)} of {total} pages failed"
|
|
141
|
+
elif previous and len(fetched.pages) < MIN_PAGE_SHARE * previous[0]:
|
|
142
|
+
result.error = f"only {len(fetched.pages)} pages, the index has {previous[0]}"
|
|
143
|
+
else:
|
|
144
|
+
result.pages, result.sections, kept = _store_site(w, site, fetched, old)
|
|
145
|
+
result.kept_pages = tuple(kept)
|
|
146
|
+
result.replaced = True
|
|
147
|
+
if not result.replaced and old is not None and w.copy_system(old, name):
|
|
148
|
+
result.error += "; kept the previous pages"
|
|
149
|
+
lost = len(result.failed_pages) - len(result.kept_pages)
|
|
150
|
+
notes = ((f", {len(result.kept_pages)} kept from the previous index" if result.kept_pages else "")
|
|
151
|
+
+ (f", {lost} not indexed" if lost else ""))
|
|
152
|
+
log(f"{name}: " + (f"{result.pages} pages, {result.sections} sections{notes}" if result.replaced
|
|
153
|
+
else f"not updated: {result.error}"))
|
|
154
|
+
results.append(result)
|
|
155
|
+
if any(r.replaced for r in results):
|
|
156
|
+
w.set_meta("jscdocs_version", __version__)
|
|
157
|
+
w.set_meta("license", sites.LICENSE)
|
|
158
|
+
w.set_meta("attribution", sites.ATTRIBUTION)
|
|
159
|
+
w.set_meta("synced_at", now())
|
|
160
|
+
if old is not None:
|
|
161
|
+
old.close()
|
|
162
|
+
old = None
|
|
163
|
+
w.commit()
|
|
164
|
+
committed = True
|
|
165
|
+
finally:
|
|
166
|
+
if old is not None:
|
|
167
|
+
old.close()
|
|
168
|
+
if not committed:
|
|
169
|
+
w.abort()
|
|
170
|
+
return results
|
jscdocs/core/evaluate.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Measure search quality against a file of questions with the sections that answer them.
|
|
2
|
+
|
|
3
|
+
The file is TOML with `[[question]]` tables: `q` (the query), `system` (optional) and `expect` (refs such as
|
|
4
|
+
`jureca/batchsystem#internet-access`; `*` as the system accepts any). A question counts as answered at rank k when
|
|
5
|
+
one of its expected sections is the k-th hit, or the k-th hit carries the same text on that system (`also_on`).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import sqlite3
|
|
9
|
+
import tomllib
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from . import search
|
|
14
|
+
|
|
15
|
+
DEPTH = 10 # hits looked at per question
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Question:
|
|
20
|
+
q: str
|
|
21
|
+
system: str | None
|
|
22
|
+
expect: list[str]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Outcome:
|
|
27
|
+
question: Question
|
|
28
|
+
rank: int | None # 1-based rank of the first expected hit, None if not in the first DEPTH
|
|
29
|
+
page_rank: int | None # the same for a hit on an expected page, any section
|
|
30
|
+
top: list[str] # refs of the first three hits
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@dataclass
|
|
34
|
+
class Report:
|
|
35
|
+
outcomes: list[Outcome]
|
|
36
|
+
|
|
37
|
+
def share(self, k: int, page: bool = False) -> float:
|
|
38
|
+
ranks = [o.page_rank if page else o.rank for o in self.outcomes]
|
|
39
|
+
return sum(1 for r in ranks if r and r <= k) / len(self.outcomes)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def mrr(self) -> float:
|
|
43
|
+
return sum(1 / o.rank for o in self.outcomes if o.rank) / len(self.outcomes)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def load(path: Path) -> list[Question]:
|
|
47
|
+
"""The questions of a file. Raises OSError when it cannot be read, and ValueError when it is not TOML, holds no
|
|
48
|
+
question, or a question lacks `q` or `expect`."""
|
|
49
|
+
with path.open("rb") as f:
|
|
50
|
+
data = tomllib.load(f)
|
|
51
|
+
try:
|
|
52
|
+
questions = [Question(d["q"], d.get("system"), list(d["expect"])) for d in data.get("question", [])]
|
|
53
|
+
except KeyError as e:
|
|
54
|
+
raise ValueError(f"a question lacks {e}") from None
|
|
55
|
+
if not questions:
|
|
56
|
+
raise ValueError("no [[question]] tables")
|
|
57
|
+
return questions
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _matches(ref: str, expect: list[str]) -> bool:
|
|
61
|
+
_, _, rest = ref.partition("/")
|
|
62
|
+
return any(e == ref or e.startswith("*/") and e[2:] == rest for e in expect)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _page_matches(ref: str, expect: list[str]) -> bool:
|
|
66
|
+
page = lambda r: r.partition("#")[0] # noqa: E731
|
|
67
|
+
return _matches(page(ref) + "#", [page(e) + "#" for e in expect])
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _refs(hit: search.Hit) -> list[str]:
|
|
71
|
+
tail = f"{hit.page}#{hit.anchor}"
|
|
72
|
+
return [f"{s}/{tail}" for s in [hit.system, *hit.also_on]]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def run(db: sqlite3.Connection, questions: list[Question]) -> Report:
|
|
76
|
+
outcomes = []
|
|
77
|
+
for qn in questions:
|
|
78
|
+
hits = search.search(db, qn.q, qn.system, limit=DEPTH)
|
|
79
|
+
rank = next((i for i, h in enumerate(hits, 1) if any(_matches(r, qn.expect) for r in _refs(h))), None)
|
|
80
|
+
page_rank = next((i for i, h in enumerate(hits, 1) if any(_page_matches(r, qn.expect) for r in _refs(h))),
|
|
81
|
+
None)
|
|
82
|
+
outcomes.append(Outcome(qn, rank, page_rank, [_refs(h)[0] for h in hits[:3]]))
|
|
83
|
+
return Report(outcomes)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def render(report: Report, verbose: bool = False) -> str:
|
|
87
|
+
n = len(report.outcomes)
|
|
88
|
+
lines = [f"{n} questions: first hit {report.share(1):.0%}, top 3 {report.share(3):.0%}, "
|
|
89
|
+
f"top {DEPTH} {report.share(DEPTH):.0%}, MRR {report.mrr:.2f}; "
|
|
90
|
+
f"right page first {report.share(1, page=True):.0%}, in top 3 {report.share(3, page=True):.0%}"]
|
|
91
|
+
for o in report.outcomes:
|
|
92
|
+
if verbose or not o.rank or o.rank > 1:
|
|
93
|
+
where = f"rank {o.rank}" if o.rank else "missed"
|
|
94
|
+
lines.append(f"\n[{where}] {o.question.q!r}" + (f" ({o.question.system})" if o.question.system else ""))
|
|
95
|
+
lines.append(f" expected: {', '.join(o.question.expect)}")
|
|
96
|
+
lines.extend(f" {i}. {r}" for i, r in enumerate(o.top, 1))
|
|
97
|
+
return "\n".join(lines)
|