jscdocs 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,107 @@
1
+ """Keep the index fresh from a long-running process: sync it in a background thread when it is missing or old.
2
+
3
+ A refresh downloads the published index (sync.py). The server keeps answering from the old index meanwhile, and
4
+ sees the new one when the sync swaps the file in. A refresh that leaves the index as it was is tried again after
5
+ RETRY_AFTER: one that failed, and one that found the published index no newer than the index, which happens when
6
+ the index turns old just before the next one is published. A refresh that finds another process syncing
7
+ (lock.Busy) does nothing, since that process will replace the index.
8
+ """
9
+
10
+ import logging
11
+ import threading
12
+ import time
13
+ from collections.abc import Callable
14
+ from datetime import UTC, datetime
15
+ from pathlib import Path
16
+
17
+ from . import lock, published, sync
18
+
19
+ log = logging.getLogger(__name__)
20
+
21
+ CHECK_EVERY = 600.0 # seconds between staleness checks from tool calls
22
+ RETRY_AFTER = 3600.0 # seconds before a refresh that changed nothing is tried again; the index stays due meanwhile
23
+
24
+
25
+ class Refresher:
26
+ def __init__(self, path: Path, max_age_hours: float,
27
+ run_sync: Callable[..., bool] = sync.sync):
28
+ self.path = path
29
+ self.max_age_hours = max_age_hours # 0: never sync on its own
30
+ self._run_sync = run_sync
31
+ self._thread: threading.Thread | None = None
32
+ self.started: float | None = None
33
+ self.error: str | None = None
34
+ self._put_off: float | None = None # time.monotonic() of the last refresh that left the index due
35
+ self._checked = 0.0
36
+
37
+ @property
38
+ def enabled(self) -> bool:
39
+ return self.max_age_hours > 0
40
+
41
+ @property
42
+ def running(self) -> bool:
43
+ return self._thread is not None and self._thread.is_alive()
44
+
45
+ def due(self) -> bool:
46
+ if not self.enabled:
47
+ return False
48
+ last = sync.synced_at(self.path)
49
+ return last is None or (datetime.now(UTC) - last).total_seconds() > self.max_age_hours * 3600
50
+
51
+ def start_if_due(self) -> bool:
52
+ """Start a background sync when the index is missing or older than max_age_hours. True if one started."""
53
+ now = self._checked = time.monotonic()
54
+ retry_later = self._put_off is not None and now - self._put_off < RETRY_AFTER
55
+ if self.running or retry_later or not self.due():
56
+ return False
57
+ self.started, self.error = time.time(), None
58
+ self._thread = threading.Thread(target=self._run, name="jscdocs-refresh", daemon=True)
59
+ self._thread.start()
60
+ return True
61
+
62
+ def check(self) -> None:
63
+ """start_if_due, at most every CHECK_EVERY seconds: for a server that runs for days."""
64
+ if time.monotonic() - self._checked >= CHECK_EVERY:
65
+ self.start_if_due()
66
+
67
+ def _run(self) -> None:
68
+ log.info("refreshing the documentation index at %s", self.path)
69
+ try:
70
+ replaced = self._run_sync(self.path, log=log.info)
71
+ except lock.Busy as e:
72
+ log.info("not refreshing: %s", e)
73
+ except published.Unavailable as e:
74
+ self._fail(str(e))
75
+ except Exception as e: # noqa: BLE001 # a failed refresh must never take the server down
76
+ self._fail(f"{type(e).__name__}: {e}")
77
+ else:
78
+ if replaced:
79
+ self._put_off = None
80
+ log.info("index refreshed")
81
+ else:
82
+ self._put_off = time.monotonic()
83
+ log.info("the published index is not newer than the index; checking again later")
84
+
85
+ def _fail(self, error: str) -> None:
86
+ self.error, self._put_off = error, time.monotonic()
87
+ log.warning("refreshing the index failed: %s", error)
88
+
89
+ def wait(self, timeout: float | None = None) -> None:
90
+ if self._thread is not None:
91
+ self._thread.join(timeout)
92
+
93
+ def missing_index_message(self) -> str:
94
+ """What a tool says while there is no index."""
95
+ if self.running and self.started is not None:
96
+ since = time.strftime("%H:%M", time.localtime(self.started))
97
+ return (f"The documentation index is being downloaded (started {since}), which takes a few seconds. "
98
+ "Try again shortly.")
99
+ if self.error:
100
+ return (f"The documentation index could not be downloaded ({self.error}). Ask the user to run "
101
+ "`jscdocs sync` and check its output.")
102
+ h = lock.holder(lock.path_for(self.path))
103
+ if h is not None and not lock.stale(h):
104
+ return (f"The documentation index is being downloaded by another process ({h.describe()}). Try again "
105
+ "shortly.")
106
+ return (f"There is no documentation index this server can read at {self.path}. Ask the user to run "
107
+ "`jscdocs sync`, then try again.")
jscdocs/core/search.py ADDED
@@ -0,0 +1,294 @@
1
+ """Queries over the index: search, read a section or page, and list systems and pages.
2
+
3
+ The functions return plain dataclasses; `render_*` turns them into the text the CLI prints and the MCP tools
4
+ return, so both front ends answer alike.
5
+ """
6
+
7
+ import re
8
+ import sqlite3
9
+ from dataclasses import dataclass, field
10
+ from datetime import UTC, datetime
11
+ from urllib.parse import urlparse
12
+
13
+ from . import config, sites
14
+
15
+ _TERM = re.compile(r'"[^"]+"|\S+')
16
+ _WORD = re.compile(r"\w+")
17
+ # Words that carry no meaning in a question. A query is matched by any of its terms, and these would match nearly
18
+ # every section, so they are dropped unless the query has no other term.
19
+ _STOP = frozenset(
20
+ "a an and are as at be by can do does for from how i in is it my of on or the to use using what when where "
21
+ "which who why with".split()
22
+ )
23
+ _SNIPPET_TOKENS = 32
24
+ MAX_LIMIT = 50 # most hits one search returns
25
+ _SUBSECTIONS_SHOWN = 6 # subsection titles listed for a hit that has no text of its own
26
+
27
+ # Ranking parameters, chosen with `jscdocs eval` (tests/eval/questions.toml).
28
+ TITLE_WEIGHT = 8.0 # BM25 column weights: title, breadcrumb, text (1.0)
29
+ BREADCRUMB_WEIGHT = 1.0 # higher let every subsection of a matching page outrank the section with the answer
30
+ CANDIDATES = 200 # sections matching any term that are scored
31
+ # Pages that log changes over years mention nearly every partition, module and option, so they rank below the
32
+ # pages that explain them. They still match, so a question about a past change finds them.
33
+ LOG_PAGES = frozenset({"changelog", "stages-changelog", "maintenance", "JUST6-migration", "data-incident-20210126",
34
+ "buildup-issues"})
35
+ LOG_WEIGHT = 0.4
36
+ ISSUES_WEIGHT = 0.7 # known-issues: often the answer to "why does X fail", rarely to "how do I X"
37
+
38
+
39
+ class NotFound(LookupError):
40
+ pass
41
+
42
+
43
+ @dataclass
44
+ class Hit:
45
+ system: str
46
+ page: str
47
+ anchor: str
48
+ breadcrumb: str
49
+ url: str
50
+ snippet: str # an excerpt of the section's text, or its subsection titles when it has no text of its own
51
+ also_on: list[str] = field(default_factory=list)
52
+
53
+
54
+ @dataclass
55
+ class SystemInfo:
56
+ name: str
57
+ display: str
58
+ base_url: str
59
+ synced_at: str
60
+ page_count: int
61
+ note: str
62
+
63
+
64
+ def _terms(text: str) -> list[str]:
65
+ """The query's terms as FTS5 phrases, without stopwords unless nothing else is left.
66
+
67
+ Each term becomes a quoted phrase of its words, so `dc-gpu`, `$SCRATCH` or `CUDA_VISIBLE_DEVICES` match the
68
+ tokens the indexer made of them, without FTS5 reading `-` or `:` as operators.
69
+ """
70
+ phrases = []
71
+ for term in _TERM.findall(text):
72
+ words = _WORD.findall(term.lower())
73
+ if words:
74
+ phrases.append((" ".join(words), len(words) == 1 and words[0] in _STOP))
75
+ content = [p for p, stop in phrases if not stop]
76
+ return [f'"{p}"' for p in dict.fromkeys(content or [p for p, _ in phrases])]
77
+
78
+
79
+ def _section_url(page_url: str, anchor: str) -> str:
80
+ return f"{page_url}#{anchor}" if anchor else page_url
81
+
82
+
83
+ def _weight(page: str) -> float:
84
+ if page in LOG_PAGES:
85
+ return LOG_WEIGHT
86
+ return ISSUES_WEIGHT if page == "known-issues" else 1.0
87
+
88
+
89
+ def search(db: sqlite3.Connection, query: str, system: str | None = None, limit: int = 8) -> list[Hit]:
90
+ """Ranked sections for a query. Sections with the same text on several systems, in their subsections too, are
91
+ returned once.
92
+
93
+ Candidates are the sections matching any term, scored by BM25 (titles weigh most), times a page weight that
94
+ puts change logs below the pages that explain things. BM25 already favours sections with more of the terms;
95
+ requiring all of them (AND) or weighting the share of terms found both ranked worse in `jscdocs eval`.
96
+ """
97
+ if system:
98
+ system = sites.get(system).name
99
+ limit = max(1, min(limit, MAX_LIMIT))
100
+ terms = _terms(query)
101
+ if not terms:
102
+ return []
103
+ match = " OR ".join(terms)
104
+ where = "sections_fts MATCH ?" + (" AND s.system = ?" if system else "")
105
+ sql = ( # bm25() is negative, and the best match has the lowest value; s.id settles ties the same way each time
106
+ "SELECT s.id, s.system, s.page, s.anchor, s.breadcrumb, s.hash, p.url, bm25(sections_fts, ?, ?, 1.0) AS score"
107
+ " FROM sections_fts JOIN sections s ON s.id = sections_fts.rowid"
108
+ f" JOIN pages p ON p.system = s.system AND p.name = s.page WHERE {where} ORDER BY score, s.id LIMIT ?"
109
+ )
110
+ filt = (system,) if system else ()
111
+ rows = db.execute(sql, (TITLE_WEIGHT, BREADCRUMB_WEIGHT, match, *filt, CANDIDATES)).fetchall()
112
+ rows.sort(key=lambda r: r["score"] * _weight(r["page"]))
113
+
114
+ hits: list[Hit] = []
115
+ ids: list[int] = [] # the section id of each hit
116
+ by_hash: dict[str, Hit] = {}
117
+ for r in rows:
118
+ if r["hash"] in by_hash:
119
+ if r["system"] not in by_hash[r["hash"]].also_on and r["system"] != by_hash[r["hash"]].system:
120
+ by_hash[r["hash"]].also_on.append(r["system"])
121
+ continue
122
+ if len(hits) >= limit:
123
+ continue # keep scanning only to fill also_on
124
+ hit = Hit(r["system"], r["page"], r["anchor"], r["breadcrumb"], _section_url(r["url"], r["anchor"]), "")
125
+ by_hash[r["hash"]] = hit
126
+ hits.append(hit)
127
+ ids.append(r["id"])
128
+ if hits:
129
+ # snippet() takes most of a search's time, so it runs for the hits returned, not for every matching section
130
+ snippets = dict(db.execute(
131
+ f"SELECT rowid, snippet(sections_fts, 2, '**', '**', ' ... ', {_SNIPPET_TOKENS}) FROM sections_fts"
132
+ f" WHERE sections_fts MATCH ? AND rowid IN ({', '.join('?' * len(ids))})", (match, *ids)))
133
+ for hit, id_ in zip(hits, ids):
134
+ hit.snippet = " ".join(snippets[id_].split()) or _subsections(db, id_)
135
+ return hits
136
+
137
+
138
+ def _subsections(db: sqlite3.Connection, section_id: int) -> str:
139
+ """What a hit shows in place of a snippet when all of the section's text is in its subsections: their titles."""
140
+ sec = db.execute("SELECT system, page, ord, depth FROM sections WHERE id = ?", (section_id,)).fetchone()
141
+ below: list[tuple[int, str]] = [] # depth and title of the sections inside this one
142
+ for r in db.execute("SELECT depth, title FROM sections WHERE system = ? AND page = ? AND ord > ? ORDER BY ord",
143
+ (sec["system"], sec["page"], sec["ord"])):
144
+ if r["depth"] <= sec["depth"]:
145
+ break
146
+ below.append((r["depth"], r["title"]))
147
+ if not below:
148
+ return ""
149
+ top = min(depth for depth, _ in below)
150
+ titles = [title for depth, title in below if depth == top and title]
151
+ more = f", and {len(titles) - _SUBSECTIONS_SHOWN} more" if len(titles) > _SUBSECTIONS_SHOWN else ""
152
+ return "Subsections: " + ", ".join(titles[:_SUBSECTIONS_SHOWN]) + more
153
+
154
+
155
+ def parse_ref(ref: str) -> tuple[str, str, str]:
156
+ """(system, page, anchor) from `jureca/batchsystem#partitions`, `jureca/batchsystem.html` or a full URL."""
157
+ ref = ref.strip()
158
+ if ref.startswith(("http://", "https://")):
159
+ u = urlparse(ref)
160
+ base = urlparse(sites.BASE_URL).path
161
+ if not u.path.startswith(base):
162
+ raise NotFound(f"{ref} is not a JSC documentation URL (they start with {sites.BASE_URL})")
163
+ path, anchor = u.path[len(base):], u.fragment
164
+ else:
165
+ path, _, anchor = ref.partition("#")
166
+ system, _, page = path.strip("/").partition("/")
167
+ page = page.removesuffix(".html") or "index"
168
+ return sites.get(system).name, page, anchor
169
+
170
+
171
+ @dataclass
172
+ class ReadResult:
173
+ system: str
174
+ url: str
175
+ breadcrumb: str
176
+ markdown: str # the requested part of the text
177
+ offset: int
178
+ next_offset: int | None # where the next part starts, or None at the end
179
+ note: str
180
+ synced_at: str
181
+
182
+
183
+ def read(db: sqlite3.Connection, ref: str, subsections: bool = True, offset: int = 0,
184
+ max_chars: int = 12000) -> ReadResult:
185
+ """A section (with its subsections unless told otherwise) or a whole page, as Markdown, paged by characters."""
186
+ system, page, anchor = parse_ref(ref)
187
+ rows = db.execute("SELECT anchor, aliases, depth, title, breadcrumb, body, p.url FROM sections s JOIN pages p"
188
+ " ON p.system = s.system AND p.name = s.page WHERE s.system = ? AND s.page = ? ORDER BY s.ord",
189
+ (system, page)).fetchall()
190
+ if not rows:
191
+ raise NotFound(f"no page {page!r} on {system}; list_docs shows the pages")
192
+ start, top_depth = 0, 0
193
+ if anchor:
194
+ match = [i for i, r in enumerate(rows) if r["anchor"] == anchor or anchor in r["aliases"].split()]
195
+ if not match:
196
+ raise NotFound(f"no section #{anchor} on {system}/{page}")
197
+ start, top_depth = match[0], rows[match[0]]["depth"]
198
+ end = start + 1
199
+ if subsections or not anchor:
200
+ while end < len(rows) and (not anchor or rows[end]["depth"] > top_depth):
201
+ end += 1
202
+ parts = []
203
+ for r in rows[start:end]:
204
+ heading = "#" * max(1, min(r["depth"], 6)) + " " + r["title"] if r["title"] else ""
205
+ parts.append("\n\n".join(p for p in (heading, r["body"]) if p))
206
+ text = "\n\n".join(p for p in parts if p)
207
+ offset = max(0, offset)
208
+ chunk = text[offset:offset + max_chars]
209
+ if offset + max_chars < len(text):
210
+ cut = chunk.rfind("\n\n") # end the part at a paragraph break when there is one
211
+ if cut > max_chars // 2:
212
+ chunk = chunk[:cut]
213
+ nxt = offset + len(chunk) if offset + len(chunk) < len(text) else None
214
+ info = system_info(db, system)
215
+ first = rows[start]
216
+ return ReadResult(system, _section_url(first["url"], first["anchor"] if anchor else ""), first["breadcrumb"],
217
+ chunk, offset, nxt, info.note, info.synced_at)
218
+
219
+
220
+ def _info(r: sqlite3.Row) -> SystemInfo:
221
+ return SystemInfo(r["name"], r["display"], r["base_url"], r["synced_at"], r["page_count"], r["note"])
222
+
223
+
224
+ def system_info(db: sqlite3.Connection, system: str) -> SystemInfo:
225
+ r = db.execute("SELECT * FROM systems WHERE name = ?", (system,)).fetchone()
226
+ if r is None:
227
+ raise NotFound(f"{system} is not in the index yet; run `jscdocs sync`")
228
+ return _info(r)
229
+
230
+
231
+ def systems(db: sqlite3.Connection) -> list[SystemInfo]:
232
+ """The indexed systems, in the order of sites.SITES."""
233
+ rows = {r["name"]: r for r in db.execute("SELECT * FROM systems")}
234
+ return [_info(rows[n]) for n in sites.SITES if n in rows]
235
+
236
+
237
+ def toc(db: sqlite3.Connection, system: str, max_depth: int = 2) -> list[tuple[int, str, str]]:
238
+ """(depth, title, ref) of a system's pages and their sections down to `max_depth`."""
239
+ system = sites.get(system).name
240
+ system_info(db, system)
241
+ out = []
242
+ for r in db.execute("SELECT s.page, s.anchor, s.title, s.depth FROM sections s JOIN pages p"
243
+ " ON p.system = s.system AND p.name = s.page WHERE s.system = ? AND s.depth <= ?"
244
+ " ORDER BY p.ord, s.ord", (system, max_depth)):
245
+ if r["title"]:
246
+ out.append((r["depth"], r["title"], f"{system}/{r['page']}" + (f"#{r['anchor']}" if r["depth"] > 1 else "")))
247
+ return out
248
+
249
+
250
+ def age_days(synced_at: str) -> float:
251
+ try:
252
+ then = datetime.fromisoformat(synced_at)
253
+ except ValueError:
254
+ return float("inf")
255
+ return (datetime.now(UTC) - then).total_seconds() / 86400
256
+
257
+
258
+ def _stale(synced_at: str) -> str:
259
+ days = age_days(synced_at)
260
+ return f"Warning: this index is {days:.0f} days old; run `jscdocs sync`.\n" if days > config.STALE_DAYS else ""
261
+
262
+
263
+ def render_hits(db: sqlite3.Connection, hits: list[Hit], query: str) -> str:
264
+ if not hits:
265
+ return f"No sections match {query!r}. Try other words, or list_docs for a system's contents."
266
+ lines = []
267
+ for i, h in enumerate(hits, 1):
268
+ also = f" (same text on {', '.join(sites.get(s).display for s in h.also_on)})" if h.also_on else ""
269
+ lines.append(f"{i}. [{sites.get(h.system).display}] {h.breadcrumb}{also}\n {h.url}"
270
+ + (f"\n {h.snippet}" if h.snippet else ""))
271
+ synced = min(system_info(db, s).synced_at for s in {h.system for h in hits})
272
+ return _stale(synced) + "\n\n".join(lines) + f"\n\nIndex synced {synced[:10]}."
273
+
274
+
275
+ def render_read(r: ReadResult) -> str:
276
+ head = f"Source: {r.url} (synced {r.synced_at[:10]})\n"
277
+ if r.note and r.offset == 0:
278
+ head += f"\nNote for {sites.get(r.system).display}:\n{r.note}\n"
279
+ tail = f"\n\n[More: continue with offset={r.next_offset}]" if r.next_offset is not None else ""
280
+ return _stale(r.synced_at) + head + "\n" + r.markdown + tail
281
+
282
+
283
+ def render_systems(infos: list[SystemInfo]) -> str:
284
+ if not infos:
285
+ return "The index is empty; run `jscdocs sync`."
286
+ lines = []
287
+ for s in infos:
288
+ note = " Has a note (shown with read_docs)." if s.note else ""
289
+ lines.append(f"- {s.name} ({s.display}): {s.page_count} pages, synced {s.synced_at[:10]}, {s.base_url}.{note}")
290
+ return "\n".join(lines)
291
+
292
+
293
+ def render_toc(entries: list[tuple[int, str, str]]) -> str:
294
+ return "\n".join(f"{' ' * (d - 1)}- {t} ({ref})" for d, t, ref in entries)
jscdocs/core/sites.py ADDED
@@ -0,0 +1,55 @@
1
+ """The JSC documentation sites jscdocs indexes.
2
+
3
+ Each system has its own Sphinx build under BASE_URL. The parent directory is not listable (403), so the list is
4
+ fixed here. jsccloud and exastore also have sites but are out of scope.
5
+ """
6
+
7
+ from dataclasses import dataclass
8
+
9
+ BASE_URL = "https://apps.fz-juelich.de/jsc/hps/"
10
+
11
+ # Every index names these two in its `meta` table, and a published index in its manifest too: an index holds the
12
+ # documentation's text, so it is under the documentation's licence, not under the MIT licence of the code.
13
+ LICENSE = "CC BY-NC-SA 4.0 (https://creativecommons.org/licenses/by-nc-sa/4.0/)"
14
+ ATTRIBUTION = (f"User documentation of the JSC HPC systems ({BASE_URL}), by the system administrators at "
15
+ "Forschungszentrum Juelich; converted to Markdown and split into sections by jscdocs")
16
+
17
+
18
+ @dataclass(frozen=True)
19
+ class Site:
20
+ name: str # lower case, as in the URL and in tool arguments
21
+ display: str # as JSC writes it
22
+
23
+ @property
24
+ def base_url(self) -> str:
25
+ return f"{BASE_URL}{self.name}/"
26
+
27
+ def url(self, path: str) -> str:
28
+ """The absolute URL of a path relative to the site, such as `filesystems.html#quota`."""
29
+ return self.base_url + path
30
+
31
+
32
+ SITES: dict[str, Site] = {
33
+ s.name: s
34
+ for s in (
35
+ Site("jureca", "JURECA"),
36
+ Site("juwels", "JUWELS"),
37
+ Site("jusuf", "JUSUF"),
38
+ Site("jupiter", "JUPITER"),
39
+ Site("jedi", "JEDI"),
40
+ Site("judac", "JUDAC"),
41
+ )
42
+ }
43
+
44
+
45
+ class UnknownSystem(ValueError):
46
+ def __init__(self, name: str):
47
+ super().__init__(f"unknown system {name!r}; known systems: {', '.join(SITES)}")
48
+
49
+
50
+ def get(name: str) -> Site:
51
+ """The site of a system, by name in any case. Raises UnknownSystem with the valid names."""
52
+ try:
53
+ return SITES[name.strip().lower()]
54
+ except KeyError:
55
+ raise UnknownSystem(name) from None