jscdocs 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jscdocs/__init__.py +8 -0
- jscdocs/build.py +65 -0
- jscdocs/cli.py +190 -0
- jscdocs/core/__init__.py +1 -0
- jscdocs/core/agent_cli.py +131 -0
- jscdocs/core/agent_config.py +370 -0
- jscdocs/core/config.py +31 -0
- jscdocs/core/crawl.py +170 -0
- jscdocs/core/evaluate.py +97 -0
- jscdocs/core/extract.py +403 -0
- jscdocs/core/fetch.py +100 -0
- jscdocs/core/init.py +382 -0
- jscdocs/core/inventory.py +31 -0
- jscdocs/core/lock.py +125 -0
- jscdocs/core/published.py +125 -0
- jscdocs/core/refresh.py +107 -0
- jscdocs/core/search.py +294 -0
- jscdocs/core/sites.py +55 -0
- jscdocs/core/store.py +233 -0
- jscdocs/core/sync.py +47 -0
- jscdocs/mcp/__init__.py +1 -0
- jscdocs/mcp/server.py +86 -0
- jscdocs/mcp/tools.py +120 -0
- jscdocs-0.2.0.dist-info/METADATA +127 -0
- jscdocs-0.2.0.dist-info/RECORD +28 -0
- jscdocs-0.2.0.dist-info/WHEEL +4 -0
- jscdocs-0.2.0.dist-info/entry_points.txt +3 -0
- jscdocs-0.2.0.dist-info/licenses/LICENSE +21 -0
jscdocs/core/refresh.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Keep the index fresh from a long-running process: sync it in a background thread when it is missing or old.
|
|
2
|
+
|
|
3
|
+
A refresh downloads the published index (sync.py). The server keeps answering from the old index meanwhile, and
|
|
4
|
+
sees the new one when the sync swaps the file in. A refresh that leaves the index as it was is tried again after
|
|
5
|
+
RETRY_AFTER: one that failed, and one that found the published index no newer than the index, which happens when
|
|
6
|
+
the index turns old just before the next one is published. A refresh that finds another process syncing
|
|
7
|
+
(lock.Busy) does nothing, since that process will replace the index.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import threading
|
|
12
|
+
import time
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from datetime import UTC, datetime
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from . import lock, published, sync
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
CHECK_EVERY = 600.0 # seconds between staleness checks from tool calls
|
|
22
|
+
RETRY_AFTER = 3600.0 # seconds before a refresh that changed nothing is tried again; the index stays due meanwhile
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class Refresher:
|
|
26
|
+
def __init__(self, path: Path, max_age_hours: float,
|
|
27
|
+
run_sync: Callable[..., bool] = sync.sync):
|
|
28
|
+
self.path = path
|
|
29
|
+
self.max_age_hours = max_age_hours # 0: never sync on its own
|
|
30
|
+
self._run_sync = run_sync
|
|
31
|
+
self._thread: threading.Thread | None = None
|
|
32
|
+
self.started: float | None = None
|
|
33
|
+
self.error: str | None = None
|
|
34
|
+
self._put_off: float | None = None # time.monotonic() of the last refresh that left the index due
|
|
35
|
+
self._checked = 0.0
|
|
36
|
+
|
|
37
|
+
@property
|
|
38
|
+
def enabled(self) -> bool:
|
|
39
|
+
return self.max_age_hours > 0
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def running(self) -> bool:
|
|
43
|
+
return self._thread is not None and self._thread.is_alive()
|
|
44
|
+
|
|
45
|
+
def due(self) -> bool:
|
|
46
|
+
if not self.enabled:
|
|
47
|
+
return False
|
|
48
|
+
last = sync.synced_at(self.path)
|
|
49
|
+
return last is None or (datetime.now(UTC) - last).total_seconds() > self.max_age_hours * 3600
|
|
50
|
+
|
|
51
|
+
def start_if_due(self) -> bool:
|
|
52
|
+
"""Start a background sync when the index is missing or older than max_age_hours. True if one started."""
|
|
53
|
+
now = self._checked = time.monotonic()
|
|
54
|
+
retry_later = self._put_off is not None and now - self._put_off < RETRY_AFTER
|
|
55
|
+
if self.running or retry_later or not self.due():
|
|
56
|
+
return False
|
|
57
|
+
self.started, self.error = time.time(), None
|
|
58
|
+
self._thread = threading.Thread(target=self._run, name="jscdocs-refresh", daemon=True)
|
|
59
|
+
self._thread.start()
|
|
60
|
+
return True
|
|
61
|
+
|
|
62
|
+
def check(self) -> None:
|
|
63
|
+
"""start_if_due, at most every CHECK_EVERY seconds: for a server that runs for days."""
|
|
64
|
+
if time.monotonic() - self._checked >= CHECK_EVERY:
|
|
65
|
+
self.start_if_due()
|
|
66
|
+
|
|
67
|
+
def _run(self) -> None:
|
|
68
|
+
log.info("refreshing the documentation index at %s", self.path)
|
|
69
|
+
try:
|
|
70
|
+
replaced = self._run_sync(self.path, log=log.info)
|
|
71
|
+
except lock.Busy as e:
|
|
72
|
+
log.info("not refreshing: %s", e)
|
|
73
|
+
except published.Unavailable as e:
|
|
74
|
+
self._fail(str(e))
|
|
75
|
+
except Exception as e: # noqa: BLE001 # a failed refresh must never take the server down
|
|
76
|
+
self._fail(f"{type(e).__name__}: {e}")
|
|
77
|
+
else:
|
|
78
|
+
if replaced:
|
|
79
|
+
self._put_off = None
|
|
80
|
+
log.info("index refreshed")
|
|
81
|
+
else:
|
|
82
|
+
self._put_off = time.monotonic()
|
|
83
|
+
log.info("the published index is not newer than the index; checking again later")
|
|
84
|
+
|
|
85
|
+
def _fail(self, error: str) -> None:
|
|
86
|
+
self.error, self._put_off = error, time.monotonic()
|
|
87
|
+
log.warning("refreshing the index failed: %s", error)
|
|
88
|
+
|
|
89
|
+
def wait(self, timeout: float | None = None) -> None:
|
|
90
|
+
if self._thread is not None:
|
|
91
|
+
self._thread.join(timeout)
|
|
92
|
+
|
|
93
|
+
def missing_index_message(self) -> str:
|
|
94
|
+
"""What a tool says while there is no index."""
|
|
95
|
+
if self.running and self.started is not None:
|
|
96
|
+
since = time.strftime("%H:%M", time.localtime(self.started))
|
|
97
|
+
return (f"The documentation index is being downloaded (started {since}), which takes a few seconds. "
|
|
98
|
+
"Try again shortly.")
|
|
99
|
+
if self.error:
|
|
100
|
+
return (f"The documentation index could not be downloaded ({self.error}). Ask the user to run "
|
|
101
|
+
"`jscdocs sync` and check its output.")
|
|
102
|
+
h = lock.holder(lock.path_for(self.path))
|
|
103
|
+
if h is not None and not lock.stale(h):
|
|
104
|
+
return (f"The documentation index is being downloaded by another process ({h.describe()}). Try again "
|
|
105
|
+
"shortly.")
|
|
106
|
+
return (f"There is no documentation index this server can read at {self.path}. Ask the user to run "
|
|
107
|
+
"`jscdocs sync`, then try again.")
|
jscdocs/core/search.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""Queries over the index: search, read a section or page, and list systems and pages.
|
|
2
|
+
|
|
3
|
+
The functions return plain dataclasses; `render_*` turns them into the text the CLI prints and the MCP tools
|
|
4
|
+
return, so both front ends answer alike.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
import sqlite3
|
|
9
|
+
from dataclasses import dataclass, field
|
|
10
|
+
from datetime import UTC, datetime
|
|
11
|
+
from urllib.parse import urlparse
|
|
12
|
+
|
|
13
|
+
from . import config, sites
|
|
14
|
+
|
|
15
|
+
_TERM = re.compile(r'"[^"]+"|\S+')
|
|
16
|
+
_WORD = re.compile(r"\w+")
|
|
17
|
+
# Words that carry no meaning in a question. A query is matched by any of its terms, and these would match nearly
|
|
18
|
+
# every section, so they are dropped unless the query has no other term.
|
|
19
|
+
_STOP = frozenset(
|
|
20
|
+
"a an and are as at be by can do does for from how i in is it my of on or the to use using what when where "
|
|
21
|
+
"which who why with".split()
|
|
22
|
+
)
|
|
23
|
+
_SNIPPET_TOKENS = 32
|
|
24
|
+
MAX_LIMIT = 50 # most hits one search returns
|
|
25
|
+
_SUBSECTIONS_SHOWN = 6 # subsection titles listed for a hit that has no text of its own
|
|
26
|
+
|
|
27
|
+
# Ranking parameters, chosen with `jscdocs eval` (tests/eval/questions.toml).
|
|
28
|
+
TITLE_WEIGHT = 8.0 # BM25 column weights: title, breadcrumb, text (1.0)
|
|
29
|
+
BREADCRUMB_WEIGHT = 1.0 # higher let every subsection of a matching page outrank the section with the answer
|
|
30
|
+
CANDIDATES = 200 # sections matching any term that are scored
|
|
31
|
+
# Pages that log changes over years mention nearly every partition, module and option, so they rank below the
|
|
32
|
+
# pages that explain them. They still match, so a question about a past change finds them.
|
|
33
|
+
LOG_PAGES = frozenset({"changelog", "stages-changelog", "maintenance", "JUST6-migration", "data-incident-20210126",
|
|
34
|
+
"buildup-issues"})
|
|
35
|
+
LOG_WEIGHT = 0.4
|
|
36
|
+
ISSUES_WEIGHT = 0.7 # known-issues: often the answer to "why does X fail", rarely to "how do I X"
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class NotFound(LookupError):
|
|
40
|
+
pass
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class Hit:
|
|
45
|
+
system: str
|
|
46
|
+
page: str
|
|
47
|
+
anchor: str
|
|
48
|
+
breadcrumb: str
|
|
49
|
+
url: str
|
|
50
|
+
snippet: str # an excerpt of the section's text, or its subsection titles when it has no text of its own
|
|
51
|
+
also_on: list[str] = field(default_factory=list)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass
|
|
55
|
+
class SystemInfo:
|
|
56
|
+
name: str
|
|
57
|
+
display: str
|
|
58
|
+
base_url: str
|
|
59
|
+
synced_at: str
|
|
60
|
+
page_count: int
|
|
61
|
+
note: str
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _terms(text: str) -> list[str]:
|
|
65
|
+
"""The query's terms as FTS5 phrases, without stopwords unless nothing else is left.
|
|
66
|
+
|
|
67
|
+
Each term becomes a quoted phrase of its words, so `dc-gpu`, `$SCRATCH` or `CUDA_VISIBLE_DEVICES` match the
|
|
68
|
+
tokens the indexer made of them, without FTS5 reading `-` or `:` as operators.
|
|
69
|
+
"""
|
|
70
|
+
phrases = []
|
|
71
|
+
for term in _TERM.findall(text):
|
|
72
|
+
words = _WORD.findall(term.lower())
|
|
73
|
+
if words:
|
|
74
|
+
phrases.append((" ".join(words), len(words) == 1 and words[0] in _STOP))
|
|
75
|
+
content = [p for p, stop in phrases if not stop]
|
|
76
|
+
return [f'"{p}"' for p in dict.fromkeys(content or [p for p, _ in phrases])]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _section_url(page_url: str, anchor: str) -> str:
|
|
80
|
+
return f"{page_url}#{anchor}" if anchor else page_url
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _weight(page: str) -> float:
|
|
84
|
+
if page in LOG_PAGES:
|
|
85
|
+
return LOG_WEIGHT
|
|
86
|
+
return ISSUES_WEIGHT if page == "known-issues" else 1.0
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def search(db: sqlite3.Connection, query: str, system: str | None = None, limit: int = 8) -> list[Hit]:
|
|
90
|
+
"""Ranked sections for a query. Sections with the same text on several systems, in their subsections too, are
|
|
91
|
+
returned once.
|
|
92
|
+
|
|
93
|
+
Candidates are the sections matching any term, scored by BM25 (titles weigh most), times a page weight that
|
|
94
|
+
puts change logs below the pages that explain things. BM25 already favours sections with more of the terms;
|
|
95
|
+
requiring all of them (AND) or weighting the share of terms found both ranked worse in `jscdocs eval`.
|
|
96
|
+
"""
|
|
97
|
+
if system:
|
|
98
|
+
system = sites.get(system).name
|
|
99
|
+
limit = max(1, min(limit, MAX_LIMIT))
|
|
100
|
+
terms = _terms(query)
|
|
101
|
+
if not terms:
|
|
102
|
+
return []
|
|
103
|
+
match = " OR ".join(terms)
|
|
104
|
+
where = "sections_fts MATCH ?" + (" AND s.system = ?" if system else "")
|
|
105
|
+
sql = ( # bm25() is negative, and the best match has the lowest value; s.id settles ties the same way each time
|
|
106
|
+
"SELECT s.id, s.system, s.page, s.anchor, s.breadcrumb, s.hash, p.url, bm25(sections_fts, ?, ?, 1.0) AS score"
|
|
107
|
+
" FROM sections_fts JOIN sections s ON s.id = sections_fts.rowid"
|
|
108
|
+
f" JOIN pages p ON p.system = s.system AND p.name = s.page WHERE {where} ORDER BY score, s.id LIMIT ?"
|
|
109
|
+
)
|
|
110
|
+
filt = (system,) if system else ()
|
|
111
|
+
rows = db.execute(sql, (TITLE_WEIGHT, BREADCRUMB_WEIGHT, match, *filt, CANDIDATES)).fetchall()
|
|
112
|
+
rows.sort(key=lambda r: r["score"] * _weight(r["page"]))
|
|
113
|
+
|
|
114
|
+
hits: list[Hit] = []
|
|
115
|
+
ids: list[int] = [] # the section id of each hit
|
|
116
|
+
by_hash: dict[str, Hit] = {}
|
|
117
|
+
for r in rows:
|
|
118
|
+
if r["hash"] in by_hash:
|
|
119
|
+
if r["system"] not in by_hash[r["hash"]].also_on and r["system"] != by_hash[r["hash"]].system:
|
|
120
|
+
by_hash[r["hash"]].also_on.append(r["system"])
|
|
121
|
+
continue
|
|
122
|
+
if len(hits) >= limit:
|
|
123
|
+
continue # keep scanning only to fill also_on
|
|
124
|
+
hit = Hit(r["system"], r["page"], r["anchor"], r["breadcrumb"], _section_url(r["url"], r["anchor"]), "")
|
|
125
|
+
by_hash[r["hash"]] = hit
|
|
126
|
+
hits.append(hit)
|
|
127
|
+
ids.append(r["id"])
|
|
128
|
+
if hits:
|
|
129
|
+
# snippet() takes most of a search's time, so it runs for the hits returned, not for every matching section
|
|
130
|
+
snippets = dict(db.execute(
|
|
131
|
+
f"SELECT rowid, snippet(sections_fts, 2, '**', '**', ' ... ', {_SNIPPET_TOKENS}) FROM sections_fts"
|
|
132
|
+
f" WHERE sections_fts MATCH ? AND rowid IN ({', '.join('?' * len(ids))})", (match, *ids)))
|
|
133
|
+
for hit, id_ in zip(hits, ids):
|
|
134
|
+
hit.snippet = " ".join(snippets[id_].split()) or _subsections(db, id_)
|
|
135
|
+
return hits
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _subsections(db: sqlite3.Connection, section_id: int) -> str:
|
|
139
|
+
"""What a hit shows in place of a snippet when all of the section's text is in its subsections: their titles."""
|
|
140
|
+
sec = db.execute("SELECT system, page, ord, depth FROM sections WHERE id = ?", (section_id,)).fetchone()
|
|
141
|
+
below: list[tuple[int, str]] = [] # depth and title of the sections inside this one
|
|
142
|
+
for r in db.execute("SELECT depth, title FROM sections WHERE system = ? AND page = ? AND ord > ? ORDER BY ord",
|
|
143
|
+
(sec["system"], sec["page"], sec["ord"])):
|
|
144
|
+
if r["depth"] <= sec["depth"]:
|
|
145
|
+
break
|
|
146
|
+
below.append((r["depth"], r["title"]))
|
|
147
|
+
if not below:
|
|
148
|
+
return ""
|
|
149
|
+
top = min(depth for depth, _ in below)
|
|
150
|
+
titles = [title for depth, title in below if depth == top and title]
|
|
151
|
+
more = f", and {len(titles) - _SUBSECTIONS_SHOWN} more" if len(titles) > _SUBSECTIONS_SHOWN else ""
|
|
152
|
+
return "Subsections: " + ", ".join(titles[:_SUBSECTIONS_SHOWN]) + more
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def parse_ref(ref: str) -> tuple[str, str, str]:
|
|
156
|
+
"""(system, page, anchor) from `jureca/batchsystem#partitions`, `jureca/batchsystem.html` or a full URL."""
|
|
157
|
+
ref = ref.strip()
|
|
158
|
+
if ref.startswith(("http://", "https://")):
|
|
159
|
+
u = urlparse(ref)
|
|
160
|
+
base = urlparse(sites.BASE_URL).path
|
|
161
|
+
if not u.path.startswith(base):
|
|
162
|
+
raise NotFound(f"{ref} is not a JSC documentation URL (they start with {sites.BASE_URL})")
|
|
163
|
+
path, anchor = u.path[len(base):], u.fragment
|
|
164
|
+
else:
|
|
165
|
+
path, _, anchor = ref.partition("#")
|
|
166
|
+
system, _, page = path.strip("/").partition("/")
|
|
167
|
+
page = page.removesuffix(".html") or "index"
|
|
168
|
+
return sites.get(system).name, page, anchor
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@dataclass
|
|
172
|
+
class ReadResult:
|
|
173
|
+
system: str
|
|
174
|
+
url: str
|
|
175
|
+
breadcrumb: str
|
|
176
|
+
markdown: str # the requested part of the text
|
|
177
|
+
offset: int
|
|
178
|
+
next_offset: int | None # where the next part starts, or None at the end
|
|
179
|
+
note: str
|
|
180
|
+
synced_at: str
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def read(db: sqlite3.Connection, ref: str, subsections: bool = True, offset: int = 0,
|
|
184
|
+
max_chars: int = 12000) -> ReadResult:
|
|
185
|
+
"""A section (with its subsections unless told otherwise) or a whole page, as Markdown, paged by characters."""
|
|
186
|
+
system, page, anchor = parse_ref(ref)
|
|
187
|
+
rows = db.execute("SELECT anchor, aliases, depth, title, breadcrumb, body, p.url FROM sections s JOIN pages p"
|
|
188
|
+
" ON p.system = s.system AND p.name = s.page WHERE s.system = ? AND s.page = ? ORDER BY s.ord",
|
|
189
|
+
(system, page)).fetchall()
|
|
190
|
+
if not rows:
|
|
191
|
+
raise NotFound(f"no page {page!r} on {system}; list_docs shows the pages")
|
|
192
|
+
start, top_depth = 0, 0
|
|
193
|
+
if anchor:
|
|
194
|
+
match = [i for i, r in enumerate(rows) if r["anchor"] == anchor or anchor in r["aliases"].split()]
|
|
195
|
+
if not match:
|
|
196
|
+
raise NotFound(f"no section #{anchor} on {system}/{page}")
|
|
197
|
+
start, top_depth = match[0], rows[match[0]]["depth"]
|
|
198
|
+
end = start + 1
|
|
199
|
+
if subsections or not anchor:
|
|
200
|
+
while end < len(rows) and (not anchor or rows[end]["depth"] > top_depth):
|
|
201
|
+
end += 1
|
|
202
|
+
parts = []
|
|
203
|
+
for r in rows[start:end]:
|
|
204
|
+
heading = "#" * max(1, min(r["depth"], 6)) + " " + r["title"] if r["title"] else ""
|
|
205
|
+
parts.append("\n\n".join(p for p in (heading, r["body"]) if p))
|
|
206
|
+
text = "\n\n".join(p for p in parts if p)
|
|
207
|
+
offset = max(0, offset)
|
|
208
|
+
chunk = text[offset:offset + max_chars]
|
|
209
|
+
if offset + max_chars < len(text):
|
|
210
|
+
cut = chunk.rfind("\n\n") # end the part at a paragraph break when there is one
|
|
211
|
+
if cut > max_chars // 2:
|
|
212
|
+
chunk = chunk[:cut]
|
|
213
|
+
nxt = offset + len(chunk) if offset + len(chunk) < len(text) else None
|
|
214
|
+
info = system_info(db, system)
|
|
215
|
+
first = rows[start]
|
|
216
|
+
return ReadResult(system, _section_url(first["url"], first["anchor"] if anchor else ""), first["breadcrumb"],
|
|
217
|
+
chunk, offset, nxt, info.note, info.synced_at)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _info(r: sqlite3.Row) -> SystemInfo:
|
|
221
|
+
return SystemInfo(r["name"], r["display"], r["base_url"], r["synced_at"], r["page_count"], r["note"])
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def system_info(db: sqlite3.Connection, system: str) -> SystemInfo:
|
|
225
|
+
r = db.execute("SELECT * FROM systems WHERE name = ?", (system,)).fetchone()
|
|
226
|
+
if r is None:
|
|
227
|
+
raise NotFound(f"{system} is not in the index yet; run `jscdocs sync`")
|
|
228
|
+
return _info(r)
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def systems(db: sqlite3.Connection) -> list[SystemInfo]:
|
|
232
|
+
"""The indexed systems, in the order of sites.SITES."""
|
|
233
|
+
rows = {r["name"]: r for r in db.execute("SELECT * FROM systems")}
|
|
234
|
+
return [_info(rows[n]) for n in sites.SITES if n in rows]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def toc(db: sqlite3.Connection, system: str, max_depth: int = 2) -> list[tuple[int, str, str]]:
|
|
238
|
+
"""(depth, title, ref) of a system's pages and their sections down to `max_depth`."""
|
|
239
|
+
system = sites.get(system).name
|
|
240
|
+
system_info(db, system)
|
|
241
|
+
out = []
|
|
242
|
+
for r in db.execute("SELECT s.page, s.anchor, s.title, s.depth FROM sections s JOIN pages p"
|
|
243
|
+
" ON p.system = s.system AND p.name = s.page WHERE s.system = ? AND s.depth <= ?"
|
|
244
|
+
" ORDER BY p.ord, s.ord", (system, max_depth)):
|
|
245
|
+
if r["title"]:
|
|
246
|
+
out.append((r["depth"], r["title"], f"{system}/{r['page']}" + (f"#{r['anchor']}" if r["depth"] > 1 else "")))
|
|
247
|
+
return out
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def age_days(synced_at: str) -> float:
|
|
251
|
+
try:
|
|
252
|
+
then = datetime.fromisoformat(synced_at)
|
|
253
|
+
except ValueError:
|
|
254
|
+
return float("inf")
|
|
255
|
+
return (datetime.now(UTC) - then).total_seconds() / 86400
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def _stale(synced_at: str) -> str:
|
|
259
|
+
days = age_days(synced_at)
|
|
260
|
+
return f"Warning: this index is {days:.0f} days old; run `jscdocs sync`.\n" if days > config.STALE_DAYS else ""
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def render_hits(db: sqlite3.Connection, hits: list[Hit], query: str) -> str:
|
|
264
|
+
if not hits:
|
|
265
|
+
return f"No sections match {query!r}. Try other words, or list_docs for a system's contents."
|
|
266
|
+
lines = []
|
|
267
|
+
for i, h in enumerate(hits, 1):
|
|
268
|
+
also = f" (same text on {', '.join(sites.get(s).display for s in h.also_on)})" if h.also_on else ""
|
|
269
|
+
lines.append(f"{i}. [{sites.get(h.system).display}] {h.breadcrumb}{also}\n {h.url}"
|
|
270
|
+
+ (f"\n {h.snippet}" if h.snippet else ""))
|
|
271
|
+
synced = min(system_info(db, s).synced_at for s in {h.system for h in hits})
|
|
272
|
+
return _stale(synced) + "\n\n".join(lines) + f"\n\nIndex synced {synced[:10]}."
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def render_read(r: ReadResult) -> str:
|
|
276
|
+
head = f"Source: {r.url} (synced {r.synced_at[:10]})\n"
|
|
277
|
+
if r.note and r.offset == 0:
|
|
278
|
+
head += f"\nNote for {sites.get(r.system).display}:\n{r.note}\n"
|
|
279
|
+
tail = f"\n\n[More: continue with offset={r.next_offset}]" if r.next_offset is not None else ""
|
|
280
|
+
return _stale(r.synced_at) + head + "\n" + r.markdown + tail
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
def render_systems(infos: list[SystemInfo]) -> str:
|
|
284
|
+
if not infos:
|
|
285
|
+
return "The index is empty; run `jscdocs sync`."
|
|
286
|
+
lines = []
|
|
287
|
+
for s in infos:
|
|
288
|
+
note = " Has a note (shown with read_docs)." if s.note else ""
|
|
289
|
+
lines.append(f"- {s.name} ({s.display}): {s.page_count} pages, synced {s.synced_at[:10]}, {s.base_url}.{note}")
|
|
290
|
+
return "\n".join(lines)
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def render_toc(entries: list[tuple[int, str, str]]) -> str:
|
|
294
|
+
return "\n".join(f"{' ' * (d - 1)}- {t} ({ref})" for d, t, ref in entries)
|
jscdocs/core/sites.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""The JSC documentation sites jscdocs indexes.
|
|
2
|
+
|
|
3
|
+
Each system has its own Sphinx build under BASE_URL. The parent directory is not listable (403), so the list is
|
|
4
|
+
fixed here. jsccloud and exastore also have sites but are out of scope.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
|
|
9
|
+
BASE_URL = "https://apps.fz-juelich.de/jsc/hps/"
|
|
10
|
+
|
|
11
|
+
# Every index names these two in its `meta` table, and a published index in its manifest too: an index holds the
|
|
12
|
+
# documentation's text, so it is under the documentation's licence, not under the MIT licence of the code.
|
|
13
|
+
LICENSE = "CC BY-NC-SA 4.0 (https://creativecommons.org/licenses/by-nc-sa/4.0/)"
|
|
14
|
+
ATTRIBUTION = (f"User documentation of the JSC HPC systems ({BASE_URL}), by the system administrators at "
|
|
15
|
+
"Forschungszentrum Juelich; converted to Markdown and split into sections by jscdocs")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(frozen=True)
|
|
19
|
+
class Site:
|
|
20
|
+
name: str # lower case, as in the URL and in tool arguments
|
|
21
|
+
display: str # as JSC writes it
|
|
22
|
+
|
|
23
|
+
@property
|
|
24
|
+
def base_url(self) -> str:
|
|
25
|
+
return f"{BASE_URL}{self.name}/"
|
|
26
|
+
|
|
27
|
+
def url(self, path: str) -> str:
|
|
28
|
+
"""The absolute URL of a path relative to the site, such as `filesystems.html#quota`."""
|
|
29
|
+
return self.base_url + path
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
SITES: dict[str, Site] = {
|
|
33
|
+
s.name: s
|
|
34
|
+
for s in (
|
|
35
|
+
Site("jureca", "JURECA"),
|
|
36
|
+
Site("juwels", "JUWELS"),
|
|
37
|
+
Site("jusuf", "JUSUF"),
|
|
38
|
+
Site("jupiter", "JUPITER"),
|
|
39
|
+
Site("jedi", "JEDI"),
|
|
40
|
+
Site("judac", "JUDAC"),
|
|
41
|
+
)
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class UnknownSystem(ValueError):
|
|
46
|
+
def __init__(self, name: str):
|
|
47
|
+
super().__init__(f"unknown system {name!r}; known systems: {', '.join(SITES)}")
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def get(name: str) -> Site:
|
|
51
|
+
"""The site of a system, by name in any case. Raises UnknownSystem with the valid names."""
|
|
52
|
+
try:
|
|
53
|
+
return SITES[name.strip().lower()]
|
|
54
|
+
except KeyError:
|
|
55
|
+
raise UnknownSystem(name) from None
|