jscdocs 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jscdocs/__init__.py +8 -0
- jscdocs/build.py +65 -0
- jscdocs/cli.py +190 -0
- jscdocs/core/__init__.py +1 -0
- jscdocs/core/agent_cli.py +131 -0
- jscdocs/core/agent_config.py +370 -0
- jscdocs/core/config.py +31 -0
- jscdocs/core/crawl.py +170 -0
- jscdocs/core/evaluate.py +97 -0
- jscdocs/core/extract.py +403 -0
- jscdocs/core/fetch.py +100 -0
- jscdocs/core/init.py +382 -0
- jscdocs/core/inventory.py +31 -0
- jscdocs/core/lock.py +125 -0
- jscdocs/core/published.py +125 -0
- jscdocs/core/refresh.py +107 -0
- jscdocs/core/search.py +294 -0
- jscdocs/core/sites.py +55 -0
- jscdocs/core/store.py +233 -0
- jscdocs/core/sync.py +47 -0
- jscdocs/mcp/__init__.py +1 -0
- jscdocs/mcp/server.py +86 -0
- jscdocs/mcp/tools.py +120 -0
- jscdocs-0.2.0.dist-info/METADATA +127 -0
- jscdocs-0.2.0.dist-info/RECORD +28 -0
- jscdocs-0.2.0.dist-info/WHEEL +4 -0
- jscdocs-0.2.0.dist-info/entry_points.txt +3 -0
- jscdocs-0.2.0.dist-info/licenses/LICENSE +21 -0
jscdocs/core/extract.py
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"""Turn a rendered Sphinx page into sections of Markdown.
|
|
2
|
+
|
|
3
|
+
Only `div[itemprop="articleBody"]` is read. Each `<section id>` becomes one Section holding its own content as
|
|
4
|
+
Markdown blocks, without its subsections, so that a search hit points at the smallest matching section. The
|
|
5
|
+
converter is written for the markup Sphinx and the Read the Docs theme produce:
|
|
6
|
+
|
|
7
|
+
- Tables are expanded into a full grid. Sphinx writes a multi-line grid-table row as one row of `rowspan` cells
|
|
8
|
+
followed by empty `<tr>`s, so rows without cells of their own are dropped. Tables whose cells are single short
|
|
9
|
+
lines become Markdown tables, others one record per row.
|
|
10
|
+
- Admonitions become quoted blocks, `> **Note:** ...`.
|
|
11
|
+
- Highlighted code becomes a fenced block, with the language from the `highlight-<lang>` class.
|
|
12
|
+
- Links become absolute URLs, so an agent can pass them on; heading anchor links (`a.headerlink`) are dropped.
|
|
13
|
+
|
|
14
|
+
Banners repeated on every page of a site, such as the JUPITER Early Access warning, are found by `site_notes`
|
|
15
|
+
across a site's pages, not here.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import re
|
|
19
|
+
from collections import Counter
|
|
20
|
+
from collections.abc import Iterable, Sequence
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from urllib.parse import urljoin
|
|
23
|
+
|
|
24
|
+
from bs4 import BeautifulSoup, NavigableString, Tag
|
|
25
|
+
|
|
26
|
+
_WS = re.compile(r"\s+")
|
|
27
|
+
_INT = re.compile(r"\s*(-?\d+)")
|
|
28
|
+
_HEADINGS = ("h1", "h2", "h3", "h4", "h5", "h6")
|
|
29
|
+
_INLINE_TABLE_MAX = 120 # longest cell that still fits a Markdown table row
|
|
30
|
+
_MAX_COLSPAN = 1000 # as browsers cap it; a larger value is a mistake in the page
|
|
31
|
+
_NOTE_SHARE = 0.5 # an admonition on at least this share of a site's pages is a banner
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ExtractError(ValueError):
|
|
35
|
+
pass
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class Section:
|
|
40
|
+
anchor: str # the section's HTML id; "" for content before the first section
|
|
41
|
+
title: str # "" for content before the first section, unless it stands in for a missing page title
|
|
42
|
+
depth: int # 1 for the page title (h1)
|
|
43
|
+
path: tuple[str, ...] # titles from the page title down to this section
|
|
44
|
+
aliases: tuple[str, ...] = () # other ids that point at this section (Sphinx labels)
|
|
45
|
+
blocks: list[str] = field(default_factory=list) # Markdown blocks of the section's own content
|
|
46
|
+
|
|
47
|
+
@property
|
|
48
|
+
def markdown(self) -> str:
|
|
49
|
+
return "\n\n".join(self.blocks)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
@dataclass
|
|
53
|
+
class ExtractedPage:
|
|
54
|
+
url: str
|
|
55
|
+
title: str
|
|
56
|
+
sections: list[Section]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def extract(html: str | bytes, url: str) -> ExtractedPage:
|
|
60
|
+
"""Split a rendered page into sections. `url` is the page's address, used to make links absolute.
|
|
61
|
+
|
|
62
|
+
Raises ExtractError when the page has no article body or cannot be converted.
|
|
63
|
+
"""
|
|
64
|
+
try:
|
|
65
|
+
return _extract(html, url)
|
|
66
|
+
except ExtractError:
|
|
67
|
+
raise
|
|
68
|
+
except Exception as e: # noqa: BLE001 # unforeseen markup must fail this page, not the whole sync
|
|
69
|
+
raise ExtractError(f"{url}: cannot convert the page ({type(e).__name__}: {e})") from e
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _extract(html: str | bytes, url: str) -> ExtractedPage:
|
|
73
|
+
soup = BeautifulSoup(html, "html.parser")
|
|
74
|
+
body = soup.select_one('div[itemprop="articleBody"]')
|
|
75
|
+
if body is None:
|
|
76
|
+
raise ExtractError(f"{url}: no article body (not a Sphinx page?)")
|
|
77
|
+
for a in body.select("a.headerlink"):
|
|
78
|
+
a.decompose()
|
|
79
|
+
conv = _Converter(url)
|
|
80
|
+
sections: list[Section] = []
|
|
81
|
+
lead = Section("", "", 0, ())
|
|
82
|
+
_walk(body, conv, sections, lead, (), set())
|
|
83
|
+
h1 = next((s.title for s in sections if s.depth == 1), "")
|
|
84
|
+
title = h1 or (soup.title.get_text(strip=True) if soup.title else "")
|
|
85
|
+
if lead.blocks:
|
|
86
|
+
lead.path = (title,)
|
|
87
|
+
if not h1: # no section holds the page title, so the lead does; otherwise the title would be shown twice
|
|
88
|
+
lead.title, lead.depth = title, 1
|
|
89
|
+
sections.insert(0, lead)
|
|
90
|
+
return ExtractedPage(url, title, sections)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _walk(node: Tag, conv: "_Converter", out: list[Section], current: Section, path: tuple[str, ...],
|
|
94
|
+
seen: set[str]) -> None:
|
|
95
|
+
"""Add the blocks under `node` to `current`, starting a new Section at each <section>.
|
|
96
|
+
|
|
97
|
+
`seen` holds the page's anchors so far. A section needs an id of its own to be found and read, so one without
|
|
98
|
+
an id, or with an id already used, stays part of the enclosing section, under its title in bold.
|
|
99
|
+
"""
|
|
100
|
+
for child in node.children:
|
|
101
|
+
if isinstance(child, Tag) and child.name == "section":
|
|
102
|
+
heading = child.find(_HEADINGS, recursive=False)
|
|
103
|
+
title = conv.inline(heading) if heading else ""
|
|
104
|
+
anchor = child.get("id", "")
|
|
105
|
+
if not anchor or anchor in seen:
|
|
106
|
+
if title:
|
|
107
|
+
current.blocks.append(f"**{title}**")
|
|
108
|
+
_walk(child, conv, out, current, path, seen)
|
|
109
|
+
continue
|
|
110
|
+
seen.add(anchor)
|
|
111
|
+
depth = int(heading.name[1]) if heading else current.depth + 1
|
|
112
|
+
aliases = tuple(
|
|
113
|
+
s["id"] for s in child.find_all("span", recursive=False) if s.get("id") and not s.get_text(strip=True)
|
|
114
|
+
)
|
|
115
|
+
sec = Section(anchor, title, depth, path + (title,), aliases)
|
|
116
|
+
out.append(sec)
|
|
117
|
+
_walk(child, conv, out, sec, sec.path, seen)
|
|
118
|
+
elif isinstance(child, Tag) and child.name in _HEADINGS:
|
|
119
|
+
continue # the section's own title, stored in Section.title
|
|
120
|
+
else:
|
|
121
|
+
current.blocks.extend(conv.blocks(child))
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _Converter:
|
|
125
|
+
def __init__(self, url: str):
|
|
126
|
+
self.url = url
|
|
127
|
+
|
|
128
|
+
# Block level: each returns a list of Markdown blocks.
|
|
129
|
+
|
|
130
|
+
def blocks(self, node) -> list[str]:
|
|
131
|
+
if isinstance(node, NavigableString):
|
|
132
|
+
text = _WS.sub(" ", str(node)).strip()
|
|
133
|
+
return [text] if text and type(node) is NavigableString else []
|
|
134
|
+
if not isinstance(node, Tag):
|
|
135
|
+
return []
|
|
136
|
+
name, classes = node.name, node.get("class", [])
|
|
137
|
+
if name in ("script", "style", "nav"):
|
|
138
|
+
return []
|
|
139
|
+
if name == "p":
|
|
140
|
+
if "admonition-title" in classes:
|
|
141
|
+
return []
|
|
142
|
+
text = self.inline(node)
|
|
143
|
+
if "rubric" in classes and text:
|
|
144
|
+
text = f"**{text}**"
|
|
145
|
+
return [text] if text else []
|
|
146
|
+
if name in ("ul", "ol"):
|
|
147
|
+
return [self.list_block(node)]
|
|
148
|
+
if name == "dl":
|
|
149
|
+
return [self.definition_list(node)]
|
|
150
|
+
if name == "table":
|
|
151
|
+
return [self.table(node)]
|
|
152
|
+
if name == "pre":
|
|
153
|
+
return [self.code(node.get_text(), "")]
|
|
154
|
+
if name == "div" and any(c.startswith("highlight-") for c in classes):
|
|
155
|
+
lang = next(c for c in classes if c.startswith("highlight-")).removeprefix("highlight-")
|
|
156
|
+
pre = node.find("pre")
|
|
157
|
+
return [self.code(pre.get_text() if pre else node.get_text(), lang)]
|
|
158
|
+
if "admonition" in classes or name == "aside" and "topic" in classes:
|
|
159
|
+
return [self.admonition(node)]
|
|
160
|
+
if name == "div" and "line-block" in classes:
|
|
161
|
+
lines = [self.inline(line) for line in node.find_all("div", class_="line")]
|
|
162
|
+
return ["\n".join(line for line in lines if line)]
|
|
163
|
+
if name == "aside" and "footnote" in classes:
|
|
164
|
+
label = node.find("span", class_="label")
|
|
165
|
+
num = label.get_text(strip=True).strip("[]") if label else ""
|
|
166
|
+
if label:
|
|
167
|
+
label.decompose()
|
|
168
|
+
text = " ".join(b for b in self.children_blocks(node) if b)
|
|
169
|
+
return [f"[{num}] {text}" if num else text]
|
|
170
|
+
if name == "blockquote":
|
|
171
|
+
inner = self.children_blocks(node)
|
|
172
|
+
return ["\n".join("> " + line if line else ">" for line in "\n\n".join(inner).splitlines())] if inner else []
|
|
173
|
+
if name == "figure":
|
|
174
|
+
return self.figure(node)
|
|
175
|
+
if name == "img":
|
|
176
|
+
return [self.image(node)]
|
|
177
|
+
if name == "hr":
|
|
178
|
+
return []
|
|
179
|
+
if name == "span" and not node.get_text(strip=True):
|
|
180
|
+
return [] # label targets such as <span id="filesystems"></span>
|
|
181
|
+
if name in ("div", "aside", "section", "article", "center", "main"):
|
|
182
|
+
return self.children_blocks(node)
|
|
183
|
+
text = self.inline(node)
|
|
184
|
+
return [text] if text else []
|
|
185
|
+
|
|
186
|
+
def children_blocks(self, node: Tag) -> list[str]:
|
|
187
|
+
out: list[str] = []
|
|
188
|
+
for child in node.children:
|
|
189
|
+
out.extend(self.blocks(child))
|
|
190
|
+
return out
|
|
191
|
+
|
|
192
|
+
def list_block(self, node: Tag) -> str:
|
|
193
|
+
ordered = node.name == "ol"
|
|
194
|
+
start = _int(node, "start") if ordered else 1
|
|
195
|
+
items = []
|
|
196
|
+
for i, li in enumerate(node.find_all("li", recursive=False)):
|
|
197
|
+
marker = f"{start + i}." if ordered else "-"
|
|
198
|
+
items.append(_indent_item(marker, self.children_blocks(li)))
|
|
199
|
+
return "\n".join(items)
|
|
200
|
+
|
|
201
|
+
def definition_list(self, node: Tag) -> str:
|
|
202
|
+
# One item per run of terms with the definitions that follow it: terms may share a definition (`-a`, `--all`)
|
|
203
|
+
# or have none.
|
|
204
|
+
groups: list[tuple[list[str], list[str]]] = [] # (terms, blocks of their definitions)
|
|
205
|
+
previous = "dd"
|
|
206
|
+
for child in node.find_all(["dt", "dd"], recursive=False):
|
|
207
|
+
if child.name == "dt" and previous == "dd" or not groups:
|
|
208
|
+
groups.append(([], []))
|
|
209
|
+
if child.name == "dt":
|
|
210
|
+
groups[-1][0].append(self.inline(child))
|
|
211
|
+
else:
|
|
212
|
+
groups[-1][1].extend(self.children_blocks(child))
|
|
213
|
+
previous = child.name
|
|
214
|
+
items = []
|
|
215
|
+
for terms, body in groups:
|
|
216
|
+
head = ", ".join(f"**{t}**" for t in terms if t)
|
|
217
|
+
joined = bool(head and body and "\n" not in body[0]) # a one-line definition follows its term
|
|
218
|
+
first = f"{head}: {body[0]}" if joined else head
|
|
219
|
+
items.append(_indent_item("-", [first, *body[1 if joined else 0:]]))
|
|
220
|
+
return "\n".join(items)
|
|
221
|
+
|
|
222
|
+
def admonition(self, node: Tag) -> str:
|
|
223
|
+
title_tag = node.find(class_=["admonition-title", "topic-title"])
|
|
224
|
+
title = self.inline(title_tag) if title_tag else node.get("class", ["Note"])[-1].title()
|
|
225
|
+
body = [b for child in node.children if child is not title_tag for b in self.blocks(child)]
|
|
226
|
+
text = f"**{title}:** " + "\n\n".join(body) if body else f"**{title}**"
|
|
227
|
+
return "\n".join("> " + line if line else ">" for line in text.splitlines())
|
|
228
|
+
|
|
229
|
+
def code(self, text: str, lang: str) -> str:
|
|
230
|
+
lang = "" if lang in ("none", "default", "text") else lang
|
|
231
|
+
text = text.strip("\n")
|
|
232
|
+
fence = "````" if "```" in text else "```"
|
|
233
|
+
return f"{fence}{lang}\n{text}\n{fence}"
|
|
234
|
+
|
|
235
|
+
def figure(self, node: Tag) -> list[str]:
|
|
236
|
+
out = [self.image(img) for img in node.find_all("img")]
|
|
237
|
+
caption = node.find("figcaption")
|
|
238
|
+
if caption:
|
|
239
|
+
out.append(f"*{self.inline(caption)}*")
|
|
240
|
+
return out
|
|
241
|
+
|
|
242
|
+
def image(self, img: Tag) -> str:
|
|
243
|
+
alt = _WS.sub(" ", img.get("alt", "")).strip()
|
|
244
|
+
return f")})"
|
|
245
|
+
|
|
246
|
+
def table(self, node: Tag) -> str:
|
|
247
|
+
head_rows = node.select("thead > tr")
|
|
248
|
+
body_rows = node.select("tbody > tr") or [tr for tr in node.find_all("tr") if tr not in head_rows]
|
|
249
|
+
# Several header rows (a group such as "Disk Space" over "Soft Limit") are joined per column.
|
|
250
|
+
head_grid = _grid(head_rows, self.cell, repeat_colspan=True)
|
|
251
|
+
head_width = max((len(r) for r in head_grid), default=0)
|
|
252
|
+
head_grid = [r + [""] * (head_width - len(r)) for r in head_grid]
|
|
253
|
+
header = [" ".join(dict.fromkeys(p for p in col if p)) for col in zip(*head_grid)]
|
|
254
|
+
rows = _grid(body_rows, self.cell)
|
|
255
|
+
width = max([len(header), *(len(r) for r in rows)], default=0)
|
|
256
|
+
header += [""] * (width - len(header))
|
|
257
|
+
rows = [r + [""] * (width - len(r)) for r in rows]
|
|
258
|
+
for prev, row in zip(rows, rows[1:]):
|
|
259
|
+
# An empty first cell groups the row under the one above (`dc-cpu` in JURECA's partition table).
|
|
260
|
+
if width > 1 and not row[0] and any(row[1:]):
|
|
261
|
+
row[0] = prev[0]
|
|
262
|
+
caption = node.find("caption")
|
|
263
|
+
title = f"**{self.inline(caption)}**\n\n" if caption else ""
|
|
264
|
+
simple = all("\n" not in c and len(c) <= _INLINE_TABLE_MAX for r in [header, *rows] for c in r)
|
|
265
|
+
if header and any(header) and simple:
|
|
266
|
+
esc = [[c.replace("|", "\\|") for c in r] for r in [header, *rows]]
|
|
267
|
+
lines = ["| " + " | ".join(esc[0]) + " |", "|" + "---|" * width]
|
|
268
|
+
lines += ["| " + " | ".join(r) + " |" for r in esc[1:]]
|
|
269
|
+
return title + "\n".join(lines)
|
|
270
|
+
records = []
|
|
271
|
+
for r in rows:
|
|
272
|
+
parts = [f"**{h}:** {c}" if h else c for h, c in zip(header, r) if c]
|
|
273
|
+
records.append(_indent_item("-", parts))
|
|
274
|
+
return title + "\n".join(records)
|
|
275
|
+
|
|
276
|
+
def cell(self, td: Tag) -> str:
|
|
277
|
+
blocks = self.children_blocks(td)
|
|
278
|
+
if len(blocks) > 1 and all("\n" not in b and not b.startswith(("- ", "| ", "```", "> ")) for b in blocks):
|
|
279
|
+
# several short paragraphs, e.g. `$HOME`, `$JUSTHOME` or "Login +", "special Computes"
|
|
280
|
+
return "".join(b if i == 0 else (" " if blocks[i - 1].endswith(("+", ":", ",")) else ", ") + b
|
|
281
|
+
for i, b in enumerate(blocks))
|
|
282
|
+
return "\n".join(blocks)
|
|
283
|
+
|
|
284
|
+
# Inline level: returns one line of Markdown.
|
|
285
|
+
|
|
286
|
+
def inline(self, node) -> str:
|
|
287
|
+
return _WS.sub(" ", self._inline(node)).strip()
|
|
288
|
+
|
|
289
|
+
def _inline(self, node) -> str:
|
|
290
|
+
if isinstance(node, NavigableString):
|
|
291
|
+
return str(node) if type(node) is NavigableString else ""
|
|
292
|
+
if not isinstance(node, Tag):
|
|
293
|
+
return ""
|
|
294
|
+
name, classes = node.name, node.get("class", [])
|
|
295
|
+
if name == "code" or name == "kbd" or name == "span" and "pre" in classes and node.parent.name != "code":
|
|
296
|
+
text = _WS.sub(" ", node.get_text()).strip()
|
|
297
|
+
return f"`{text}`" if "`" not in text else f"`` {text} ``"
|
|
298
|
+
if name == "br":
|
|
299
|
+
return " "
|
|
300
|
+
if name == "img":
|
|
301
|
+
return self.image(node)
|
|
302
|
+
inner = "".join(self._inline(c) for c in node.children)
|
|
303
|
+
if name in ("strong", "b"):
|
|
304
|
+
return f"**{inner.strip()}**" if inner.strip() else ""
|
|
305
|
+
if name in ("em", "i", "cite"):
|
|
306
|
+
return f"*{inner.strip()}*" if inner.strip() else ""
|
|
307
|
+
if name == "a":
|
|
308
|
+
if "footnote-reference" in classes:
|
|
309
|
+
return f"[{node.get_text(strip=True).strip('[]')}]"
|
|
310
|
+
href = node.get("href")
|
|
311
|
+
text = inner.strip()
|
|
312
|
+
if not href or not text:
|
|
313
|
+
return inner
|
|
314
|
+
target = urljoin(self.url, href)
|
|
315
|
+
return text if target == text or href.startswith("mailto:") else f"[{text}]({target})"
|
|
316
|
+
return inner
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _int(node: Tag, attr: str) -> int:
|
|
320
|
+
"""An integer attribute such as `colspan`, read as browsers do: the leading digits, or 1 when there are none."""
|
|
321
|
+
m = _INT.match(str(node.get(attr, "")))
|
|
322
|
+
return int(m[1]) if m else 1
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _indent_item(marker: str, blocks: Sequence[str]) -> str:
|
|
326
|
+
"""A list item: the first block after the marker, later blocks and lines indented under it."""
|
|
327
|
+
pad = " " * (len(marker) + 1)
|
|
328
|
+
lines: list[str] = []
|
|
329
|
+
for i, block in enumerate(b for b in blocks if b):
|
|
330
|
+
for j, line in enumerate(block.splitlines()):
|
|
331
|
+
if i == 0 and j == 0:
|
|
332
|
+
lines.append(f"{marker} {line}")
|
|
333
|
+
else:
|
|
334
|
+
lines.append(pad + line if line else "")
|
|
335
|
+
return "\n".join(lines) if lines else marker
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _grid(rows: Iterable[Tag], cell, repeat_colspan: bool = False) -> list[list[str]]:
|
|
339
|
+
"""Expand rowspan and colspan into a full grid, dropping rows that have no cells of their own.
|
|
340
|
+
|
|
341
|
+
A cell spanning several columns fills the first of them, or all of them with `repeat_colspan` (for headers).
|
|
342
|
+
"""
|
|
343
|
+
grid: list[list[str]] = []
|
|
344
|
+
pending: dict[int, tuple[str, int]] = {} # column -> (text, rows still covered)
|
|
345
|
+
for tr in rows:
|
|
346
|
+
own = tr.find_all(["td", "th"], recursive=False)
|
|
347
|
+
row: list[str] = []
|
|
348
|
+
col = 0
|
|
349
|
+
cells = iter(own)
|
|
350
|
+
while True:
|
|
351
|
+
if col in pending:
|
|
352
|
+
text, left = pending[col]
|
|
353
|
+
row.append(text)
|
|
354
|
+
pending[col] = (text, left - 1)
|
|
355
|
+
if left - 1 == 0:
|
|
356
|
+
del pending[col]
|
|
357
|
+
col += 1
|
|
358
|
+
continue
|
|
359
|
+
td = next(cells, None)
|
|
360
|
+
if td is None:
|
|
361
|
+
break
|
|
362
|
+
text = cell(td)
|
|
363
|
+
span = max(1, min(_int(td, "colspan"), _MAX_COLSPAN))
|
|
364
|
+
down = max(1, _int(td, "rowspan"))
|
|
365
|
+
for k in range(span):
|
|
366
|
+
fill = text if k == 0 or repeat_colspan else ""
|
|
367
|
+
row.append(fill)
|
|
368
|
+
if down > 1:
|
|
369
|
+
pending[col + k] = (fill, down - 1)
|
|
370
|
+
col += span
|
|
371
|
+
while any(c > col for c in pending) or col in pending: # spans to the right of the last own cell
|
|
372
|
+
if col in pending:
|
|
373
|
+
text, left = pending.pop(col)
|
|
374
|
+
row.append(text)
|
|
375
|
+
if left - 1:
|
|
376
|
+
pending[col] = (text, left - 1)
|
|
377
|
+
else:
|
|
378
|
+
row.append("")
|
|
379
|
+
col += 1
|
|
380
|
+
if own:
|
|
381
|
+
grid.append(row)
|
|
382
|
+
return grid
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def site_notes(pages: Sequence[ExtractedPage], min_pages: int = 3) -> list[str]:
|
|
386
|
+
"""Remove admonitions that recur on at least half of a site's pages (_NOTE_SHARE), and on at least `min_pages`
|
|
387
|
+
of them, and return them once each.
|
|
388
|
+
|
|
389
|
+
These are banners such as the JUPITER Early Access warning: they matter, but repeated on every page they would
|
|
390
|
+
crowd search results, so they become the site's note instead.
|
|
391
|
+
"""
|
|
392
|
+
if len(pages) < min_pages:
|
|
393
|
+
return []
|
|
394
|
+
counts: Counter[str] = Counter()
|
|
395
|
+
for page in pages:
|
|
396
|
+
counts.update({b for s in page.sections for b in s.blocks if b.startswith("> **")})
|
|
397
|
+
notes = [b for b, n in counts.items() if n >= max(min_pages, _NOTE_SHARE * len(pages))]
|
|
398
|
+
if notes:
|
|
399
|
+
drop = set(notes)
|
|
400
|
+
for page in pages:
|
|
401
|
+
for s in page.sections:
|
|
402
|
+
s.blocks = [b for b in s.blocks if b not in drop]
|
|
403
|
+
return sorted(notes, key=lambda b: -counts[b])
|
jscdocs/core/fetch.py
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
"""A polite HTTP client, for the documentation sites and for the published index.
|
|
2
|
+
|
|
3
|
+
Requests are sequential and spaced by at least `interval` seconds, carry a User-Agent that names jscdocs, and are
|
|
4
|
+
retried with backoff on server errors and network failures. A request that cannot succeed is not retried: a client
|
|
5
|
+
error, and any https URL on a Python that was built without the ssl module. Conditional requests are not used: every page of a
|
|
6
|
+
site carries the same Last-Modified time, since each site is rebuilt as a whole, so they would save nothing.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import http.client
|
|
10
|
+
import time
|
|
11
|
+
import urllib.error
|
|
12
|
+
import urllib.request
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
|
|
15
|
+
from .. import __version__
|
|
16
|
+
|
|
17
|
+
USER_AGENT = f"jscdocs/{__version__} (documentation index for AI agents)"
|
|
18
|
+
MAX_BYTES = 10 * 1024 * 1024 # of a page; the largest today is about 160 KB
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class FetchError(Exception):
|
|
22
|
+
def __init__(self, url: str, reason: str, status: int | None = None):
|
|
23
|
+
super().__init__(f"{url}: {reason}")
|
|
24
|
+
self.status = status
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
Opener = Callable[[urllib.request.Request, float], object]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _has_https() -> bool:
|
|
31
|
+
"""Whether this Python can open https URLs: http.client has HTTPSConnection only when ssl could be imported."""
|
|
32
|
+
return hasattr(http.client, "HTTPSConnection")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _urlopen(request: urllib.request.Request, timeout: float):
|
|
36
|
+
return urllib.request.urlopen(request, timeout=timeout)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class Fetcher:
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
interval: float = 1.0,
|
|
43
|
+
retries: int = 3,
|
|
44
|
+
timeout: float = 30.0,
|
|
45
|
+
opener: Opener = _urlopen,
|
|
46
|
+
clock: Callable[[], float] = time.monotonic,
|
|
47
|
+
sleep: Callable[[float], None] = time.sleep,
|
|
48
|
+
):
|
|
49
|
+
self.interval = interval
|
|
50
|
+
self.retries = retries
|
|
51
|
+
self.timeout = timeout
|
|
52
|
+
self._open = opener
|
|
53
|
+
self._clock = clock
|
|
54
|
+
self._sleep = sleep
|
|
55
|
+
self._last: float | None = None
|
|
56
|
+
|
|
57
|
+
def _wait_turn(self) -> None:
|
|
58
|
+
if self._last is not None:
|
|
59
|
+
delay = self._last + self.interval - self._clock()
|
|
60
|
+
if delay > 0:
|
|
61
|
+
self._sleep(delay)
|
|
62
|
+
self._last = self._clock()
|
|
63
|
+
|
|
64
|
+
def get(self, url: str, max_bytes: int | None = None) -> bytes:
|
|
65
|
+
"""The body at a URL, of at most `max_bytes` (MAX_BYTES when not given).
|
|
66
|
+
|
|
67
|
+
Raises FetchError on a client error at once, or when every retry failed.
|
|
68
|
+
"""
|
|
69
|
+
limit = max_bytes or MAX_BYTES
|
|
70
|
+
if url.startswith("https:") and not _has_https():
|
|
71
|
+
# urllib would report "unknown url type: https" for every attempt
|
|
72
|
+
raise FetchError(url, "this Python has no ssl module and cannot open https URLs; run jscdocs with a "
|
|
73
|
+
"Python that was built with OpenSSL")
|
|
74
|
+
try:
|
|
75
|
+
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
76
|
+
except ValueError:
|
|
77
|
+
raise FetchError(url, "not a URL: it has no scheme such as https://") from None
|
|
78
|
+
reason, status = "no attempt", None
|
|
79
|
+
for attempt in range(self.retries + 1):
|
|
80
|
+
if attempt:
|
|
81
|
+
self._sleep(min(2.0**attempt, 30.0))
|
|
82
|
+
self._wait_turn()
|
|
83
|
+
try:
|
|
84
|
+
with self._open(request, self.timeout) as r:
|
|
85
|
+
body = r.read(limit + 1)
|
|
86
|
+
if len(body) > limit:
|
|
87
|
+
raise FetchError(url, f"larger than {limit} bytes")
|
|
88
|
+
if getattr(r, "length", None):
|
|
89
|
+
# The connection closed before Content-Length bytes had come. http.client raises this
|
|
90
|
+
# itself only for read() without a size, and for chunked bodies.
|
|
91
|
+
raise http.client.IncompleteRead(body, r.length)
|
|
92
|
+
return body
|
|
93
|
+
except urllib.error.HTTPError as e:
|
|
94
|
+
reason, status = f"HTTP {e.code}", e.code
|
|
95
|
+
if e.code < 500 and e.code != 429:
|
|
96
|
+
raise FetchError(url, reason, status) from e
|
|
97
|
+
except (OSError, http.client.HTTPException) as e:
|
|
98
|
+
# OSError covers URLError, timeouts and resets; HTTPException a body that ends early (IncompleteRead)
|
|
99
|
+
reason, status = str(getattr(e, "reason", e)) or type(e).__name__, None
|
|
100
|
+
raise FetchError(url, f"{reason} after {self.retries + 1} attempts", status)
|