jscdocs 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,403 @@
1
+ """Turn a rendered Sphinx page into sections of Markdown.
2
+
3
+ Only `div[itemprop="articleBody"]` is read. Each `<section id>` becomes one Section holding its own content as
4
+ Markdown blocks, without its subsections, so that a search hit points at the smallest matching section. The
5
+ converter is written for the markup Sphinx and the Read the Docs theme produce:
6
+
7
+ - Tables are expanded into a full grid. Sphinx writes a multi-line grid-table row as one row of `rowspan` cells
8
+ followed by empty `<tr>`s, so rows without cells of their own are dropped. Tables whose cells are single short
9
+ lines become Markdown tables, others one record per row.
10
+ - Admonitions become quoted blocks, `> **Note:** ...`.
11
+ - Highlighted code becomes a fenced block, with the language from the `highlight-<lang>` class.
12
+ - Links become absolute URLs, so an agent can pass them on; heading anchor links (`a.headerlink`) are dropped.
13
+
14
+ Banners repeated on every page of a site, such as the JUPITER Early Access warning, are found by `site_notes`
15
+ across a site's pages, not here.
16
+ """
17
+
18
+ import re
19
+ from collections import Counter
20
+ from collections.abc import Iterable, Sequence
21
+ from dataclasses import dataclass, field
22
+ from urllib.parse import urljoin
23
+
24
+ from bs4 import BeautifulSoup, NavigableString, Tag
25
+
26
+ _WS = re.compile(r"\s+")
27
+ _INT = re.compile(r"\s*(-?\d+)")
28
+ _HEADINGS = ("h1", "h2", "h3", "h4", "h5", "h6")
29
+ _INLINE_TABLE_MAX = 120 # longest cell that still fits a Markdown table row
30
+ _MAX_COLSPAN = 1000 # as browsers cap it; a larger value is a mistake in the page
31
+ _NOTE_SHARE = 0.5 # an admonition on at least this share of a site's pages is a banner
32
+
33
+
34
+ class ExtractError(ValueError):
35
+ pass
36
+
37
+
38
+ @dataclass
39
+ class Section:
40
+ anchor: str # the section's HTML id; "" for content before the first section
41
+ title: str # "" for content before the first section, unless it stands in for a missing page title
42
+ depth: int # 1 for the page title (h1)
43
+ path: tuple[str, ...] # titles from the page title down to this section
44
+ aliases: tuple[str, ...] = () # other ids that point at this section (Sphinx labels)
45
+ blocks: list[str] = field(default_factory=list) # Markdown blocks of the section's own content
46
+
47
+ @property
48
+ def markdown(self) -> str:
49
+ return "\n\n".join(self.blocks)
50
+
51
+
52
+ @dataclass
53
+ class ExtractedPage:
54
+ url: str
55
+ title: str
56
+ sections: list[Section]
57
+
58
+
59
+ def extract(html: str | bytes, url: str) -> ExtractedPage:
60
+ """Split a rendered page into sections. `url` is the page's address, used to make links absolute.
61
+
62
+ Raises ExtractError when the page has no article body or cannot be converted.
63
+ """
64
+ try:
65
+ return _extract(html, url)
66
+ except ExtractError:
67
+ raise
68
+ except Exception as e: # noqa: BLE001 # unforeseen markup must fail this page, not the whole sync
69
+ raise ExtractError(f"{url}: cannot convert the page ({type(e).__name__}: {e})") from e
70
+
71
+
72
+ def _extract(html: str | bytes, url: str) -> ExtractedPage:
73
+ soup = BeautifulSoup(html, "html.parser")
74
+ body = soup.select_one('div[itemprop="articleBody"]')
75
+ if body is None:
76
+ raise ExtractError(f"{url}: no article body (not a Sphinx page?)")
77
+ for a in body.select("a.headerlink"):
78
+ a.decompose()
79
+ conv = _Converter(url)
80
+ sections: list[Section] = []
81
+ lead = Section("", "", 0, ())
82
+ _walk(body, conv, sections, lead, (), set())
83
+ h1 = next((s.title for s in sections if s.depth == 1), "")
84
+ title = h1 or (soup.title.get_text(strip=True) if soup.title else "")
85
+ if lead.blocks:
86
+ lead.path = (title,)
87
+ if not h1: # no section holds the page title, so the lead does; otherwise the title would be shown twice
88
+ lead.title, lead.depth = title, 1
89
+ sections.insert(0, lead)
90
+ return ExtractedPage(url, title, sections)
91
+
92
+
93
+ def _walk(node: Tag, conv: "_Converter", out: list[Section], current: Section, path: tuple[str, ...],
94
+ seen: set[str]) -> None:
95
+ """Add the blocks under `node` to `current`, starting a new Section at each <section>.
96
+
97
+ `seen` holds the page's anchors so far. A section needs an id of its own to be found and read, so one without
98
+ an id, or with an id already used, stays part of the enclosing section, under its title in bold.
99
+ """
100
+ for child in node.children:
101
+ if isinstance(child, Tag) and child.name == "section":
102
+ heading = child.find(_HEADINGS, recursive=False)
103
+ title = conv.inline(heading) if heading else ""
104
+ anchor = child.get("id", "")
105
+ if not anchor or anchor in seen:
106
+ if title:
107
+ current.blocks.append(f"**{title}**")
108
+ _walk(child, conv, out, current, path, seen)
109
+ continue
110
+ seen.add(anchor)
111
+ depth = int(heading.name[1]) if heading else current.depth + 1
112
+ aliases = tuple(
113
+ s["id"] for s in child.find_all("span", recursive=False) if s.get("id") and not s.get_text(strip=True)
114
+ )
115
+ sec = Section(anchor, title, depth, path + (title,), aliases)
116
+ out.append(sec)
117
+ _walk(child, conv, out, sec, sec.path, seen)
118
+ elif isinstance(child, Tag) and child.name in _HEADINGS:
119
+ continue # the section's own title, stored in Section.title
120
+ else:
121
+ current.blocks.extend(conv.blocks(child))
122
+
123
+
124
+ class _Converter:
125
+ def __init__(self, url: str):
126
+ self.url = url
127
+
128
+ # Block level: each returns a list of Markdown blocks.
129
+
130
+ def blocks(self, node) -> list[str]:
131
+ if isinstance(node, NavigableString):
132
+ text = _WS.sub(" ", str(node)).strip()
133
+ return [text] if text and type(node) is NavigableString else []
134
+ if not isinstance(node, Tag):
135
+ return []
136
+ name, classes = node.name, node.get("class", [])
137
+ if name in ("script", "style", "nav"):
138
+ return []
139
+ if name == "p":
140
+ if "admonition-title" in classes:
141
+ return []
142
+ text = self.inline(node)
143
+ if "rubric" in classes and text:
144
+ text = f"**{text}**"
145
+ return [text] if text else []
146
+ if name in ("ul", "ol"):
147
+ return [self.list_block(node)]
148
+ if name == "dl":
149
+ return [self.definition_list(node)]
150
+ if name == "table":
151
+ return [self.table(node)]
152
+ if name == "pre":
153
+ return [self.code(node.get_text(), "")]
154
+ if name == "div" and any(c.startswith("highlight-") for c in classes):
155
+ lang = next(c for c in classes if c.startswith("highlight-")).removeprefix("highlight-")
156
+ pre = node.find("pre")
157
+ return [self.code(pre.get_text() if pre else node.get_text(), lang)]
158
+ if "admonition" in classes or name == "aside" and "topic" in classes:
159
+ return [self.admonition(node)]
160
+ if name == "div" and "line-block" in classes:
161
+ lines = [self.inline(line) for line in node.find_all("div", class_="line")]
162
+ return ["\n".join(line for line in lines if line)]
163
+ if name == "aside" and "footnote" in classes:
164
+ label = node.find("span", class_="label")
165
+ num = label.get_text(strip=True).strip("[]") if label else ""
166
+ if label:
167
+ label.decompose()
168
+ text = " ".join(b for b in self.children_blocks(node) if b)
169
+ return [f"[{num}] {text}" if num else text]
170
+ if name == "blockquote":
171
+ inner = self.children_blocks(node)
172
+ return ["\n".join("> " + line if line else ">" for line in "\n\n".join(inner).splitlines())] if inner else []
173
+ if name == "figure":
174
+ return self.figure(node)
175
+ if name == "img":
176
+ return [self.image(node)]
177
+ if name == "hr":
178
+ return []
179
+ if name == "span" and not node.get_text(strip=True):
180
+ return [] # label targets such as <span id="filesystems"></span>
181
+ if name in ("div", "aside", "section", "article", "center", "main"):
182
+ return self.children_blocks(node)
183
+ text = self.inline(node)
184
+ return [text] if text else []
185
+
186
+ def children_blocks(self, node: Tag) -> list[str]:
187
+ out: list[str] = []
188
+ for child in node.children:
189
+ out.extend(self.blocks(child))
190
+ return out
191
+
192
+ def list_block(self, node: Tag) -> str:
193
+ ordered = node.name == "ol"
194
+ start = _int(node, "start") if ordered else 1
195
+ items = []
196
+ for i, li in enumerate(node.find_all("li", recursive=False)):
197
+ marker = f"{start + i}." if ordered else "-"
198
+ items.append(_indent_item(marker, self.children_blocks(li)))
199
+ return "\n".join(items)
200
+
201
+ def definition_list(self, node: Tag) -> str:
202
+ # One item per run of terms with the definitions that follow it: terms may share a definition (`-a`, `--all`)
203
+ # or have none.
204
+ groups: list[tuple[list[str], list[str]]] = [] # (terms, blocks of their definitions)
205
+ previous = "dd"
206
+ for child in node.find_all(["dt", "dd"], recursive=False):
207
+ if child.name == "dt" and previous == "dd" or not groups:
208
+ groups.append(([], []))
209
+ if child.name == "dt":
210
+ groups[-1][0].append(self.inline(child))
211
+ else:
212
+ groups[-1][1].extend(self.children_blocks(child))
213
+ previous = child.name
214
+ items = []
215
+ for terms, body in groups:
216
+ head = ", ".join(f"**{t}**" for t in terms if t)
217
+ joined = bool(head and body and "\n" not in body[0]) # a one-line definition follows its term
218
+ first = f"{head}: {body[0]}" if joined else head
219
+ items.append(_indent_item("-", [first, *body[1 if joined else 0:]]))
220
+ return "\n".join(items)
221
+
222
+ def admonition(self, node: Tag) -> str:
223
+ title_tag = node.find(class_=["admonition-title", "topic-title"])
224
+ title = self.inline(title_tag) if title_tag else node.get("class", ["Note"])[-1].title()
225
+ body = [b for child in node.children if child is not title_tag for b in self.blocks(child)]
226
+ text = f"**{title}:** " + "\n\n".join(body) if body else f"**{title}**"
227
+ return "\n".join("> " + line if line else ">" for line in text.splitlines())
228
+
229
+ def code(self, text: str, lang: str) -> str:
230
+ lang = "" if lang in ("none", "default", "text") else lang
231
+ text = text.strip("\n")
232
+ fence = "````" if "```" in text else "```"
233
+ return f"{fence}{lang}\n{text}\n{fence}"
234
+
235
+ def figure(self, node: Tag) -> list[str]:
236
+ out = [self.image(img) for img in node.find_all("img")]
237
+ caption = node.find("figcaption")
238
+ if caption:
239
+ out.append(f"*{self.inline(caption)}*")
240
+ return out
241
+
242
+ def image(self, img: Tag) -> str:
243
+ alt = _WS.sub(" ", img.get("alt", "")).strip()
244
+ return f"![{alt}]({urljoin(self.url, img.get('src', ''))})"
245
+
246
+ def table(self, node: Tag) -> str:
247
+ head_rows = node.select("thead > tr")
248
+ body_rows = node.select("tbody > tr") or [tr for tr in node.find_all("tr") if tr not in head_rows]
249
+ # Several header rows (a group such as "Disk Space" over "Soft Limit") are joined per column.
250
+ head_grid = _grid(head_rows, self.cell, repeat_colspan=True)
251
+ head_width = max((len(r) for r in head_grid), default=0)
252
+ head_grid = [r + [""] * (head_width - len(r)) for r in head_grid]
253
+ header = [" ".join(dict.fromkeys(p for p in col if p)) for col in zip(*head_grid)]
254
+ rows = _grid(body_rows, self.cell)
255
+ width = max([len(header), *(len(r) for r in rows)], default=0)
256
+ header += [""] * (width - len(header))
257
+ rows = [r + [""] * (width - len(r)) for r in rows]
258
+ for prev, row in zip(rows, rows[1:]):
259
+ # An empty first cell groups the row under the one above (`dc-cpu` in JURECA's partition table).
260
+ if width > 1 and not row[0] and any(row[1:]):
261
+ row[0] = prev[0]
262
+ caption = node.find("caption")
263
+ title = f"**{self.inline(caption)}**\n\n" if caption else ""
264
+ simple = all("\n" not in c and len(c) <= _INLINE_TABLE_MAX for r in [header, *rows] for c in r)
265
+ if header and any(header) and simple:
266
+ esc = [[c.replace("|", "\\|") for c in r] for r in [header, *rows]]
267
+ lines = ["| " + " | ".join(esc[0]) + " |", "|" + "---|" * width]
268
+ lines += ["| " + " | ".join(r) + " |" for r in esc[1:]]
269
+ return title + "\n".join(lines)
270
+ records = []
271
+ for r in rows:
272
+ parts = [f"**{h}:** {c}" if h else c for h, c in zip(header, r) if c]
273
+ records.append(_indent_item("-", parts))
274
+ return title + "\n".join(records)
275
+
276
+ def cell(self, td: Tag) -> str:
277
+ blocks = self.children_blocks(td)
278
+ if len(blocks) > 1 and all("\n" not in b and not b.startswith(("- ", "| ", "```", "> ")) for b in blocks):
279
+ # several short paragraphs, e.g. `$HOME`, `$JUSTHOME` or "Login +", "special Computes"
280
+ return "".join(b if i == 0 else (" " if blocks[i - 1].endswith(("+", ":", ",")) else ", ") + b
281
+ for i, b in enumerate(blocks))
282
+ return "\n".join(blocks)
283
+
284
+ # Inline level: returns one line of Markdown.
285
+
286
+ def inline(self, node) -> str:
287
+ return _WS.sub(" ", self._inline(node)).strip()
288
+
289
+ def _inline(self, node) -> str:
290
+ if isinstance(node, NavigableString):
291
+ return str(node) if type(node) is NavigableString else ""
292
+ if not isinstance(node, Tag):
293
+ return ""
294
+ name, classes = node.name, node.get("class", [])
295
+ if name == "code" or name == "kbd" or name == "span" and "pre" in classes and node.parent.name != "code":
296
+ text = _WS.sub(" ", node.get_text()).strip()
297
+ return f"`{text}`" if "`" not in text else f"`` {text} ``"
298
+ if name == "br":
299
+ return " "
300
+ if name == "img":
301
+ return self.image(node)
302
+ inner = "".join(self._inline(c) for c in node.children)
303
+ if name in ("strong", "b"):
304
+ return f"**{inner.strip()}**" if inner.strip() else ""
305
+ if name in ("em", "i", "cite"):
306
+ return f"*{inner.strip()}*" if inner.strip() else ""
307
+ if name == "a":
308
+ if "footnote-reference" in classes:
309
+ return f"[{node.get_text(strip=True).strip('[]')}]"
310
+ href = node.get("href")
311
+ text = inner.strip()
312
+ if not href or not text:
313
+ return inner
314
+ target = urljoin(self.url, href)
315
+ return text if target == text or href.startswith("mailto:") else f"[{text}]({target})"
316
+ return inner
317
+
318
+
319
+ def _int(node: Tag, attr: str) -> int:
320
+ """An integer attribute such as `colspan`, read as browsers do: the leading digits, or 1 when there are none."""
321
+ m = _INT.match(str(node.get(attr, "")))
322
+ return int(m[1]) if m else 1
323
+
324
+
325
+ def _indent_item(marker: str, blocks: Sequence[str]) -> str:
326
+ """A list item: the first block after the marker, later blocks and lines indented under it."""
327
+ pad = " " * (len(marker) + 1)
328
+ lines: list[str] = []
329
+ for i, block in enumerate(b for b in blocks if b):
330
+ for j, line in enumerate(block.splitlines()):
331
+ if i == 0 and j == 0:
332
+ lines.append(f"{marker} {line}")
333
+ else:
334
+ lines.append(pad + line if line else "")
335
+ return "\n".join(lines) if lines else marker
336
+
337
+
338
+ def _grid(rows: Iterable[Tag], cell, repeat_colspan: bool = False) -> list[list[str]]:
339
+ """Expand rowspan and colspan into a full grid, dropping rows that have no cells of their own.
340
+
341
+ A cell spanning several columns fills the first of them, or all of them with `repeat_colspan` (for headers).
342
+ """
343
+ grid: list[list[str]] = []
344
+ pending: dict[int, tuple[str, int]] = {} # column -> (text, rows still covered)
345
+ for tr in rows:
346
+ own = tr.find_all(["td", "th"], recursive=False)
347
+ row: list[str] = []
348
+ col = 0
349
+ cells = iter(own)
350
+ while True:
351
+ if col in pending:
352
+ text, left = pending[col]
353
+ row.append(text)
354
+ pending[col] = (text, left - 1)
355
+ if left - 1 == 0:
356
+ del pending[col]
357
+ col += 1
358
+ continue
359
+ td = next(cells, None)
360
+ if td is None:
361
+ break
362
+ text = cell(td)
363
+ span = max(1, min(_int(td, "colspan"), _MAX_COLSPAN))
364
+ down = max(1, _int(td, "rowspan"))
365
+ for k in range(span):
366
+ fill = text if k == 0 or repeat_colspan else ""
367
+ row.append(fill)
368
+ if down > 1:
369
+ pending[col + k] = (fill, down - 1)
370
+ col += span
371
+ while any(c > col for c in pending) or col in pending: # spans to the right of the last own cell
372
+ if col in pending:
373
+ text, left = pending.pop(col)
374
+ row.append(text)
375
+ if left - 1:
376
+ pending[col] = (text, left - 1)
377
+ else:
378
+ row.append("")
379
+ col += 1
380
+ if own:
381
+ grid.append(row)
382
+ return grid
383
+
384
+
385
+ def site_notes(pages: Sequence[ExtractedPage], min_pages: int = 3) -> list[str]:
386
+ """Remove admonitions that recur on at least half of a site's pages (_NOTE_SHARE), and on at least `min_pages`
387
+ of them, and return them once each.
388
+
389
+ These are banners such as the JUPITER Early Access warning: they matter, but repeated on every page they would
390
+ crowd search results, so they become the site's note instead.
391
+ """
392
+ if len(pages) < min_pages:
393
+ return []
394
+ counts: Counter[str] = Counter()
395
+ for page in pages:
396
+ counts.update({b for s in page.sections for b in s.blocks if b.startswith("> **")})
397
+ notes = [b for b, n in counts.items() if n >= max(min_pages, _NOTE_SHARE * len(pages))]
398
+ if notes:
399
+ drop = set(notes)
400
+ for page in pages:
401
+ for s in page.sections:
402
+ s.blocks = [b for b in s.blocks if b not in drop]
403
+ return sorted(notes, key=lambda b: -counts[b])
jscdocs/core/fetch.py ADDED
@@ -0,0 +1,100 @@
1
+ """A polite HTTP client, for the documentation sites and for the published index.
2
+
3
+ Requests are sequential and spaced by at least `interval` seconds, carry a User-Agent that names jscdocs, and are
4
+ retried with backoff on server errors and network failures. A request that cannot succeed is not retried: a client
5
+ error, and any https URL on a Python that was built without the ssl module. Conditional requests are not used: every page of a
6
+ site carries the same Last-Modified time, since each site is rebuilt as a whole, so they would save nothing.
7
+ """
8
+
9
+ import http.client
10
+ import time
11
+ import urllib.error
12
+ import urllib.request
13
+ from collections.abc import Callable
14
+
15
+ from .. import __version__
16
+
17
+ USER_AGENT = f"jscdocs/{__version__} (documentation index for AI agents)"
18
+ MAX_BYTES = 10 * 1024 * 1024 # of a page; the largest today is about 160 KB
19
+
20
+
21
+ class FetchError(Exception):
22
+ def __init__(self, url: str, reason: str, status: int | None = None):
23
+ super().__init__(f"{url}: {reason}")
24
+ self.status = status
25
+
26
+
27
+ Opener = Callable[[urllib.request.Request, float], object]
28
+
29
+
30
+ def _has_https() -> bool:
31
+ """Whether this Python can open https URLs: http.client has HTTPSConnection only when ssl could be imported."""
32
+ return hasattr(http.client, "HTTPSConnection")
33
+
34
+
35
+ def _urlopen(request: urllib.request.Request, timeout: float):
36
+ return urllib.request.urlopen(request, timeout=timeout)
37
+
38
+
39
+ class Fetcher:
40
+ def __init__(
41
+ self,
42
+ interval: float = 1.0,
43
+ retries: int = 3,
44
+ timeout: float = 30.0,
45
+ opener: Opener = _urlopen,
46
+ clock: Callable[[], float] = time.monotonic,
47
+ sleep: Callable[[float], None] = time.sleep,
48
+ ):
49
+ self.interval = interval
50
+ self.retries = retries
51
+ self.timeout = timeout
52
+ self._open = opener
53
+ self._clock = clock
54
+ self._sleep = sleep
55
+ self._last: float | None = None
56
+
57
+ def _wait_turn(self) -> None:
58
+ if self._last is not None:
59
+ delay = self._last + self.interval - self._clock()
60
+ if delay > 0:
61
+ self._sleep(delay)
62
+ self._last = self._clock()
63
+
64
+ def get(self, url: str, max_bytes: int | None = None) -> bytes:
65
+ """The body at a URL, of at most `max_bytes` (MAX_BYTES when not given).
66
+
67
+ Raises FetchError on a client error at once, or when every retry failed.
68
+ """
69
+ limit = max_bytes or MAX_BYTES
70
+ if url.startswith("https:") and not _has_https():
71
+ # urllib would report "unknown url type: https" for every attempt
72
+ raise FetchError(url, "this Python has no ssl module and cannot open https URLs; run jscdocs with a "
73
+ "Python that was built with OpenSSL")
74
+ try:
75
+ request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
76
+ except ValueError:
77
+ raise FetchError(url, "not a URL: it has no scheme such as https://") from None
78
+ reason, status = "no attempt", None
79
+ for attempt in range(self.retries + 1):
80
+ if attempt:
81
+ self._sleep(min(2.0**attempt, 30.0))
82
+ self._wait_turn()
83
+ try:
84
+ with self._open(request, self.timeout) as r:
85
+ body = r.read(limit + 1)
86
+ if len(body) > limit:
87
+ raise FetchError(url, f"larger than {limit} bytes")
88
+ if getattr(r, "length", None):
89
+ # The connection closed before Content-Length bytes had come. http.client raises this
90
+ # itself only for read() without a size, and for chunked bodies.
91
+ raise http.client.IncompleteRead(body, r.length)
92
+ return body
93
+ except urllib.error.HTTPError as e:
94
+ reason, status = f"HTTP {e.code}", e.code
95
+ if e.code < 500 and e.code != 429:
96
+ raise FetchError(url, reason, status) from e
97
+ except (OSError, http.client.HTTPException) as e:
98
+ # OSError covers URLError, timeouts and resets; HTTPException a body that ends early (IncompleteRead)
99
+ reason, status = str(getattr(e, "reason", e)) or type(e).__name__, None
100
+ raise FetchError(url, f"{reason} after {self.retries + 1} attempts", status)