primarysource 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (303) hide show
  1. primarysource/__init__.py +5 -0
  2. primarysource/bbf/__init__.py +56 -0
  3. primarysource/bbf/_xml.py +63 -0
  4. primarysource/bbf/catalogue.json +777 -0
  5. primarysource/bbf/catalogue.py +137 -0
  6. primarysource/bbf/cli.py +79 -0
  7. primarysource/bbf/datamodel.py +281 -0
  8. primarysource/bbf/importer.py +151 -0
  9. primarysource/bbf/library.py +380 -0
  10. primarysource/bbf/server.py +72 -0
  11. primarysource/budget.py +63 -0
  12. primarysource/c2sp/__init__.py +53 -0
  13. primarysource/c2sp/_fetch.py +5 -0
  14. primarysource/c2sp/catalogue.json +231 -0
  15. primarysource/c2sp/catalogue.py +95 -0
  16. primarysource/c2sp/cli.py +62 -0
  17. primarysource/c2sp/importer.py +90 -0
  18. primarysource/c2sp/library.py +309 -0
  19. primarysource/c2sp/markdown.py +31 -0
  20. primarysource/c2sp/server.py +68 -0
  21. primarysource/cavp/__init__.py +42 -0
  22. primarysource/cavp/cli.py +84 -0
  23. primarysource/cavp/data/vectors.toml +47 -0
  24. primarysource/cavp/importer.py +156 -0
  25. primarysource/cavp/model.py +279 -0
  26. primarysource/cavp/parse.py +269 -0
  27. primarysource/cavp/paths.py +17 -0
  28. primarysource/cavp/server.py +67 -0
  29. primarysource/cavp/tools.py +169 -0
  30. primarysource/cli.py +107 -0
  31. primarysource/cursor.py +82 -0
  32. primarysource/cve/__init__.py +5 -0
  33. primarysource/cve/__main__.py +6 -0
  34. primarysource/cve/cli.py +144 -0
  35. primarysource/cve/collection.py +20 -0
  36. primarysource/cve/config.py +58 -0
  37. primarysource/cve/errors.py +16 -0
  38. primarysource/cve/kernel.py +232 -0
  39. primarysource/cve/mcp.py +139 -0
  40. primarysource/cve/parse.py +318 -0
  41. primarysource/cve/py.typed +0 -0
  42. primarysource/cve/service.py +760 -0
  43. primarysource/cve/store.py +214 -0
  44. primarysource/cve/sync.py +438 -0
  45. primarysource/cve/views.py +304 -0
  46. primarysource/debpolicy/__init__.py +16 -0
  47. primarysource/debpolicy/catalogue.py +263 -0
  48. primarysource/debpolicy/cli.py +12 -0
  49. primarysource/debpolicy/server.py +11 -0
  50. primarysource/draft/__init__.py +11 -0
  51. primarysource/draft/catalogue.py +117 -0
  52. primarysource/draft/cli.py +149 -0
  53. primarysource/draft/collection.py +21 -0
  54. primarysource/draft/datatracker.py +158 -0
  55. primarysource/draft/importer.py +163 -0
  56. primarysource/draft/licence.py +89 -0
  57. primarysource/draft/names.py +64 -0
  58. primarysource/draft/net.py +133 -0
  59. primarysource/draft/segment.py +192 -0
  60. primarysource/draft/server.py +91 -0
  61. primarysource/draft/status.py +146 -0
  62. primarysource/draft/store.py +199 -0
  63. primarysource/draft/tracked.json +104 -0
  64. primarysource/draft/views.py +430 -0
  65. primarysource/draft/xmlmeta.py +74 -0
  66. primarysource/ecma/__init__.py +52 -0
  67. primarysource/ecma/_pdf.py +143 -0
  68. primarysource/ecma/catalogue.json +22 -0
  69. primarysource/ecma/catalogue.py +100 -0
  70. primarysource/ecma/cli.py +67 -0
  71. primarysource/ecma/identify.py +25 -0
  72. primarysource/ecma/importer.py +108 -0
  73. primarysource/ecma/library.py +240 -0
  74. primarysource/ecma/server.py +67 -0
  75. primarysource/errors.py +29 -0
  76. primarysource/fetch.py +360 -0
  77. primarysource/fsutil.py +150 -0
  78. primarysource/gitdocs/__init__.py +15 -0
  79. primarysource/gitdocs/catalogue.py +116 -0
  80. primarysource/gitdocs/cli.py +12 -0
  81. primarysource/gitdocs/server.py +8 -0
  82. primarysource/home.py +65 -0
  83. primarysource/iana/__init__.py +32 -0
  84. primarysource/iana/catalogue.json +244 -0
  85. primarysource/iana/catalogue.py +69 -0
  86. primarysource/iana/cli.py +69 -0
  87. primarysource/iana/importer.py +157 -0
  88. primarysource/iana/model.py +325 -0
  89. primarysource/iana/reader.py +315 -0
  90. primarysource/iana/server.py +67 -0
  91. primarysource/iana/store.py +83 -0
  92. primarysource/iana/xmlsafe.py +47 -0
  93. primarysource/identify.py +186 -0
  94. primarysource/ieee/__init__.py +58 -0
  95. primarysource/ieee/_html.py +49 -0
  96. primarysource/ieee/catalogue.json +74 -0
  97. primarysource/ieee/catalogue.py +123 -0
  98. primarysource/ieee/cli.py +79 -0
  99. primarysource/ieee/identify.py +24 -0
  100. primarysource/ieee/ieee802.py +47 -0
  101. primarysource/ieee/importer.py +305 -0
  102. primarysource/ieee/posix.py +233 -0
  103. primarysource/ieee/server.py +79 -0
  104. primarysource/index.py +446 -0
  105. primarysource/iso/__init__.py +51 -0
  106. primarysource/iso/cli.py +98 -0
  107. primarysource/iso/designation.py +95 -0
  108. primarysource/iso/identify.py +100 -0
  109. primarysource/iso/importer.py +224 -0
  110. primarysource/iso/library.py +249 -0
  111. primarysource/iso/server.py +69 -0
  112. primarysource/itu/__init__.py +41 -0
  113. primarysource/itu/__main__.py +8 -0
  114. primarysource/itu/asn1mod.py +478 -0
  115. primarysource/itu/catalog.py +472 -0
  116. primarysource/itu/cli.py +445 -0
  117. primarysource/itu/collection.py +20 -0
  118. primarysource/itu/corpus.py +118 -0
  119. primarysource/itu/errata.py +16 -0
  120. primarysource/itu/formal.py +565 -0
  121. primarysource/itu/identify.py +26 -0
  122. primarysource/itu/index.py +490 -0
  123. primarysource/itu/mcp.py +474 -0
  124. primarysource/itu/py.typed +0 -0
  125. primarysource/itu/sources.py +600 -0
  126. primarysource/itu/text.py +609 -0
  127. primarysource/kconfig/__init__.py +16 -0
  128. primarysource/kconfig/catalogue.py +98 -0
  129. primarysource/kconfig/cli.py +12 -0
  130. primarysource/kconfig/server.py +8 -0
  131. primarysource/model.py +63 -0
  132. primarysource/nist/__init__.py +5 -0
  133. primarysource/nist/collection.py +46 -0
  134. primarysource/nist/data/acquired-nist.json +1657 -0
  135. primarysource/nist/data/nist-catalog.json +12090 -0
  136. primarysource/nist/data/sources.toml +1996 -0
  137. primarysource/nist/identify.py +41 -0
  138. primarysource/nist/ingest/__init__.py +1 -0
  139. primarysource/nist/ingest/acquire_nist.py +128 -0
  140. primarysource/nist/ingest/controls.py +82 -0
  141. primarysource/nist/ingest/discover_nist.py +218 -0
  142. primarysource/nist/ingest/extract.py +48 -0
  143. primarysource/nist/ingest/fetch.py +211 -0
  144. primarysource/nist/ingest/index.py +14 -0
  145. primarysource/nist/ingest/oscal.py +480 -0
  146. primarysource/nist/ingest/prose_layouts.py +236 -0
  147. primarysource/nist/ingest/sources.py +94 -0
  148. primarysource/nist/ingest/status.py +255 -0
  149. primarysource/nist/models.py +7 -0
  150. primarysource/nist/paths.py +33 -0
  151. primarysource/nist/server/__init__.py +3 -0
  152. primarysource/nist/server/corpus.py +38 -0
  153. primarysource/nist/server/mcp.py +19 -0
  154. primarysource/nist/server/tools.py +35 -0
  155. primarysource/nist/spec.py +38 -0
  156. primarysource/pdfcorpus/__init__.py +16 -0
  157. primarysource/pdfcorpus/corpus.py +159 -0
  158. primarysource/pdfcorpus/fts.py +175 -0
  159. primarysource/pdfcorpus/index.py +70 -0
  160. primarysource/pdfcorpus/models.py +30 -0
  161. primarysource/pdfcorpus/server.py +96 -0
  162. primarysource/pdfcorpus/spec.py +37 -0
  163. primarysource/pdfcorpus/split.py +391 -0
  164. primarysource/pdfcorpus/tools.py +244 -0
  165. primarysource/pep/__init__.py +15 -0
  166. primarysource/pep/catalogue.py +191 -0
  167. primarysource/pep/cli.py +12 -0
  168. primarysource/pep/server.py +8 -0
  169. primarysource/pkgdocs/__init__.py +19 -0
  170. primarysource/pkgdocs/fetch.py +356 -0
  171. primarysource/pkgdocs/index.py +89 -0
  172. primarysource/pkgdocs/model.py +76 -0
  173. primarysource/pkgdocs/readers/__init__.py +1 -0
  174. primarysource/pkgdocs/readers/_common.py +134 -0
  175. primarysource/pkgdocs/readers/asciidoc.py +160 -0
  176. primarysource/pkgdocs/readers/latex.py +315 -0
  177. primarysource/pkgdocs/readers/markdown.py +173 -0
  178. primarysource/pkgdocs/readers/pod.py +132 -0
  179. primarysource/pkgdocs/readers/rst.py +239 -0
  180. primarysource/pkgdocs/readers/scdoc.py +108 -0
  181. primarysource/pkgdocs/runtime.py +552 -0
  182. primarysource/pkgdocs/serving.py +158 -0
  183. primarysource/pkgdocs/store.py +167 -0
  184. primarysource/pms/__init__.py +16 -0
  185. primarysource/pms/catalogue.py +88 -0
  186. primarysource/pms/cli.py +12 -0
  187. primarysource/pms/server.py +8 -0
  188. primarysource/py.typed +0 -0
  189. primarysource/readers/__init__.py +10 -0
  190. primarysource/readers/bookmarks.py +192 -0
  191. primarysource/readers/html.py +1312 -0
  192. primarysource/readers/markdown.py +189 -0
  193. primarysource/readers/pdf.py +187 -0
  194. primarysource/readers/xml.py +230 -0
  195. primarysource/refs.py +87 -0
  196. primarysource/registry.py +161 -0
  197. primarysource/render.py +49 -0
  198. primarysource/rfc/__init__.py +3 -0
  199. primarysource/rfc/__main__.py +6 -0
  200. primarysource/rfc/cite.py +204 -0
  201. primarysource/rfc/cli.py +903 -0
  202. primarysource/rfc/collection.py +19 -0
  203. primarysource/rfc/config.py +138 -0
  204. primarysource/rfc/errata.py +203 -0
  205. primarysource/rfc/fsutil.py +82 -0
  206. primarysource/rfc/graph.py +213 -0
  207. primarysource/rfc/index/__init__.py +1 -0
  208. primarysource/rfc/index/chunker.py +281 -0
  209. primarysource/rfc/index/embed.py +166 -0
  210. primarysource/rfc/index/hybrid.py +305 -0
  211. primarysource/rfc/index/lexical.py +223 -0
  212. primarysource/rfc/index/rfcxml.py +673 -0
  213. primarysource/rfc/index/search.py +508 -0
  214. primarysource/rfc/index/txtseg.py +585 -0
  215. primarysource/rfc/library.py +294 -0
  216. primarysource/rfc/mcp/__init__.py +1 -0
  217. primarysource/rfc/mcp/server.py +381 -0
  218. primarysource/rfc/packs/__init__.py +1 -0
  219. primarysource/rfc/packs/builder.py +228 -0
  220. primarysource/rfc/packs/collections/all-current-txt.json +10 -0
  221. primarysource/rfc/packs/collections/essentials.json +19 -0
  222. primarysource/rfc/packs/collections/fyi-current.json +10 -0
  223. primarysource/rfc/packs/collections/standards-current.json +10 -0
  224. primarysource/rfc/packs/collections/year-2025.json +10 -0
  225. primarysource/rfc/packs/collections/year-2026.json +10 -0
  226. primarysource/rfc/packs/collections.py +163 -0
  227. primarysource/rfc/packs/docsetexport.py +163 -0
  228. primarysource/rfc/packs/epub.py +159 -0
  229. primarysource/rfc/packs/export.py +306 -0
  230. primarysource/rfc/packs/htmlexport.py +82 -0
  231. primarysource/rfc/packs/serve.py +143 -0
  232. primarysource/rfc/packs/zimexport.py +171 -0
  233. primarysource/rfc/query.py +466 -0
  234. primarysource/rfc/subjects.py +457 -0
  235. primarysource/rfc/sync/__init__.py +1 -0
  236. primarysource/rfc/sync/downloader.py +132 -0
  237. primarysource/rfc/sync/manifest.py +162 -0
  238. primarysource/rfc/sync/scanner.py +49 -0
  239. primarysource/rfc/sync/state.py +464 -0
  240. primarysource/rfc/views.py +1067 -0
  241. primarysource/rpmdocs/__init__.py +15 -0
  242. primarysource/rpmdocs/catalogue.py +117 -0
  243. primarysource/rpmdocs/cli.py +12 -0
  244. primarysource/rpmdocs/server.py +8 -0
  245. primarysource/scaffold.py +223 -0
  246. primarysource/serve.py +75 -0
  247. primarysource/settings.py +152 -0
  248. primarysource/usb/__init__.py +5 -0
  249. primarysource/usb/collection.py +37 -0
  250. primarysource/usb/data/acquired.json +2025 -0
  251. primarysource/usb/data/sources-usb.toml +852 -0
  252. primarysource/usb/data/sources.toml +10 -0
  253. primarysource/usb/data/usb-catalog.json +1663 -0
  254. primarysource/usb/data/usb-classification.json +1986 -0
  255. primarysource/usb/data/usb-corpus-build.json +608 -0
  256. primarysource/usb/data/usb-pdf-choice.json +839 -0
  257. primarysource/usb/data/usb-split-measure.json +1803 -0
  258. primarysource/usb/ingest/__init__.py +1 -0
  259. primarysource/usb/ingest/acquire.py +130 -0
  260. primarysource/usb/ingest/discover_usb.py +88 -0
  261. primarysource/usb/ingest/extract.py +48 -0
  262. primarysource/usb/ingest/fetch.py +197 -0
  263. primarysource/usb/ingest/index.py +14 -0
  264. primarysource/usb/ingest/sources.py +85 -0
  265. primarysource/usb/models.py +7 -0
  266. primarysource/usb/paths.py +33 -0
  267. primarysource/usb/server/__init__.py +3 -0
  268. primarysource/usb/server/corpus.py +38 -0
  269. primarysource/usb/server/mcp.py +19 -0
  270. primarysource/usb/server/tools.py +35 -0
  271. primarysource/usb/spec.py +20 -0
  272. primarysource/usbids/__init__.py +56 -0
  273. primarysource/usbids/cli.py +65 -0
  274. primarysource/usbids/importer.py +272 -0
  275. primarysource/usbids/parse.py +283 -0
  276. primarysource/usbids/server.py +55 -0
  277. primarysource/usbids/store.py +80 -0
  278. primarysource/usbids/tools.py +174 -0
  279. primarysource/zim/__init__.py +5 -0
  280. primarysource/zim/__main__.py +8 -0
  281. primarysource/zim/_libzim.py +37 -0
  282. primarysource/zim/catalog.py +594 -0
  283. primarysource/zim/cli.py +555 -0
  284. primarysource/zim/collection.py +41 -0
  285. primarysource/zim/errors.py +100 -0
  286. primarysource/zim/groups.py +30 -0
  287. primarysource/zim/library.py +668 -0
  288. primarysource/zim/manage.py +636 -0
  289. primarysource/zim/markdown.py +1247 -0
  290. primarysource/zim/mcp.py +201 -0
  291. primarysource/zim/purl.py +130 -0
  292. primarysource/zim/py.typed +0 -0
  293. primarysource/zim/refs.py +90 -0
  294. primarysource/zim/service.py +1182 -0
  295. primarysource/zim/snapshot.py +197 -0
  296. primarysource/zim/sync.py +269 -0
  297. primarysource/zim/views.py +230 -0
  298. primarysource-0.1.0.dist-info/METADATA +118 -0
  299. primarysource-0.1.0.dist-info/RECORD +303 -0
  300. primarysource-0.1.0.dist-info/WHEEL +4 -0
  301. primarysource-0.1.0.dist-info/entry_points.txt +2 -0
  302. primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
  303. primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
@@ -0,0 +1,552 @@
1
+ """What a pkgdocs collection does: import a pinned edition, then docs, outline, read, search, cite, status.
2
+
3
+ A collection is a `Spec`: its trees (each a pinned source repository, with the files to take from it and a
4
+ `build` that turns them into documents), its licence and placement, and the spelling of its document ids. This
5
+ module holds the behaviour every one of them shares, so a collection directory is data plus a `build`.
6
+
7
+ Refs: `[<collection>:]<doc>[@<commit>][§<section>][#<anchor>][¶<n>]`. `§` takes a section id (`5.6.1`), an anchor
8
+ (`s-f-Source`), a heading, or a unique part of one; `@<commit>` (four or more hex digits, or a pin's label) picks
9
+ an imported edition other than the newest; `¶n` is the nth block of the section.
10
+
11
+ A read of `§s¶n` starts at block n and runs on through the subsections of `s`, so a count past the section's own
12
+ blocks goes on into them, one unit per subsection heading and per block, in reading order. That is what makes a
13
+ read's `next:` footer a ref: `§s¶<n>` where the next page starts. Lists (`docs`, `outline`, `search`) take a
14
+ 0-based `start` and print `next: start=<n>`.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import re
20
+ from collections.abc import Callable, Mapping, Sequence
21
+ from dataclasses import dataclass, field
22
+ from datetime import UTC, datetime
23
+ from difflib import get_close_matches
24
+ from pathlib import Path
25
+
26
+ from primarysource import cursor
27
+ from primarysource.budget import page
28
+ from primarysource.errors import LibError
29
+ from primarysource.home import collection_home
30
+ from primarysource.model import Unit, span_hash
31
+ from primarysource.pkgdocs import fetch, index, store
32
+ from primarysource.pkgdocs.model import Doc, Sec
33
+ from primarysource.refs import Ref, RefGrammar
34
+ from primarysource.registry import Collection
35
+ from primarysource.render import next_start, render
36
+
37
+ DOC_PATTERN = r"[A-Za-z0-9][A-Za-z0-9._/+-]*(?:@[0-9A-Za-z][0-9A-Za-z._-]{1,39})?"
38
+ SECTION_PATTERN = r"[^#¶\n]+?"
39
+ _HASH_ESCAPE = "%23"
40
+ _BARE = re.compile(r"(?P<section>\d+(?:\.\d+)+)(?:@(?P<commit>[0-9A-Za-z][\w.-]{1,39}))?(?P<para>¶\d+)?")
41
+
42
+
43
+ @dataclass(frozen=True, slots=True)
44
+ class Licence:
45
+ """The licence of a collection's text and where it may be placed. `redistribute` is `layer` (may be baked
46
+ into an image), `volume` (user-supplied, never in an image) or `none`. `evidence` names the file in the
47
+ source repository the licence was read from. `attribution` is the credit shown by `status`; `credit` is the
48
+ shorter form a licence that requires attribution (CC-BY-SA) needs in every citation."""
49
+
50
+ spdx: str
51
+ redistribute: str
52
+ attribution: str = ""
53
+ evidence: str = ""
54
+ credit: str = ""
55
+
56
+
57
+ @dataclass(frozen=True, slots=True)
58
+ class Pin:
59
+ """One edition of a tree: a full commit, a human label (a tag), its date, the `fetch.manifest_digest` of the
60
+ files taken from it, and where to fetch it (the sources are mirrors of each other and must give that digest)."""
61
+
62
+ commit: str
63
+ label: str
64
+ date: str
65
+ files_sha256: str | None
66
+ sources: tuple[fetch.Source, ...]
67
+
68
+
69
+ @dataclass(frozen=True, slots=True)
70
+ class Build:
71
+ """What a tree's `build` returns: the documents, chapter aliases (`5` -> `ch-controlfields`) and counts of the
72
+ files it left out, by reason."""
73
+
74
+ docs: tuple[Doc, ...]
75
+ aliases: Mapping[str, str] = field(default_factory=dict)
76
+ skipped: Mapping[str, int] = field(default_factory=dict)
77
+
78
+
79
+ @dataclass(frozen=True, slots=True)
80
+ class TreeSpec:
81
+ """One pinned repository (or one subtree of it). `pins[0]` is the current edition."""
82
+
83
+ name: str
84
+ title: str
85
+ pins: tuple[Pin, ...]
86
+ include: Callable[[str], bool]
87
+ build: Callable[[Mapping[str, bytes], Pin], Build]
88
+
89
+
90
+ @dataclass(frozen=True, slots=True)
91
+ class Spec:
92
+ """Everything that is particular to one collection."""
93
+
94
+ name: str
95
+ title: str
96
+ licence: Licence
97
+ trees: tuple[TreeSpec, ...]
98
+ instructions: str
99
+ example_refs: tuple[str, ...] = ()
100
+
101
+ @property
102
+ def prefix(self) -> str:
103
+ return f"{self.name}_"
104
+
105
+ @property
106
+ def grammar(self) -> RefGrammar:
107
+ return RefGrammar(DOC_PATTERN, SECTION_PATTERN)
108
+
109
+
110
+ @dataclass(frozen=True, slots=True)
111
+ class Report:
112
+ """What one tree's import did."""
113
+
114
+ tree: str
115
+ commit: str
116
+ label: str
117
+ path: Path
118
+ docs: int
119
+ sections: int
120
+ units: int
121
+ files_sha256: str
122
+ pinned: bool
123
+ skipped: dict[str, int]
124
+ warnings: int
125
+
126
+
127
+ def _norm(text: str) -> str:
128
+ return " ".join(text.casefold().split()).rstrip(":.;,")
129
+
130
+
131
+ def _now() -> str:
132
+ return datetime.now(UTC).isoformat(timespec="seconds")
133
+
134
+
135
+ class Library:
136
+ """A collection's installed editions, read through a `Spec`. `home` replaces `$PRIMARYSOURCE_HOME` (tests)."""
137
+
138
+ def __init__(self, spec: Spec, home: Path | None = None) -> None:
139
+ self.spec = spec
140
+ self._home = home
141
+
142
+ # -- locations -------------------------------------------------------------------------------------------
143
+
144
+ def base(self, tree: str) -> Path:
145
+ root = self._home / self.spec.name if self._home is not None else collection_home(self.spec.name)
146
+ return root / tree
147
+
148
+ def _import(self) -> str:
149
+ return f"primarysource {self.spec.name} import"
150
+
151
+ def _editions(self) -> list[tuple[TreeSpec, store.Edition]]:
152
+ """The newest installed edition of each tree that has one."""
153
+ found: list[tuple[TreeSpec, store.Edition]] = []
154
+ for tree in self.spec.trees:
155
+ editions = store.installed(self.base(tree.name))
156
+ if editions:
157
+ found.append((tree, editions[0]))
158
+ if not found:
159
+ raise LibError(
160
+ f"{self.spec.title} is not imported (nothing under {self.base('')})",
161
+ f"{self.spec.prefix}status says what is pinned",
162
+ self._import(),
163
+ )
164
+ return found
165
+
166
+ def _by_commit(self, token: str) -> list[tuple[TreeSpec, store.Edition]]:
167
+ want = token.lower()
168
+ matches: list[tuple[TreeSpec, store.Edition]] = []
169
+ for tree in self.spec.trees:
170
+ labels = {p.label.lower(): p.commit for p in tree.pins}
171
+ for edition in store.installed(self.base(tree.name)):
172
+ if edition.commit.startswith(want) or labels.get(want) == edition.commit:
173
+ matches.append((tree, edition))
174
+ return matches
175
+
176
+ # -- import ----------------------------------------------------------------------------------------------
177
+
178
+ def import_edition(
179
+ self,
180
+ tree: str | None = None,
181
+ commit: str | None = None,
182
+ archive: str | None = None,
183
+ *,
184
+ opener: fetch.Opener | None = None,
185
+ throttle: fetch.Throttle | None = None,
186
+ limits: fetch.Limits | None = None,
187
+ allow_unpinned: bool = False,
188
+ ) -> list[Report]:
189
+ """Fetch (or read `archive`, a downloaded copy of) each selected tree at its pinned commit and install it."""
190
+ chosen = [t for t in self.spec.trees if tree in (None, t.name)]
191
+ if not chosen:
192
+ names = ", ".join(t.name for t in self.spec.trees)
193
+ raise LibError(f"no tree named {tree!r} in {self.spec.name} (trees: {names})", "")
194
+ if archive is not None and len(chosen) != 1:
195
+ names = ", ".join(t.name for t in self.spec.trees)
196
+ raise LibError("--archive needs --tree, because each tree has its own archive", f"trees: {names}")
197
+ return [self._import_tree(t, commit, archive, opener, throttle, limits, allow_unpinned) for t in chosen]
198
+
199
+ def _import_tree(
200
+ self,
201
+ tree: TreeSpec,
202
+ commit: str | None,
203
+ archive: str | None,
204
+ opener: fetch.Opener | None,
205
+ throttle: fetch.Throttle | None,
206
+ limits: fetch.Limits | None,
207
+ allow_unpinned: bool,
208
+ ) -> Report:
209
+ pin = tree.pins[0]
210
+ if commit is not None:
211
+ matches = [p for p in tree.pins if p.commit.startswith(commit.lower()) or p.label == commit]
212
+ if len(matches) != 1:
213
+ have = ", ".join(f"{p.label} ({p.commit[:12]})" for p in tree.pins)
214
+ raise LibError(f"{commit!r} is not one pinned edition of {tree.name} (pinned: {have})", "")
215
+ pin = matches[0]
216
+ expect = pin.files_sha256
217
+ if expect is None and not allow_unpinned:
218
+ raise LibError(
219
+ f"{tree.name} {pin.label} has no recorded digest",
220
+ "record the digest in the catalogue, or pass --allow-unpinned to trust this one fetch",
221
+ )
222
+ if allow_unpinned:
223
+ expect = None
224
+ if archive is not None:
225
+ first = pin.sources[0]
226
+ source = fetch.Source(archive, first.strip, subdir=first.subdir)
227
+ fetched = fetch.read_local(archive, source, include=tree.include, expect=expect, limits=limits)
228
+ else:
229
+ fetched = fetch.fetch_tree(
230
+ pin.sources, include=tree.include, expect=expect, limits=limits, opener=opener, throttle=throttle
231
+ )
232
+ built = tree.build(fetched.files, pin)
233
+ ids = [d.id for d in built.docs]
234
+ if not ids:
235
+ raise LibError(f"nothing in the {tree.name} archive was selected", "the source layout may have changed")
236
+ if len(set(ids)) != len(ids):
237
+ repeated = sorted({i for i in ids if ids.count(i) > 1})[:5]
238
+ raise LibError(
239
+ f"the build produced duplicate document ids: {', '.join(repeated)}", "report the collection"
240
+ )
241
+ record = {
242
+ "collection": self.spec.name,
243
+ "tree": tree.name,
244
+ "label": pin.label,
245
+ "date": pin.date,
246
+ "url": fetched.url,
247
+ "files_sha256": fetched.files_sha256,
248
+ "tar_sha256": fetched.tar_sha256,
249
+ "gz_sha256": fetched.gz_sha256,
250
+ "pinned": fetched.pinned,
251
+ "archive_members": fetched.members,
252
+ "selected_files": len(fetched.files),
253
+ "selected_bytes": fetched.kept_bytes,
254
+ "skipped": {**fetched.skipped, **dict(built.skipped)},
255
+ "licence": {
256
+ "spdx": self.spec.licence.spdx,
257
+ "redistribute": self.spec.licence.redistribute,
258
+ "attribution": self.spec.licence.attribution,
259
+ "evidence": self.spec.licence.evidence,
260
+ },
261
+ "imported": _now(),
262
+ }
263
+ path = store.write_edition(self.base(tree.name), pin.commit, record, built.docs, built.aliases)
264
+ sections = sum(len(d.sections) for d in built.docs)
265
+ return Report(
266
+ tree.name,
267
+ pin.commit,
268
+ pin.label,
269
+ path,
270
+ len(built.docs),
271
+ sections,
272
+ sum(len(s.units) for d in built.docs for s in d.sections),
273
+ fetched.files_sha256,
274
+ fetched.pinned,
275
+ {**fetched.skipped, **dict(built.skipped)},
276
+ sum(len(d.warnings) for d in built.docs),
277
+ )
278
+
279
+ # -- ref resolution --------------------------------------------------------------------------------------
280
+
281
+ def _parse(self, text: str) -> tuple[Ref, str | None]:
282
+ """(parsed ref, commit token or None). A leading `<collection>:` is accepted and ignored."""
283
+ cleaned = text.strip()
284
+ head = f"{self.spec.name}:"
285
+ if cleaned.lower().startswith(head):
286
+ cleaned = cleaned[len(head) :]
287
+ bare = _BARE.fullmatch(cleaned)
288
+ if bare: # `5.6.1`, `5.6.1@aceb1daf`, `5.6.1¶2`: a section number with no document
289
+ ref = self.spec.grammar.parse(f"§{bare['section']}{bare['para'] or ''}")
290
+ return ref, bare["commit"]
291
+ ref = self.spec.grammar.parse(cleaned)
292
+ doc, commit = ref.doc, None
293
+ if doc is not None and "@" in doc:
294
+ doc, _, commit = doc.partition("@")
295
+ section = ref.section.replace(_HASH_ESCAPE, "#") if ref.section else ref.section
296
+ return Ref(doc or None, section, ref.para, ref.anchor, ref.lines), commit
297
+
298
+ def _find_doc(self, token: str, commit: str | None) -> tuple[TreeSpec, store.Edition, str]:
299
+ pool = self._by_commit(commit) if commit else self._editions()
300
+ if commit and not pool:
301
+ raise LibError(f"no imported edition matches {commit!r}", f"{self.spec.prefix}status lists them")
302
+ for tree, edition in pool:
303
+ if token in edition.entries:
304
+ return tree, edition, token
305
+ if token in edition.aliases:
306
+ return tree, edition, edition.aliases[token]
307
+ lowered = {k.lower(): k for _, e in pool for k in e.entries}
308
+ if token.lower() in lowered:
309
+ for tree, edition in pool:
310
+ if lowered[token.lower()] in edition.entries:
311
+ return tree, edition, lowered[token.lower()]
312
+ known = sorted({k for _, e in pool for k in e.entries})
313
+ close = get_close_matches(token, known, n=5, cutoff=0.6)
314
+ hint = f"did you mean: {', '.join(close)}" if close else f"{self.spec.prefix}docs lists them"
315
+ raise LibError(f"no document {token!r} in {self.spec.title}", hint)
316
+
317
+ def _section(self, doc: Doc, token: str | None, anchor: str | None) -> tuple[Sec, str]:
318
+ """The section a ref names and a warning for the footer (empty when there is none)."""
319
+ if token is None and anchor is None:
320
+ if not doc.sections:
321
+ raise LibError(f"{doc.id} has no sections", "")
322
+ return doc.sections[0], ""
323
+ for key in (token, anchor):
324
+ if not key:
325
+ continue
326
+ found = [s for s in doc.sections if s.id == key] or [s for s in doc.sections if s.anchor == key]
327
+ if not found:
328
+ found = [s for s in doc.sections if s.anchor.lower() == key.lower()]
329
+ if found:
330
+ return found[0], ""
331
+ want = _norm(token or anchor or "")
332
+ exact = [s for s in doc.sections if _norm(s.title) == want]
333
+ if exact:
334
+ return exact[0], (
335
+ f"{len(exact)} sections are titled {token!r}; the first was read" if len(exact) > 1 else ""
336
+ )
337
+ partial = [s for s in doc.sections if want and want in _norm(s.title)]
338
+ if len(partial) > 1:
339
+ starts = [s for s in partial if _norm(s.title).startswith(want)]
340
+ partial = starts if len(starts) == 1 else partial
341
+ if len(partial) == 1:
342
+ return partial[0], ""
343
+ if partial:
344
+ listing = "; ".join(f"{s.id} {s.title}" for s in partial[:8])
345
+ raise LibError(f"{token!r} matches {len(partial)} sections of {doc.id}", f"name one by id: {listing}")
346
+ titles = [s.title for s in doc.sections]
347
+ close = get_close_matches(token or "", titles, n=3, cutoff=0.5)
348
+ hint = f"did you mean: {'; '.join(close)}" if close else f"{self.spec.prefix}outline('{doc.id}') lists them"
349
+ raise LibError(f"no section {token or anchor!r} in {doc.id}", hint)
350
+
351
+ def _target(self, text: str, *, subtree: bool = False) -> tuple[store.Edition, Doc, Sec, str, int | None]:
352
+ """The edition, document, section, warning and `¶n` a ref names. `subtree` lets `¶n` count on past the
353
+ section's own blocks into its subsections, as a read does."""
354
+ ref, commit = self._parse(text)
355
+ if ref.lines is not None:
356
+ raise LibError("a line range is not a ref in this collection", "name a section: <doc>§<section>")
357
+ doc_token, section = ref.doc, ref.section
358
+ if doc_token is None:
359
+ if section is None:
360
+ raise LibError(f"not a ref: {text!r}", "expected <doc>§<section>[¶n]")
361
+ # a bare section number: its first component names a chapter alias, as in `5.6.1`
362
+ doc_token = section.split(".")[0]
363
+ elif section is None and ref.anchor is None and doc_token not in self._known_ids(commit) and "." in doc_token:
364
+ doc_token, section = doc_token.split(".")[0], doc_token
365
+ _, edition, doc_id = self._find_doc(doc_token, commit)
366
+ doc = edition.load(doc_id)
367
+ sec, warning = self._section(doc, section, ref.anchor)
368
+ count = len(self._ordered(doc, sec)) - 1 if subtree else len(sec.units)
369
+ if ref.para is not None and not 1 <= ref.para <= count:
370
+ what = "blocks" if count == len(sec.units) else "blocks and headings with its subsections"
371
+ raise LibError(
372
+ f"{doc.id}§{sec.id} has {count} {what}; there is no ¶{ref.para}",
373
+ f"{self.spec.prefix}read('{self._ref(edition, doc, sec)}')",
374
+ )
375
+ return edition, doc, sec, warning, ref.para
376
+
377
+ def _known_ids(self, commit: str | None) -> set[str]:
378
+ pool = self._by_commit(commit) if commit else self._editions()
379
+ return {k for _, e in pool for k in (*e.entries, *e.aliases)}
380
+
381
+ def _ref(self, edition: store.Edition, doc: Doc, sec: Sec, para: int | None = None) -> str:
382
+ text = f"{self.spec.name}:{doc.id}@{edition.commit[:12]}§{sec.id}"
383
+ return text + (f"¶{para}" if para else "")
384
+
385
+ def _citation(self, edition: store.Edition, doc: Doc, sec: Sec, para: int | None) -> str:
386
+ source = doc.meta.get("path", "")
387
+ label = f" {edition.label}" if edition.label else ""
388
+ tree = str(edition.record.get("tree", ""))
389
+ lic = self.spec.licence
390
+ parts = [
391
+ f'{doc.title}, "{sec.title}" §{sec.id}' + (f" ¶{para}" if para else ""),
392
+ f"{self.spec.title} ({tree}{label}, commit {edition.commit[:12]})",
393
+ ]
394
+ if source:
395
+ parts.append(source)
396
+ parts.append(f"licence {lic.spdx}" + (f" ({lic.credit})" if lic.credit else ""))
397
+ return self._ref(edition, doc, sec, para) + " -- " + "; ".join(parts)
398
+
399
+ # -- the tools -------------------------------------------------------------------------------------------
400
+
401
+ @staticmethod
402
+ def _ordered(doc: Doc, sec: Sec) -> list[str]:
403
+ """The units a read of `sec` pages through: its heading, its blocks, then each subsection's heading and
404
+ blocks. `¶n` is `ordered[n]`, so the section's own `¶n` keeps its meaning."""
405
+ ordered: list[str] = []
406
+ inside = False
407
+ for candidate in doc.sections:
408
+ if candidate is sec:
409
+ inside = True
410
+ elif inside and candidate.depth <= sec.depth:
411
+ break
412
+ if inside:
413
+ ordered.append("#" * min(candidate.depth, 6) + f" {candidate.id} {candidate.title}")
414
+ ordered.extend(candidate.units)
415
+ return ordered
416
+
417
+ def docs(self, prefix: str = "", limit: int = 200, start: int = 0) -> str:
418
+ """One line per document: `<id> <title>`, optionally those whose id starts with `prefix`."""
419
+ rows: list[str] = []
420
+ for _, edition in self._editions():
421
+ rows.extend(f"{e.id} {e.title}" for e in edition.entries.values() if e.id.startswith(prefix))
422
+ if not rows:
423
+ return f"no documents with an id starting {prefix!r}\n" if prefix else "no documents\n"
424
+ chosen, nxt = cursor.window(rows, start, limit)
425
+ return render("\n".join(chosen), {"unit": f"{len(rows)} documents", "next": nxt})
426
+
427
+ def outline(self, ref: str, start: int = 0, max_tokens: int = 4000) -> str:
428
+ """A document's sections, each with the ref that reads it. `ref` names a document (`gitformat-pack`)."""
429
+ parsed, commit = self._parse(ref)
430
+ token = parsed.doc or (parsed.section or "").split(".")[0]
431
+ if not token:
432
+ raise LibError(f"not a document: {ref!r}", f"primarysource {self.spec.name} docs")
433
+ _, edition, doc_id = self._find_doc(token, commit)
434
+ doc = edition.load(doc_id)
435
+ lines = [
436
+ f"{' ' * (s.depth - 1)}{s.id} {s.title} [{len(s.units)}] {self._ref(edition, doc, s)}"
437
+ for s in doc.sections
438
+ ]
439
+ if start < 0:
440
+ raise LibError(f"start is a 0-based offset, not {start}", "pass the start= value of the last next: line")
441
+ taken = page(lines, start + 1, max_tokens)
442
+ footer: dict[str, str] = {"cite": self._citation(edition, doc, doc.sections[0], None)} if doc.sections else {}
443
+ footer["unit"] = "block (the [n] after a title counts them)"
444
+ footer["next"] = next_start(None if taken.next_start is None else taken.next_start - 1)
445
+ if doc.warnings:
446
+ footer["warning"] = "; ".join(doc.warnings)
447
+ return render(f"{doc.title}\n" + "\n".join(taken.units), footer)
448
+
449
+ def read(self, ref: str, max_tokens: int = 2000) -> str:
450
+ """The text of a section, its subsections included, cut at block boundaries: from its heading, or from block
451
+ n with `¶n`. The `next:` footer is the ref that reads the next page."""
452
+ edition, doc, sec, warning, para = self._target(ref, subtree=True)
453
+ if max_tokens < 1:
454
+ raise LibError("max_tokens must be positive", "")
455
+ taken = page(self._ordered(doc, sec), (para or 0) + 1, max_tokens)
456
+ body = "\n\n".join(taken.units)
457
+ one = para if para is not None and len(taken.units) == 1 and para <= len(sec.units) else None
458
+ footer = {
459
+ "cite": self._citation(edition, doc, sec, one),
460
+ "unit": "block (¶n counts the blocks of one section, then on through its subsections)",
461
+ "span": span_hash(body),
462
+ "next": "" if taken.next_start is None else self._ref(edition, doc, sec, taken.next_start - 1),
463
+ }
464
+ notes = [w for w in (warning, *doc.warnings) if w]
465
+ if notes:
466
+ footer["warning"] = "; ".join(notes)
467
+ return render(body, footer)
468
+
469
+ def cite(self, ref: str) -> str:
470
+ """The citation for a ref, with the hash of the exact text cited, and nothing else."""
471
+ edition, doc, sec, _, para = self._target(ref)
472
+ text = sec.units[para - 1] if para else "\n\n".join(sec.units)
473
+ return render(self._citation(edition, doc, sec, para), {"span": span_hash(text)})
474
+
475
+ def search(self, query: str, limit: int = 10, start: int = 0, doc_prefix: str = "") -> str:
476
+ """Sections matching the words of `query`, best first, one line each with the ref that reads it."""
477
+ if start < 0 or limit < 1:
478
+ raise LibError(f"start is a 0-based offset and limit positive (got {start}, {limit})", "")
479
+ scored: list[tuple[float, str]] = []
480
+ relaxed = False
481
+ for _, edition in self._editions():
482
+ hits, was_relaxed = index.search(
483
+ edition.index_path, query, limit=start + limit + 1, doc_prefix=doc_prefix
484
+ )
485
+ relaxed = relaxed or was_relaxed
486
+ for hit in hits:
487
+ doc = edition.entries[hit.doc]
488
+ ref = f"{self.spec.name}:{hit.doc}@{edition.commit[:12]}§{hit.section}"
489
+ scored.append((hit.score, f"{ref}: {doc.title} / {hit.title} -- {hit.snippet}"))
490
+ scored.sort(key=lambda row: row[0])
491
+ window, nxt = cursor.window([line for _, line in scored], start, limit)
492
+ if not window:
493
+ return render("no sections matched", {})
494
+ footer: dict[str, str] = {"next": nxt}
495
+ if relaxed:
496
+ footer["relaxed"] = "no section had every word; these have some of them"
497
+ return render("\n".join(window), footer)
498
+
499
+ def status(self) -> str:
500
+ """What is pinned, what is imported, and the licence."""
501
+ lic = self.spec.licence
502
+ lines = [
503
+ f"{self.spec.title}",
504
+ f"licence: {lic.spdx} ({lic.redistribute}){'; ' + lic.attribution if lic.attribution else ''}",
505
+ ]
506
+ for tree in self.spec.trees:
507
+ have = {e.commit: e for e in store.installed(self.base(tree.name))}
508
+ lines.append(f"tree {tree.name}: {tree.title}")
509
+ for pin in tree.pins:
510
+ got = have.get(pin.commit)
511
+ state = (
512
+ f"imported {got.record.get('imported', '')} ({got.record.get('docs', '?')} documents)"
513
+ if got
514
+ else "not imported"
515
+ )
516
+ lines.append(f" {pin.label} {pin.commit[:12]} {pin.date}: {state}")
517
+ others = [c[:12] for c in have if c not in {p.commit for p in tree.pins}]
518
+ if others:
519
+ lines.append(f" other imported commits: {', '.join(others)}")
520
+ return "\n".join(lines) + "\n"
521
+
522
+
523
+ def pin_files(*names: str) -> Callable[[str], bool]:
524
+ """An `include` that accepts exactly these paths."""
525
+ wanted = frozenset(names)
526
+ return lambda path: path in wanted
527
+
528
+
529
+ def matching(pattern: str) -> Callable[[str], bool]:
530
+ """An `include` that accepts paths matching the regular expression `pattern` (fully)."""
531
+ compiled = re.compile(pattern)
532
+ return lambda path: compiled.fullmatch(path) is not None
533
+
534
+
535
+ def parse_docs(
536
+ files: Mapping[str, bytes], reader: Callable[[bytes, str], Doc], ids: Sequence[tuple[str, str]]
537
+ ) -> list[Doc]:
538
+ """Read `(path, doc id)` pairs with `reader`, recording the source path in each document's meta."""
539
+ return [reader(files[path], doc_id).with_meta(path=path) for path, doc_id in ids if path in files]
540
+
541
+
542
+ def make_collection(spec: Spec, package: str) -> Collection:
543
+ """The registry entry for `spec`, whose `server` and `cli` targets live in `package` (`server.py`, `cli.py`)."""
544
+ return Collection(
545
+ name=spec.name,
546
+ title=spec.title,
547
+ prefix=spec.prefix,
548
+ unit=Unit.BLOCK,
549
+ grammar=spec.grammar,
550
+ server=f"{package}.server:server",
551
+ cli=f"{package}.cli:main",
552
+ )