primarysource 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (303) hide show
  1. primarysource/__init__.py +5 -0
  2. primarysource/bbf/__init__.py +56 -0
  3. primarysource/bbf/_xml.py +63 -0
  4. primarysource/bbf/catalogue.json +777 -0
  5. primarysource/bbf/catalogue.py +137 -0
  6. primarysource/bbf/cli.py +79 -0
  7. primarysource/bbf/datamodel.py +281 -0
  8. primarysource/bbf/importer.py +151 -0
  9. primarysource/bbf/library.py +380 -0
  10. primarysource/bbf/server.py +72 -0
  11. primarysource/budget.py +63 -0
  12. primarysource/c2sp/__init__.py +53 -0
  13. primarysource/c2sp/_fetch.py +5 -0
  14. primarysource/c2sp/catalogue.json +231 -0
  15. primarysource/c2sp/catalogue.py +95 -0
  16. primarysource/c2sp/cli.py +62 -0
  17. primarysource/c2sp/importer.py +90 -0
  18. primarysource/c2sp/library.py +309 -0
  19. primarysource/c2sp/markdown.py +31 -0
  20. primarysource/c2sp/server.py +68 -0
  21. primarysource/cavp/__init__.py +42 -0
  22. primarysource/cavp/cli.py +84 -0
  23. primarysource/cavp/data/vectors.toml +47 -0
  24. primarysource/cavp/importer.py +156 -0
  25. primarysource/cavp/model.py +279 -0
  26. primarysource/cavp/parse.py +269 -0
  27. primarysource/cavp/paths.py +17 -0
  28. primarysource/cavp/server.py +67 -0
  29. primarysource/cavp/tools.py +169 -0
  30. primarysource/cli.py +107 -0
  31. primarysource/cursor.py +82 -0
  32. primarysource/cve/__init__.py +5 -0
  33. primarysource/cve/__main__.py +6 -0
  34. primarysource/cve/cli.py +144 -0
  35. primarysource/cve/collection.py +20 -0
  36. primarysource/cve/config.py +58 -0
  37. primarysource/cve/errors.py +16 -0
  38. primarysource/cve/kernel.py +232 -0
  39. primarysource/cve/mcp.py +139 -0
  40. primarysource/cve/parse.py +318 -0
  41. primarysource/cve/py.typed +0 -0
  42. primarysource/cve/service.py +760 -0
  43. primarysource/cve/store.py +214 -0
  44. primarysource/cve/sync.py +438 -0
  45. primarysource/cve/views.py +304 -0
  46. primarysource/debpolicy/__init__.py +16 -0
  47. primarysource/debpolicy/catalogue.py +263 -0
  48. primarysource/debpolicy/cli.py +12 -0
  49. primarysource/debpolicy/server.py +11 -0
  50. primarysource/draft/__init__.py +11 -0
  51. primarysource/draft/catalogue.py +117 -0
  52. primarysource/draft/cli.py +149 -0
  53. primarysource/draft/collection.py +21 -0
  54. primarysource/draft/datatracker.py +158 -0
  55. primarysource/draft/importer.py +163 -0
  56. primarysource/draft/licence.py +89 -0
  57. primarysource/draft/names.py +64 -0
  58. primarysource/draft/net.py +133 -0
  59. primarysource/draft/segment.py +192 -0
  60. primarysource/draft/server.py +91 -0
  61. primarysource/draft/status.py +146 -0
  62. primarysource/draft/store.py +199 -0
  63. primarysource/draft/tracked.json +104 -0
  64. primarysource/draft/views.py +430 -0
  65. primarysource/draft/xmlmeta.py +74 -0
  66. primarysource/ecma/__init__.py +52 -0
  67. primarysource/ecma/_pdf.py +143 -0
  68. primarysource/ecma/catalogue.json +22 -0
  69. primarysource/ecma/catalogue.py +100 -0
  70. primarysource/ecma/cli.py +67 -0
  71. primarysource/ecma/identify.py +25 -0
  72. primarysource/ecma/importer.py +108 -0
  73. primarysource/ecma/library.py +240 -0
  74. primarysource/ecma/server.py +67 -0
  75. primarysource/errors.py +29 -0
  76. primarysource/fetch.py +360 -0
  77. primarysource/fsutil.py +150 -0
  78. primarysource/gitdocs/__init__.py +15 -0
  79. primarysource/gitdocs/catalogue.py +116 -0
  80. primarysource/gitdocs/cli.py +12 -0
  81. primarysource/gitdocs/server.py +8 -0
  82. primarysource/home.py +65 -0
  83. primarysource/iana/__init__.py +32 -0
  84. primarysource/iana/catalogue.json +244 -0
  85. primarysource/iana/catalogue.py +69 -0
  86. primarysource/iana/cli.py +69 -0
  87. primarysource/iana/importer.py +157 -0
  88. primarysource/iana/model.py +325 -0
  89. primarysource/iana/reader.py +315 -0
  90. primarysource/iana/server.py +67 -0
  91. primarysource/iana/store.py +83 -0
  92. primarysource/iana/xmlsafe.py +47 -0
  93. primarysource/identify.py +186 -0
  94. primarysource/ieee/__init__.py +58 -0
  95. primarysource/ieee/_html.py +49 -0
  96. primarysource/ieee/catalogue.json +74 -0
  97. primarysource/ieee/catalogue.py +123 -0
  98. primarysource/ieee/cli.py +79 -0
  99. primarysource/ieee/identify.py +24 -0
  100. primarysource/ieee/ieee802.py +47 -0
  101. primarysource/ieee/importer.py +305 -0
  102. primarysource/ieee/posix.py +233 -0
  103. primarysource/ieee/server.py +79 -0
  104. primarysource/index.py +446 -0
  105. primarysource/iso/__init__.py +51 -0
  106. primarysource/iso/cli.py +98 -0
  107. primarysource/iso/designation.py +95 -0
  108. primarysource/iso/identify.py +100 -0
  109. primarysource/iso/importer.py +224 -0
  110. primarysource/iso/library.py +249 -0
  111. primarysource/iso/server.py +69 -0
  112. primarysource/itu/__init__.py +41 -0
  113. primarysource/itu/__main__.py +8 -0
  114. primarysource/itu/asn1mod.py +478 -0
  115. primarysource/itu/catalog.py +472 -0
  116. primarysource/itu/cli.py +445 -0
  117. primarysource/itu/collection.py +20 -0
  118. primarysource/itu/corpus.py +118 -0
  119. primarysource/itu/errata.py +16 -0
  120. primarysource/itu/formal.py +565 -0
  121. primarysource/itu/identify.py +26 -0
  122. primarysource/itu/index.py +490 -0
  123. primarysource/itu/mcp.py +474 -0
  124. primarysource/itu/py.typed +0 -0
  125. primarysource/itu/sources.py +600 -0
  126. primarysource/itu/text.py +609 -0
  127. primarysource/kconfig/__init__.py +16 -0
  128. primarysource/kconfig/catalogue.py +98 -0
  129. primarysource/kconfig/cli.py +12 -0
  130. primarysource/kconfig/server.py +8 -0
  131. primarysource/model.py +63 -0
  132. primarysource/nist/__init__.py +5 -0
  133. primarysource/nist/collection.py +46 -0
  134. primarysource/nist/data/acquired-nist.json +1657 -0
  135. primarysource/nist/data/nist-catalog.json +12090 -0
  136. primarysource/nist/data/sources.toml +1996 -0
  137. primarysource/nist/identify.py +41 -0
  138. primarysource/nist/ingest/__init__.py +1 -0
  139. primarysource/nist/ingest/acquire_nist.py +128 -0
  140. primarysource/nist/ingest/controls.py +82 -0
  141. primarysource/nist/ingest/discover_nist.py +218 -0
  142. primarysource/nist/ingest/extract.py +48 -0
  143. primarysource/nist/ingest/fetch.py +211 -0
  144. primarysource/nist/ingest/index.py +14 -0
  145. primarysource/nist/ingest/oscal.py +480 -0
  146. primarysource/nist/ingest/prose_layouts.py +236 -0
  147. primarysource/nist/ingest/sources.py +94 -0
  148. primarysource/nist/ingest/status.py +255 -0
  149. primarysource/nist/models.py +7 -0
  150. primarysource/nist/paths.py +33 -0
  151. primarysource/nist/server/__init__.py +3 -0
  152. primarysource/nist/server/corpus.py +38 -0
  153. primarysource/nist/server/mcp.py +19 -0
  154. primarysource/nist/server/tools.py +35 -0
  155. primarysource/nist/spec.py +38 -0
  156. primarysource/pdfcorpus/__init__.py +16 -0
  157. primarysource/pdfcorpus/corpus.py +159 -0
  158. primarysource/pdfcorpus/fts.py +175 -0
  159. primarysource/pdfcorpus/index.py +70 -0
  160. primarysource/pdfcorpus/models.py +30 -0
  161. primarysource/pdfcorpus/server.py +96 -0
  162. primarysource/pdfcorpus/spec.py +37 -0
  163. primarysource/pdfcorpus/split.py +391 -0
  164. primarysource/pdfcorpus/tools.py +244 -0
  165. primarysource/pep/__init__.py +15 -0
  166. primarysource/pep/catalogue.py +191 -0
  167. primarysource/pep/cli.py +12 -0
  168. primarysource/pep/server.py +8 -0
  169. primarysource/pkgdocs/__init__.py +19 -0
  170. primarysource/pkgdocs/fetch.py +356 -0
  171. primarysource/pkgdocs/index.py +89 -0
  172. primarysource/pkgdocs/model.py +76 -0
  173. primarysource/pkgdocs/readers/__init__.py +1 -0
  174. primarysource/pkgdocs/readers/_common.py +134 -0
  175. primarysource/pkgdocs/readers/asciidoc.py +160 -0
  176. primarysource/pkgdocs/readers/latex.py +315 -0
  177. primarysource/pkgdocs/readers/markdown.py +173 -0
  178. primarysource/pkgdocs/readers/pod.py +132 -0
  179. primarysource/pkgdocs/readers/rst.py +239 -0
  180. primarysource/pkgdocs/readers/scdoc.py +108 -0
  181. primarysource/pkgdocs/runtime.py +552 -0
  182. primarysource/pkgdocs/serving.py +158 -0
  183. primarysource/pkgdocs/store.py +167 -0
  184. primarysource/pms/__init__.py +16 -0
  185. primarysource/pms/catalogue.py +88 -0
  186. primarysource/pms/cli.py +12 -0
  187. primarysource/pms/server.py +8 -0
  188. primarysource/py.typed +0 -0
  189. primarysource/readers/__init__.py +10 -0
  190. primarysource/readers/bookmarks.py +192 -0
  191. primarysource/readers/html.py +1312 -0
  192. primarysource/readers/markdown.py +189 -0
  193. primarysource/readers/pdf.py +187 -0
  194. primarysource/readers/xml.py +230 -0
  195. primarysource/refs.py +87 -0
  196. primarysource/registry.py +161 -0
  197. primarysource/render.py +49 -0
  198. primarysource/rfc/__init__.py +3 -0
  199. primarysource/rfc/__main__.py +6 -0
  200. primarysource/rfc/cite.py +204 -0
  201. primarysource/rfc/cli.py +903 -0
  202. primarysource/rfc/collection.py +19 -0
  203. primarysource/rfc/config.py +138 -0
  204. primarysource/rfc/errata.py +203 -0
  205. primarysource/rfc/fsutil.py +82 -0
  206. primarysource/rfc/graph.py +213 -0
  207. primarysource/rfc/index/__init__.py +1 -0
  208. primarysource/rfc/index/chunker.py +281 -0
  209. primarysource/rfc/index/embed.py +166 -0
  210. primarysource/rfc/index/hybrid.py +305 -0
  211. primarysource/rfc/index/lexical.py +223 -0
  212. primarysource/rfc/index/rfcxml.py +673 -0
  213. primarysource/rfc/index/search.py +508 -0
  214. primarysource/rfc/index/txtseg.py +585 -0
  215. primarysource/rfc/library.py +294 -0
  216. primarysource/rfc/mcp/__init__.py +1 -0
  217. primarysource/rfc/mcp/server.py +381 -0
  218. primarysource/rfc/packs/__init__.py +1 -0
  219. primarysource/rfc/packs/builder.py +228 -0
  220. primarysource/rfc/packs/collections/all-current-txt.json +10 -0
  221. primarysource/rfc/packs/collections/essentials.json +19 -0
  222. primarysource/rfc/packs/collections/fyi-current.json +10 -0
  223. primarysource/rfc/packs/collections/standards-current.json +10 -0
  224. primarysource/rfc/packs/collections/year-2025.json +10 -0
  225. primarysource/rfc/packs/collections/year-2026.json +10 -0
  226. primarysource/rfc/packs/collections.py +163 -0
  227. primarysource/rfc/packs/docsetexport.py +163 -0
  228. primarysource/rfc/packs/epub.py +159 -0
  229. primarysource/rfc/packs/export.py +306 -0
  230. primarysource/rfc/packs/htmlexport.py +82 -0
  231. primarysource/rfc/packs/serve.py +143 -0
  232. primarysource/rfc/packs/zimexport.py +171 -0
  233. primarysource/rfc/query.py +466 -0
  234. primarysource/rfc/subjects.py +457 -0
  235. primarysource/rfc/sync/__init__.py +1 -0
  236. primarysource/rfc/sync/downloader.py +132 -0
  237. primarysource/rfc/sync/manifest.py +162 -0
  238. primarysource/rfc/sync/scanner.py +49 -0
  239. primarysource/rfc/sync/state.py +464 -0
  240. primarysource/rfc/views.py +1067 -0
  241. primarysource/rpmdocs/__init__.py +15 -0
  242. primarysource/rpmdocs/catalogue.py +117 -0
  243. primarysource/rpmdocs/cli.py +12 -0
  244. primarysource/rpmdocs/server.py +8 -0
  245. primarysource/scaffold.py +223 -0
  246. primarysource/serve.py +75 -0
  247. primarysource/settings.py +152 -0
  248. primarysource/usb/__init__.py +5 -0
  249. primarysource/usb/collection.py +37 -0
  250. primarysource/usb/data/acquired.json +2025 -0
  251. primarysource/usb/data/sources-usb.toml +852 -0
  252. primarysource/usb/data/sources.toml +10 -0
  253. primarysource/usb/data/usb-catalog.json +1663 -0
  254. primarysource/usb/data/usb-classification.json +1986 -0
  255. primarysource/usb/data/usb-corpus-build.json +608 -0
  256. primarysource/usb/data/usb-pdf-choice.json +839 -0
  257. primarysource/usb/data/usb-split-measure.json +1803 -0
  258. primarysource/usb/ingest/__init__.py +1 -0
  259. primarysource/usb/ingest/acquire.py +130 -0
  260. primarysource/usb/ingest/discover_usb.py +88 -0
  261. primarysource/usb/ingest/extract.py +48 -0
  262. primarysource/usb/ingest/fetch.py +197 -0
  263. primarysource/usb/ingest/index.py +14 -0
  264. primarysource/usb/ingest/sources.py +85 -0
  265. primarysource/usb/models.py +7 -0
  266. primarysource/usb/paths.py +33 -0
  267. primarysource/usb/server/__init__.py +3 -0
  268. primarysource/usb/server/corpus.py +38 -0
  269. primarysource/usb/server/mcp.py +19 -0
  270. primarysource/usb/server/tools.py +35 -0
  271. primarysource/usb/spec.py +20 -0
  272. primarysource/usbids/__init__.py +56 -0
  273. primarysource/usbids/cli.py +65 -0
  274. primarysource/usbids/importer.py +272 -0
  275. primarysource/usbids/parse.py +283 -0
  276. primarysource/usbids/server.py +55 -0
  277. primarysource/usbids/store.py +80 -0
  278. primarysource/usbids/tools.py +174 -0
  279. primarysource/zim/__init__.py +5 -0
  280. primarysource/zim/__main__.py +8 -0
  281. primarysource/zim/_libzim.py +37 -0
  282. primarysource/zim/catalog.py +594 -0
  283. primarysource/zim/cli.py +555 -0
  284. primarysource/zim/collection.py +41 -0
  285. primarysource/zim/errors.py +100 -0
  286. primarysource/zim/groups.py +30 -0
  287. primarysource/zim/library.py +668 -0
  288. primarysource/zim/manage.py +636 -0
  289. primarysource/zim/markdown.py +1247 -0
  290. primarysource/zim/mcp.py +201 -0
  291. primarysource/zim/purl.py +130 -0
  292. primarysource/zim/py.typed +0 -0
  293. primarysource/zim/refs.py +90 -0
  294. primarysource/zim/service.py +1182 -0
  295. primarysource/zim/snapshot.py +197 -0
  296. primarysource/zim/sync.py +269 -0
  297. primarysource/zim/views.py +230 -0
  298. primarysource-0.1.0.dist-info/METADATA +118 -0
  299. primarysource-0.1.0.dist-info/RECORD +303 -0
  300. primarysource-0.1.0.dist-info/WHEEL +4 -0
  301. primarysource-0.1.0.dist-info/entry_points.txt +2 -0
  302. primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
  303. primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
@@ -0,0 +1,67 @@
1
+ """MCP server for the iana collection: `primarysource mcp --collection iana`.
2
+
3
+ Imported only when served, so `mcp` (the `[mcp]` extra) is needed only then. Tools carry the prefix `iana_`, answer
4
+ in text with the fixed footer, and turn a `LibError` into an MCP tool error naming the remedy. Every tool reads local
5
+ files only; none writes or fetches (importing is `primarysource iana import`).
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ from typing import Any
11
+
12
+ from mcp.server.mcpserver import MCPServer
13
+ from mcp.types import ToolAnnotations
14
+
15
+ from primarysource import __version__
16
+ from primarysource.iana import reader
17
+ from primarysource.scaffold import add_tools
18
+
19
+ READ_ONLY = ToolAnnotations(read_only_hint=True, idempotent_hint=True, destructive_hint=False, open_world_hint=False)
20
+
21
+ INSTRUCTIONS = (
22
+ "IANA protocol registries (DNS, DHCP, BGP, STUN, TLS, HTTP, ...) from the user's imported copies of the XML "
23
+ "at iana.org/assignments, which is CC0. Registry values are authoritative data: use iana_lookup for 'what is "
24
+ "DHCP option 53' and iana_search to find a registry. Refs: 'dns-parameters' (the file's registries), "
25
+ "'dns-parameters§dns-parameters-4' (one registry), 'bgp-parameters§bgp-parameters-2#3' (the record with value "
26
+ "3), '...¶4' (the 4th record). Each row carries its RFC citations (RFC4271§4.3 reads in the rfc collection). "
27
+ "The cite: footer line is the citation. If a registry is not imported the error says how."
28
+ )
29
+
30
+
31
+ def iana_read(ref: str, max_tokens: int = 2000) -> str:
32
+ """Read an IANA registry by ref, cut at record boundaries within max_tokens. A 'next: <ref>' footer line is the
33
+ ref that reads the next page: pass it back as ref.
34
+
35
+ ref: 'dns-parameters' (list its registries) | 'dns-parameters§dns-parameters-4' (a registry: procedure,
36
+ notes, then records) | 'bgp-parameters§bgp-parameters-2#3' (records whose value is 3) |
37
+ 'bgp-parameters§bgp-parameters-2¶4' (from the 4th record). 'dns-parameters#dns-parameters-4' also works."""
38
+ return reader.read(ref, max_tokens=max_tokens)
39
+
40
+
41
+ def iana_lookup(registry: str, value: str) -> str:
42
+ """The record(s) with a value in a registry, e.g. registry 'bootp-dhcp-parameters§message-type-53', value '5'.
43
+ registry may be just a file name ('dns-parameters') to try every registry in it. A number matches a record whose
44
+ value is that number or a range containing it (decimal or 0x hex); text matches a value or a column exactly,
45
+ ignoring case ('A' finds RR type A)."""
46
+ return reader.lookup(registry, value)
47
+
48
+
49
+ def iana_search(text: str, file: str | None = None, limit: int = 20, start: int = 0) -> str:
50
+ """Find records whose registry title and columns contain every word of text (case-insensitive, literal, at most
51
+ 200 characters). file: restrict to one registry file, e.g. 'stun-parameters'. One line per hit:
52
+ '<ref> <registry>: <record>'. A 'next: start=N' footer line means call again with start=N (0-based)."""
53
+ return reader.search(text, file=file, limit=limit, start=start)
54
+
55
+
56
+ def iana_list() -> str:
57
+ """The catalogue of registries this collection knows, each marked imported or not, with its updated date."""
58
+ return reader.listing()
59
+
60
+
61
+ def _server() -> Any:
62
+ server = MCPServer("primarysource-iana", version=__version__, instructions=INSTRUCTIONS)
63
+ add_tools(server, (iana_read, iana_lookup, iana_search, iana_list), annotations=READ_ONLY)
64
+ return server
65
+
66
+
67
+ server = _server()
@@ -0,0 +1,83 @@
1
+ """Where imported registry files live and how they are read back.
2
+
3
+ `$PRIMARYSOURCE_HOME/iana/<name>.xml` is the file exactly as IANA served it; `manifest.json` beside them records,
4
+ per file, its URL, sha256, size, the registry's own `updated` date, when it was fetched and the server's
5
+ `Last-Modified`. Reading parses a file once per (path, mtime, size) and checks it against its recorded sha256.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ from dataclasses import dataclass
13
+ from functools import lru_cache
14
+ from pathlib import Path
15
+ from typing import Any
16
+
17
+ from primarysource.errors import LibError
18
+ from primarysource.home import collection_home
19
+ from primarysource.iana import xmlsafe
20
+ from primarysource.iana.catalogue import NAME
21
+ from primarysource.iana.model import RegistryFile, from_tree
22
+
23
+ MANIFEST = "manifest.json"
24
+ MAX_BYTES = 32_000_000
25
+ """Ceiling on one registry file. The largest catalogued file (ipfix) is under 0.5 MB; the biggest IANA registries
26
+ (media-types, service-names-port-numbers) are a few MB."""
27
+
28
+
29
+ def home() -> Path:
30
+ return collection_home("iana")
31
+
32
+
33
+ def read_manifest(base: Path) -> dict[str, dict[str, Any]]:
34
+ path = base / MANIFEST
35
+ if not path.is_file():
36
+ return {}
37
+ try:
38
+ files = json.loads(path.read_text(encoding="utf-8"))["files"]
39
+ except (ValueError, KeyError, TypeError) as exc:
40
+ raise LibError(f"{path} is not a readable manifest", "", f"delete {path} and import again") from exc
41
+ if not isinstance(files, dict):
42
+ raise LibError(f"{path} is not a readable manifest", "", f"delete {path} and import again")
43
+ return {str(k): dict(v) for k, v in files.items() if isinstance(v, dict)}
44
+
45
+
46
+ def imported(base: Path | None = None) -> tuple[str, ...]:
47
+ """Names of the registry files present, sorted."""
48
+ root = base or home()
49
+ if not root.is_dir():
50
+ return ()
51
+ return tuple(sorted(p.stem for p in root.glob("*.xml") if NAME.fullmatch(p.stem)))
52
+
53
+
54
+ @dataclass(frozen=True, slots=True)
55
+ class Loaded:
56
+ file: RegistryFile
57
+ sha256: str
58
+ digest_ok: bool | None
59
+ """True if the manifest's sha256 equals the file's, False if it differs, None if the manifest has no entry."""
60
+
61
+
62
+ @lru_cache(maxsize=16)
63
+ def _parse(path: str, mtime_ns: int, size: int, name: str) -> tuple[RegistryFile, str]:
64
+ raw = Path(path).read_bytes()
65
+ return from_tree(xmlsafe.parse(raw, MAX_BYTES), name), hashlib.sha256(raw).hexdigest()
66
+
67
+
68
+ def load(name: str, base: Path | None = None) -> Loaded:
69
+ """One imported registry file, parsed. A missing file is an error that names the import command."""
70
+ if not NAME.fullmatch(name):
71
+ raise LibError(f"{name!r} is not a registry file name", "e.g. dns-parameters, bgp-parameters")
72
+ root = base or home()
73
+ path = root / f"{name}.xml"
74
+ if not path.is_file():
75
+ raise LibError(
76
+ f"registry {name!r} is not imported (looked in {root})",
77
+ "iana_list shows the catalogue and what is imported",
78
+ f"primarysource iana import {name}",
79
+ )
80
+ stat = path.stat()
81
+ parsed, digest = _parse(str(path), stat.st_mtime_ns, stat.st_size, name)
82
+ recorded = read_manifest(root).get(name, {}).get("sha256")
83
+ return Loaded(parsed, digest, None if recorded is None else recorded == digest)
@@ -0,0 +1,47 @@
1
+ """The iana collection's XML reader: the core's hardened `primarysource.readers.xml` with iana's bounds.
2
+
3
+ The parser moved to the core (`cl.reader_xml_hardened`). This wrapper keeps the signature the importer, the store
4
+ and their tests use, and the two choices iana makes on top of the core's defaults:
5
+
6
+ - processing instructions are dropped, not refused, because every IANA registry file opens with an
7
+ `xml-stylesheet` and an `xml-model` instruction (nothing is ever fetched or executed for them);
8
+ - up to 600,000 elements, because the largest registries are that big.
9
+
10
+ Everything the core refuses is still refused: declarations, entities, notations, external references, encodings
11
+ other than UTF-8, and input over any bound. Names come out in ElementTree's `{namespace}local` form.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import xml.etree.ElementTree as ET # noqa: S405 - only the Element type; parsing is the core's hardened expat
17
+
18
+ from primarysource.readers.xml import XmlRefused
19
+ from primarysource.readers.xml import parse as _parse
20
+
21
+ __all__ = ["HINT", "XmlRefused", "parse"]
22
+
23
+ HINT = "the file is damaged or not a registry file; fetch it again"
24
+
25
+
26
+ def parse(
27
+ raw: bytes,
28
+ limit: int,
29
+ *,
30
+ max_depth: int = 32,
31
+ max_elements: int = 600_000,
32
+ max_attributes: int = 16,
33
+ max_attribute_length: int = 4096,
34
+ max_text: int = 1_000_000,
35
+ ) -> ET.Element:
36
+ """Parse `raw` into an Element, or raise `XmlRefused` (a `LibError`). The whole document is passed at once."""
37
+ return _parse(
38
+ raw,
39
+ limit,
40
+ max_depth=max_depth,
41
+ max_elements=max_elements,
42
+ max_attributes=max_attributes,
43
+ max_attribute_length=max_attribute_length,
44
+ max_text=max_text,
45
+ ignore_processing_instructions=True,
46
+ hint=HINT,
47
+ )
@@ -0,0 +1,186 @@
1
+ """What a supplied document is: which collection knows it, and as which designation (`primarysource identify`).
2
+
3
+ People keep vendor and standards PDFs under whatever name the download gave them, so an import cannot rely on the
4
+ file name. A `Probe` reads the file name, the PDF's title metadata and the text of its first pages once; each
5
+ collection that declares an `identify` hook (`registry.Collection.identify`, a "module:function" taking a `Probe`
6
+ and returning `Candidate`s) looks for its own designations in it. `score` is the common rule every hook uses,
7
+ so candidates from different collections compare:
8
+
9
+ - the designation on the first page (the cover): 3;
10
+ - on two or more of the later probed pages (a running header): 2;
11
+ - in the PDF's title metadata: 2;
12
+ - in the file name: 1.
13
+
14
+ A hook may add evidence of its own (ISO's `Reference number` line on the cover). Nothing here imports anything;
15
+ it says what the document is and the command that imports it.
16
+ """
17
+
18
+ from __future__ import annotations
19
+
20
+ import io
21
+ import re
22
+ from collections.abc import Callable, Iterable, Sequence
23
+ from dataclasses import dataclass, field
24
+ from pathlib import Path
25
+
26
+ from primarysource.errors import LibError
27
+
28
+ PROBE_PAGES = 3
29
+ MAX_PROBE_BYTES = 256 * 1024 * 1024
30
+
31
+
32
+ @dataclass(frozen=True, slots=True)
33
+ class Probe:
34
+ """What identification reads: the file's name and path, its title metadata, and its first pages' text."""
35
+
36
+ name: str
37
+ title: str = ""
38
+ pages: tuple[str, ...] = ()
39
+ path: Path | None = None
40
+
41
+
42
+ @dataclass(frozen=True, slots=True)
43
+ class Candidate:
44
+ """One answer: `collection` knows this document as `designation`. Higher `score` is surer."""
45
+
46
+ collection: str
47
+ designation: str
48
+ score: int
49
+ evidence: tuple[str, ...] = field(default=())
50
+ command: str = ""
51
+
52
+
53
+ def probe_pdf(path: Path, pages: int = PROBE_PAGES) -> Probe:
54
+ """Read a PDF's title metadata and its first `pages` pages of text. Needs `pypdf` (any PDF extra)."""
55
+ from primarysource.readers.pdf import _pypdf
56
+
57
+ pypdf = _pypdf()
58
+ try:
59
+ if path.stat().st_size > MAX_PROBE_BYTES:
60
+ raise LibError(f"{path.name} is over {MAX_PROBE_BYTES} bytes", "")
61
+ reader = pypdf.PdfReader(io.BytesIO(path.read_bytes()), strict=False)
62
+ meta = reader.metadata
63
+ title = str(meta.title) if meta is not None and meta.title else ""
64
+ texts = tuple(str(reader.pages[i].extract_text() or "") for i in range(min(pages, len(reader.pages))))
65
+ except LibError:
66
+ raise
67
+ except Exception as exc: # noqa: BLE001 - pypdf raises many types on hostile input
68
+ raise LibError(f"could not read {path.name}: {type(exc).__name__}: {exc}", "is it a PDF?") from exc
69
+ return Probe(path.name, title, texts, path)
70
+
71
+
72
+ def score[K](probe: Probe, pattern: re.Pattern[str], key: Callable[[re.Match[str]], K | None]) -> dict[K, list[str]]:
73
+ """Every designation `pattern` finds in the probe, keyed by `key(match)` (None skips a match), with the
74
+ evidence for it; `points` turns the evidence into a score."""
75
+ found: dict[K, list[str]] = {}
76
+
77
+ def note(text: str, label: str) -> set[K]:
78
+ keys: set[K] = set()
79
+ for match in pattern.finditer(text):
80
+ value = key(match)
81
+ if value is not None:
82
+ keys.add(value)
83
+ for value in keys:
84
+ found.setdefault(value, []).append(label)
85
+ return keys
86
+
87
+ if probe.pages:
88
+ note(probe.pages[0], "cover")
89
+ later: dict[K, int] = {}
90
+ for text in probe.pages[1:]:
91
+ for value in note(text, "page"):
92
+ later[value] = later.get(value, 0) + 1
93
+ for value, count in later.items():
94
+ evidence = [e for e in found[value] if e != "page"]
95
+ found[value] = [*evidence, *(["running header"] if count >= 2 else ["one later page"])]
96
+ note(probe.title, "title metadata")
97
+ note(probe.name, "file name")
98
+ return found
99
+
100
+
101
+ POINTS = {"cover": 3, "running header": 2, "title metadata": 2, "file name": 1, "one later page": 0}
102
+
103
+
104
+ def points(evidence: Iterable[str]) -> int:
105
+ """The score of a designation's evidence under the common rule (unknown evidence counts as the hook says)."""
106
+ return sum(POINTS.get(e, 0) for e in set(evidence))
107
+
108
+
109
+ def identify(probe: Probe, hooks: Sequence[tuple[str, Callable[[Probe], list[Candidate]]]]) -> list[Candidate]:
110
+ """Every collection's candidates, surest first; a hook that fails is skipped, never fatal."""
111
+ out: list[Candidate] = []
112
+ for _name, hook in hooks:
113
+ try:
114
+ out.extend(hook(probe))
115
+ except LibError:
116
+ continue
117
+ return sorted((c for c in out if c.score > 0), key=lambda c: (-c.score, c.collection, c.designation))
118
+
119
+
120
+ def registered_hooks() -> list[tuple[str, Callable[[Probe], list[Candidate]]]]:
121
+ """The `identify` hooks of every registered collection that has one and whose code loads."""
122
+ from primarysource import registry
123
+
124
+ hooks: list[tuple[str, Callable[[Probe], list[Candidate]]]] = []
125
+ for name in registry.available():
126
+ try:
127
+ collection = registry.load(name)
128
+ if collection.identify:
129
+ hooks.append((name, registry.attribute(collection, collection.identify, "identify hook")))
130
+ except LibError:
131
+ continue
132
+ return hooks
133
+
134
+
135
+ def identify_path(path: Path) -> list[Candidate]:
136
+ """Probe one PDF and ask every registered collection what it is."""
137
+ return identify(probe_pdf(path), registered_hooks())
138
+
139
+
140
+ def describe(path: Path, found: Sequence[Candidate]) -> str:
141
+ """One report block for `primarysource identify`."""
142
+ if not found:
143
+ return f"{path.name}: not recognised as any collection's document"
144
+ best = found[0]
145
+ lines = [f"{path.name}: {best.designation} ({best.collection}; {', '.join(best.evidence)})"]
146
+ if best.command:
147
+ lines.append(f" import: {best.command}")
148
+ lines += [f" also: {c.designation} ({c.collection}, score {c.score})" for c in found[1:4]]
149
+ return "\n".join(lines)
150
+
151
+
152
+ _PDF = re.compile(r"\.pdf$", re.IGNORECASE)
153
+
154
+
155
+ def pdfs(paths: Sequence[Path]) -> list[Path]:
156
+ """The PDFs named, with directories expanded to the PDFs directly inside them."""
157
+ out: list[Path] = []
158
+ for path in paths:
159
+ if path.is_dir():
160
+ out.extend(sorted(p for p in path.iterdir() if p.is_file() and _PDF.search(p.name)))
161
+ else:
162
+ out.append(path)
163
+ return out
164
+
165
+
166
+ def candidates(
167
+ probe: Probe,
168
+ collection: str,
169
+ pattern: re.Pattern[str],
170
+ designation: Callable[[re.Match[str]], str | None],
171
+ command: Callable[[str], str],
172
+ *,
173
+ file_pattern: re.Pattern[str] | None = None,
174
+ ) -> list[Candidate]:
175
+ """A hook in one call: every designation `pattern` finds (as `designation(match)` spells it), scored by the
176
+ common rule, each with the command that imports it. `file_pattern` also reads the file name (a publisher's
177
+ download name such as `T-REC-X.690-202102`), worth what a file-name match is worth."""
178
+ found = score(probe, pattern, designation)
179
+ if file_pattern is not None:
180
+ for value in {designation(m) for m in file_pattern.finditer(probe.name)} - {None}:
181
+ if value is not None and "file name" not in found.setdefault(value, []):
182
+ found[value].append("file name")
183
+ return [
184
+ Candidate(collection, value, points(evidence), tuple(dict.fromkeys(evidence)), command(value))
185
+ for value, evidence in found.items()
186
+ ]
@@ -0,0 +1,58 @@
1
+ """IEEE standards: two collections in one subpackage, both installed by `pip install "primarysource[ieee]"`.
2
+
3
+ - `posix`: IEEE Std 1003.1-2024, which is The Open Group Base Specifications Issue 8 (one text, also published
4
+ as ISO/IEC/IEEE 9945). The free HTML edition, downloaded by the user and imported from its archive.
5
+ - `ieee802`: IEEE 802 standards a user obtained from the IEEE GET Program, as PDFs.
6
+
7
+ They are two collections, not one, because licence, placement and pin are per collection and the Open Group's
8
+ terms differ from IEEE's GET terms (PLAN.md §5.8). `docs/ieee.md` is the user-facing description.
9
+
10
+ Importing this package needs nothing outside the standard library: `pypdf` is imported only by the 802 PDF
11
+ import, so `posix` works without it, and `mcp` only by `server.py` when a collection is served
12
+ (`primarysource mcp --collection posix`). `primarysource posix ...` runs `cli.py`.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ from primarysource.model import Unit
18
+ from primarysource.refs import RefGrammar
19
+ from primarysource.registry import Collection
20
+
21
+ POSIX_DOC = r"posix-[0-9]{4}"
22
+ """An edition of POSIX by year: `posix-2024`."""
23
+
24
+ POSIX_PAGE = r"[a-z]+(?:/[A-Za-z0-9_\[][A-Za-z0-9_.+\-]*)?"
25
+ """A page of the HTML edition by its path without `.html`: `functions/open`, `utilities/pax`, `basedefs/stdio.h`,
26
+ `basedefs/V1_chap03`, `xrat/V4_xcu_chap01`. `utilities/[` is the one page whose name starts with `[`."""
27
+
28
+ IEEE802_DOC = r"802(?:\.[0-9]+[A-Za-z]*)*-[0-9]{4}"
29
+ """An 802 standard by number and year: `802.1Q-2022`, `802.3-2022`, `802.1AB-2016`, `802.15.4-2020`."""
30
+
31
+ POSIX = Collection(
32
+ name="posix",
33
+ title="IEEE Std 1003.1-2024 / The Open Group Base Specifications Issue 8",
34
+ prefix="posix_",
35
+ unit=Unit.BLOCK,
36
+ grammar=RefGrammar(POSIX_DOC, POSIX_PAGE),
37
+ server="primarysource.ieee.server:posix_server",
38
+ cli="primarysource.ieee.cli:posix_main",
39
+ )
40
+ """Refs: `posix-2024§functions/open#tag_17_372_03¶2`. `§` names the page, `#` an HTML anchor on it. `¶n` counts
41
+ converted blocks (a paragraph, a list item, a code block, a table, a definition term): with an anchor, the blocks
42
+ under that heading, the heading itself excluded and its subsections included; with only a page, every block of
43
+ the page, headings included."""
44
+
45
+ IEEE802 = Collection(
46
+ name="ieee802",
47
+ title="IEEE 802 standards (IEEE GET Program)",
48
+ prefix="ieee802_",
49
+ unit=Unit.SENTENCE,
50
+ grammar=RefGrammar(IEEE802_DOC),
51
+ server="primarysource.ieee.server:ieee802_server",
52
+ cli="primarysource.ieee.cli:ieee802_main",
53
+ identify="primarysource.ieee.identify:identify",
54
+ )
55
+ """Refs: `802.1Q-2022§5.4¶2`. `§` names a numbered clause or annex (`5.4`, `A.1`); `¶n` is the nth sentence of the
56
+ clause's extracted text, because a PDF gives sentences, not paragraphs."""
57
+
58
+ __all__ = ["IEEE802", "POSIX"]
@@ -0,0 +1,49 @@
1
+ """The ieee collections' HTML reader: the core's `primarysource.readers.html`, plus the `POSIX` profile.
2
+
3
+ The converter itself moved to the core (`cl.reader_html`); this module re-exports it so `posix.py` and anything
4
+ else that imported `primarysource.ieee._html` keeps working unchanged. What stays here is ieee's own rule set.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from primarysource.readers.html import (
10
+ GENERIC,
11
+ MAX_INDENT,
12
+ MAX_OUTPUT_CHARS,
13
+ Document,
14
+ Heading,
15
+ Link,
16
+ Profile,
17
+ convert,
18
+ fence_language,
19
+ is_chrome,
20
+ join_blocks,
21
+ resolve_href,
22
+ to_plain,
23
+ )
24
+
25
+ __all__ = [
26
+ "GENERIC",
27
+ "MAX_INDENT",
28
+ "MAX_OUTPUT_CHARS",
29
+ "POSIX",
30
+ "Document",
31
+ "Heading",
32
+ "Link",
33
+ "Profile",
34
+ "convert",
35
+ "fence_language",
36
+ "is_chrome",
37
+ "join_blocks",
38
+ "resolve_href",
39
+ "to_plain",
40
+ ]
41
+
42
+ POSIX = Profile(
43
+ chrome_classes=frozenset({"navheader", "topofpage"}),
44
+ inner_anchor=True,
45
+ small_print_centre=True,
46
+ plain_blockquote=True,
47
+ rules=False,
48
+ )
49
+ """The Open Group's POSIX pages: five rules on top of `GENERIC` (see `Profile` for what each one does)."""
@@ -0,0 +1,74 @@
1
+ {
2
+ "schema": 1,
3
+ "note": "Catalogue metadata and digests only. No document text belongs in this file or anywhere in the repository.",
4
+ "collections": {
5
+ "posix": {
6
+ "licence": "LicenseRef-OpenGroup-IEEE-personal-use",
7
+ "placement": "volume",
8
+ "terms_url": "https://pubs.opengroup.org/onlinepubs/terms.htm",
9
+ "terms_read": "2026-11-12",
10
+ "terms_summary": "Jointly owned by The Open Group and the IEEE. Reading and downloading for personal use only, after giving a name and email address per publication; no copying, amending, reprinting, sale or re-use without permission of both holders. The hard copy is authoritative.",
11
+ "editions": [
12
+ {
13
+ "id": "posix-2024",
14
+ "designations": [
15
+ "IEEE Std 1003.1-2024",
16
+ "The Open Group Base Specifications Issue 8",
17
+ "ISO/IEC/IEEE 9945:2026"
18
+ ],
19
+ "designation_note": "One text under three covers. IEEE's page for 1003.1-2024 (standards.ieee.org, read 2026-11-13) says it is simultaneously IEEE Std 1003.1-2024 and The Open Group Base Specifications Issue 8, and lists the ISO/IEC/IEEE 9945 International Standard with the same Issue 8 title. ISO/IEC/IEEE 9945:2026 (edition 2, 2026-03, replacing the 2009 first edition) is known only from search-result snippets: iso.org answered HTTP 403 to an automated fetch, so its date and its identity with this HTML are not verified from the primary page. It is sold; this collection neither holds nor imports it.",
20
+ "title": "IEEE/Open Group Standard for Information Technology--Portable Operating System Interface (POSIX) Base Specifications, Issue 8",
21
+ "publisher": "IEEE and The Open Group",
22
+ "edition_date": "2024-06-14",
23
+ "ieee_page_url": "https://standards.ieee.org/ieee/1003.1/7700/",
24
+ "ieee_status": "Active Standard (standards.ieee.org, read 2026-11-13)",
25
+ "archive_updated": "2025-04-01",
26
+ "page_url": "https://pubs.opengroup.org/onlinepubs/9799919799/",
27
+ "source_url": "https://pubs.opengroup.org/onlinepubs/9799919799/download/susv5.tgz",
28
+ "archive": "susv5.tgz",
29
+ "sha256": "11c7ee3e98febe3d58d03a02885d0fab9a3564d06a9b02742d9d138ae341a957",
30
+ "size": 4962222,
31
+ "root": "susv5-html/",
32
+ "members": 1715,
33
+ "html_files": 1616,
34
+ "html_bytes": 24248288
35
+ }
36
+ ]
37
+ },
38
+ "ieee802": {
39
+ "licence": "LicenseRef-IEEE-GET-Program",
40
+ "placement": "volume",
41
+ "terms_url": "https://ieeexplore.ieee.org/Xplorehelp/overview-of-ieee-xplore/get-program-terms-of-use",
42
+ "terms_read": "not read: the terms page answered HTTP 418 to an automated fetch on 2026-11-12",
43
+ "terms_summary": "IEEE GET Program (standards.ieee.org, read 2026-11-12): the Terms and Conditions of Use must be accepted before the PDF is available; the documents are not in the public domain and IEEE remains the sole copyright holder. Copy and automation limits are unverified.",
44
+ "editions": [
45
+ {
46
+ "id": "802.1Q-2022",
47
+ "designations": ["IEEE Std 802.1Q-2022"],
48
+ "title": "IEEE Standard for Local and Metropolitan Area Networks--Bridges and Bridged Networks",
49
+ "publisher": "IEEE",
50
+ "edition_date": "2022-12-22",
51
+ "page_url": "https://standards.ieee.org/ieee/802.1Q/10323/",
52
+ "source_url": null,
53
+ "archive": null,
54
+ "sha256": null,
55
+ "size": null,
56
+ "get_program": "the standard's page says the PDF is available at no cost through the GET program (read 2026-11-12)"
57
+ },
58
+ {
59
+ "id": "802.3-2022",
60
+ "designations": ["IEEE Std 802.3-2022"],
61
+ "title": "IEEE Standard for Ethernet",
62
+ "publisher": "IEEE",
63
+ "edition_date": "2022-07-29",
64
+ "page_url": "https://standards.ieee.org/ieee/802.3/10422/",
65
+ "source_url": null,
66
+ "archive": null,
67
+ "sha256": null,
68
+ "size": null,
69
+ "get_program": "the standard's page links the GET program; whether this edition is free there was not stated (read 2026-11-12)"
70
+ }
71
+ ]
72
+ }
73
+ }
74
+ }