primarysource 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (303) hide show
  1. primarysource/__init__.py +5 -0
  2. primarysource/bbf/__init__.py +56 -0
  3. primarysource/bbf/_xml.py +63 -0
  4. primarysource/bbf/catalogue.json +777 -0
  5. primarysource/bbf/catalogue.py +137 -0
  6. primarysource/bbf/cli.py +79 -0
  7. primarysource/bbf/datamodel.py +281 -0
  8. primarysource/bbf/importer.py +151 -0
  9. primarysource/bbf/library.py +380 -0
  10. primarysource/bbf/server.py +72 -0
  11. primarysource/budget.py +63 -0
  12. primarysource/c2sp/__init__.py +53 -0
  13. primarysource/c2sp/_fetch.py +5 -0
  14. primarysource/c2sp/catalogue.json +231 -0
  15. primarysource/c2sp/catalogue.py +95 -0
  16. primarysource/c2sp/cli.py +62 -0
  17. primarysource/c2sp/importer.py +90 -0
  18. primarysource/c2sp/library.py +309 -0
  19. primarysource/c2sp/markdown.py +31 -0
  20. primarysource/c2sp/server.py +68 -0
  21. primarysource/cavp/__init__.py +42 -0
  22. primarysource/cavp/cli.py +84 -0
  23. primarysource/cavp/data/vectors.toml +47 -0
  24. primarysource/cavp/importer.py +156 -0
  25. primarysource/cavp/model.py +279 -0
  26. primarysource/cavp/parse.py +269 -0
  27. primarysource/cavp/paths.py +17 -0
  28. primarysource/cavp/server.py +67 -0
  29. primarysource/cavp/tools.py +169 -0
  30. primarysource/cli.py +107 -0
  31. primarysource/cursor.py +82 -0
  32. primarysource/cve/__init__.py +5 -0
  33. primarysource/cve/__main__.py +6 -0
  34. primarysource/cve/cli.py +144 -0
  35. primarysource/cve/collection.py +20 -0
  36. primarysource/cve/config.py +58 -0
  37. primarysource/cve/errors.py +16 -0
  38. primarysource/cve/kernel.py +232 -0
  39. primarysource/cve/mcp.py +139 -0
  40. primarysource/cve/parse.py +318 -0
  41. primarysource/cve/py.typed +0 -0
  42. primarysource/cve/service.py +760 -0
  43. primarysource/cve/store.py +214 -0
  44. primarysource/cve/sync.py +438 -0
  45. primarysource/cve/views.py +304 -0
  46. primarysource/debpolicy/__init__.py +16 -0
  47. primarysource/debpolicy/catalogue.py +263 -0
  48. primarysource/debpolicy/cli.py +12 -0
  49. primarysource/debpolicy/server.py +11 -0
  50. primarysource/draft/__init__.py +11 -0
  51. primarysource/draft/catalogue.py +117 -0
  52. primarysource/draft/cli.py +149 -0
  53. primarysource/draft/collection.py +21 -0
  54. primarysource/draft/datatracker.py +158 -0
  55. primarysource/draft/importer.py +163 -0
  56. primarysource/draft/licence.py +89 -0
  57. primarysource/draft/names.py +64 -0
  58. primarysource/draft/net.py +133 -0
  59. primarysource/draft/segment.py +192 -0
  60. primarysource/draft/server.py +91 -0
  61. primarysource/draft/status.py +146 -0
  62. primarysource/draft/store.py +199 -0
  63. primarysource/draft/tracked.json +104 -0
  64. primarysource/draft/views.py +430 -0
  65. primarysource/draft/xmlmeta.py +74 -0
  66. primarysource/ecma/__init__.py +52 -0
  67. primarysource/ecma/_pdf.py +143 -0
  68. primarysource/ecma/catalogue.json +22 -0
  69. primarysource/ecma/catalogue.py +100 -0
  70. primarysource/ecma/cli.py +67 -0
  71. primarysource/ecma/identify.py +25 -0
  72. primarysource/ecma/importer.py +108 -0
  73. primarysource/ecma/library.py +240 -0
  74. primarysource/ecma/server.py +67 -0
  75. primarysource/errors.py +29 -0
  76. primarysource/fetch.py +360 -0
  77. primarysource/fsutil.py +150 -0
  78. primarysource/gitdocs/__init__.py +15 -0
  79. primarysource/gitdocs/catalogue.py +116 -0
  80. primarysource/gitdocs/cli.py +12 -0
  81. primarysource/gitdocs/server.py +8 -0
  82. primarysource/home.py +65 -0
  83. primarysource/iana/__init__.py +32 -0
  84. primarysource/iana/catalogue.json +244 -0
  85. primarysource/iana/catalogue.py +69 -0
  86. primarysource/iana/cli.py +69 -0
  87. primarysource/iana/importer.py +157 -0
  88. primarysource/iana/model.py +325 -0
  89. primarysource/iana/reader.py +315 -0
  90. primarysource/iana/server.py +67 -0
  91. primarysource/iana/store.py +83 -0
  92. primarysource/iana/xmlsafe.py +47 -0
  93. primarysource/identify.py +186 -0
  94. primarysource/ieee/__init__.py +58 -0
  95. primarysource/ieee/_html.py +49 -0
  96. primarysource/ieee/catalogue.json +74 -0
  97. primarysource/ieee/catalogue.py +123 -0
  98. primarysource/ieee/cli.py +79 -0
  99. primarysource/ieee/identify.py +24 -0
  100. primarysource/ieee/ieee802.py +47 -0
  101. primarysource/ieee/importer.py +305 -0
  102. primarysource/ieee/posix.py +233 -0
  103. primarysource/ieee/server.py +79 -0
  104. primarysource/index.py +446 -0
  105. primarysource/iso/__init__.py +51 -0
  106. primarysource/iso/cli.py +98 -0
  107. primarysource/iso/designation.py +95 -0
  108. primarysource/iso/identify.py +100 -0
  109. primarysource/iso/importer.py +224 -0
  110. primarysource/iso/library.py +249 -0
  111. primarysource/iso/server.py +69 -0
  112. primarysource/itu/__init__.py +41 -0
  113. primarysource/itu/__main__.py +8 -0
  114. primarysource/itu/asn1mod.py +478 -0
  115. primarysource/itu/catalog.py +472 -0
  116. primarysource/itu/cli.py +445 -0
  117. primarysource/itu/collection.py +20 -0
  118. primarysource/itu/corpus.py +118 -0
  119. primarysource/itu/errata.py +16 -0
  120. primarysource/itu/formal.py +565 -0
  121. primarysource/itu/identify.py +26 -0
  122. primarysource/itu/index.py +490 -0
  123. primarysource/itu/mcp.py +474 -0
  124. primarysource/itu/py.typed +0 -0
  125. primarysource/itu/sources.py +600 -0
  126. primarysource/itu/text.py +609 -0
  127. primarysource/kconfig/__init__.py +16 -0
  128. primarysource/kconfig/catalogue.py +98 -0
  129. primarysource/kconfig/cli.py +12 -0
  130. primarysource/kconfig/server.py +8 -0
  131. primarysource/model.py +63 -0
  132. primarysource/nist/__init__.py +5 -0
  133. primarysource/nist/collection.py +46 -0
  134. primarysource/nist/data/acquired-nist.json +1657 -0
  135. primarysource/nist/data/nist-catalog.json +12090 -0
  136. primarysource/nist/data/sources.toml +1996 -0
  137. primarysource/nist/identify.py +41 -0
  138. primarysource/nist/ingest/__init__.py +1 -0
  139. primarysource/nist/ingest/acquire_nist.py +128 -0
  140. primarysource/nist/ingest/controls.py +82 -0
  141. primarysource/nist/ingest/discover_nist.py +218 -0
  142. primarysource/nist/ingest/extract.py +48 -0
  143. primarysource/nist/ingest/fetch.py +211 -0
  144. primarysource/nist/ingest/index.py +14 -0
  145. primarysource/nist/ingest/oscal.py +480 -0
  146. primarysource/nist/ingest/prose_layouts.py +236 -0
  147. primarysource/nist/ingest/sources.py +94 -0
  148. primarysource/nist/ingest/status.py +255 -0
  149. primarysource/nist/models.py +7 -0
  150. primarysource/nist/paths.py +33 -0
  151. primarysource/nist/server/__init__.py +3 -0
  152. primarysource/nist/server/corpus.py +38 -0
  153. primarysource/nist/server/mcp.py +19 -0
  154. primarysource/nist/server/tools.py +35 -0
  155. primarysource/nist/spec.py +38 -0
  156. primarysource/pdfcorpus/__init__.py +16 -0
  157. primarysource/pdfcorpus/corpus.py +159 -0
  158. primarysource/pdfcorpus/fts.py +175 -0
  159. primarysource/pdfcorpus/index.py +70 -0
  160. primarysource/pdfcorpus/models.py +30 -0
  161. primarysource/pdfcorpus/server.py +96 -0
  162. primarysource/pdfcorpus/spec.py +37 -0
  163. primarysource/pdfcorpus/split.py +391 -0
  164. primarysource/pdfcorpus/tools.py +244 -0
  165. primarysource/pep/__init__.py +15 -0
  166. primarysource/pep/catalogue.py +191 -0
  167. primarysource/pep/cli.py +12 -0
  168. primarysource/pep/server.py +8 -0
  169. primarysource/pkgdocs/__init__.py +19 -0
  170. primarysource/pkgdocs/fetch.py +356 -0
  171. primarysource/pkgdocs/index.py +89 -0
  172. primarysource/pkgdocs/model.py +76 -0
  173. primarysource/pkgdocs/readers/__init__.py +1 -0
  174. primarysource/pkgdocs/readers/_common.py +134 -0
  175. primarysource/pkgdocs/readers/asciidoc.py +160 -0
  176. primarysource/pkgdocs/readers/latex.py +315 -0
  177. primarysource/pkgdocs/readers/markdown.py +173 -0
  178. primarysource/pkgdocs/readers/pod.py +132 -0
  179. primarysource/pkgdocs/readers/rst.py +239 -0
  180. primarysource/pkgdocs/readers/scdoc.py +108 -0
  181. primarysource/pkgdocs/runtime.py +552 -0
  182. primarysource/pkgdocs/serving.py +158 -0
  183. primarysource/pkgdocs/store.py +167 -0
  184. primarysource/pms/__init__.py +16 -0
  185. primarysource/pms/catalogue.py +88 -0
  186. primarysource/pms/cli.py +12 -0
  187. primarysource/pms/server.py +8 -0
  188. primarysource/py.typed +0 -0
  189. primarysource/readers/__init__.py +10 -0
  190. primarysource/readers/bookmarks.py +192 -0
  191. primarysource/readers/html.py +1312 -0
  192. primarysource/readers/markdown.py +189 -0
  193. primarysource/readers/pdf.py +187 -0
  194. primarysource/readers/xml.py +230 -0
  195. primarysource/refs.py +87 -0
  196. primarysource/registry.py +161 -0
  197. primarysource/render.py +49 -0
  198. primarysource/rfc/__init__.py +3 -0
  199. primarysource/rfc/__main__.py +6 -0
  200. primarysource/rfc/cite.py +204 -0
  201. primarysource/rfc/cli.py +903 -0
  202. primarysource/rfc/collection.py +19 -0
  203. primarysource/rfc/config.py +138 -0
  204. primarysource/rfc/errata.py +203 -0
  205. primarysource/rfc/fsutil.py +82 -0
  206. primarysource/rfc/graph.py +213 -0
  207. primarysource/rfc/index/__init__.py +1 -0
  208. primarysource/rfc/index/chunker.py +281 -0
  209. primarysource/rfc/index/embed.py +166 -0
  210. primarysource/rfc/index/hybrid.py +305 -0
  211. primarysource/rfc/index/lexical.py +223 -0
  212. primarysource/rfc/index/rfcxml.py +673 -0
  213. primarysource/rfc/index/search.py +508 -0
  214. primarysource/rfc/index/txtseg.py +585 -0
  215. primarysource/rfc/library.py +294 -0
  216. primarysource/rfc/mcp/__init__.py +1 -0
  217. primarysource/rfc/mcp/server.py +381 -0
  218. primarysource/rfc/packs/__init__.py +1 -0
  219. primarysource/rfc/packs/builder.py +228 -0
  220. primarysource/rfc/packs/collections/all-current-txt.json +10 -0
  221. primarysource/rfc/packs/collections/essentials.json +19 -0
  222. primarysource/rfc/packs/collections/fyi-current.json +10 -0
  223. primarysource/rfc/packs/collections/standards-current.json +10 -0
  224. primarysource/rfc/packs/collections/year-2025.json +10 -0
  225. primarysource/rfc/packs/collections/year-2026.json +10 -0
  226. primarysource/rfc/packs/collections.py +163 -0
  227. primarysource/rfc/packs/docsetexport.py +163 -0
  228. primarysource/rfc/packs/epub.py +159 -0
  229. primarysource/rfc/packs/export.py +306 -0
  230. primarysource/rfc/packs/htmlexport.py +82 -0
  231. primarysource/rfc/packs/serve.py +143 -0
  232. primarysource/rfc/packs/zimexport.py +171 -0
  233. primarysource/rfc/query.py +466 -0
  234. primarysource/rfc/subjects.py +457 -0
  235. primarysource/rfc/sync/__init__.py +1 -0
  236. primarysource/rfc/sync/downloader.py +132 -0
  237. primarysource/rfc/sync/manifest.py +162 -0
  238. primarysource/rfc/sync/scanner.py +49 -0
  239. primarysource/rfc/sync/state.py +464 -0
  240. primarysource/rfc/views.py +1067 -0
  241. primarysource/rpmdocs/__init__.py +15 -0
  242. primarysource/rpmdocs/catalogue.py +117 -0
  243. primarysource/rpmdocs/cli.py +12 -0
  244. primarysource/rpmdocs/server.py +8 -0
  245. primarysource/scaffold.py +223 -0
  246. primarysource/serve.py +75 -0
  247. primarysource/settings.py +152 -0
  248. primarysource/usb/__init__.py +5 -0
  249. primarysource/usb/collection.py +37 -0
  250. primarysource/usb/data/acquired.json +2025 -0
  251. primarysource/usb/data/sources-usb.toml +852 -0
  252. primarysource/usb/data/sources.toml +10 -0
  253. primarysource/usb/data/usb-catalog.json +1663 -0
  254. primarysource/usb/data/usb-classification.json +1986 -0
  255. primarysource/usb/data/usb-corpus-build.json +608 -0
  256. primarysource/usb/data/usb-pdf-choice.json +839 -0
  257. primarysource/usb/data/usb-split-measure.json +1803 -0
  258. primarysource/usb/ingest/__init__.py +1 -0
  259. primarysource/usb/ingest/acquire.py +130 -0
  260. primarysource/usb/ingest/discover_usb.py +88 -0
  261. primarysource/usb/ingest/extract.py +48 -0
  262. primarysource/usb/ingest/fetch.py +197 -0
  263. primarysource/usb/ingest/index.py +14 -0
  264. primarysource/usb/ingest/sources.py +85 -0
  265. primarysource/usb/models.py +7 -0
  266. primarysource/usb/paths.py +33 -0
  267. primarysource/usb/server/__init__.py +3 -0
  268. primarysource/usb/server/corpus.py +38 -0
  269. primarysource/usb/server/mcp.py +19 -0
  270. primarysource/usb/server/tools.py +35 -0
  271. primarysource/usb/spec.py +20 -0
  272. primarysource/usbids/__init__.py +56 -0
  273. primarysource/usbids/cli.py +65 -0
  274. primarysource/usbids/importer.py +272 -0
  275. primarysource/usbids/parse.py +283 -0
  276. primarysource/usbids/server.py +55 -0
  277. primarysource/usbids/store.py +80 -0
  278. primarysource/usbids/tools.py +174 -0
  279. primarysource/zim/__init__.py +5 -0
  280. primarysource/zim/__main__.py +8 -0
  281. primarysource/zim/_libzim.py +37 -0
  282. primarysource/zim/catalog.py +594 -0
  283. primarysource/zim/cli.py +555 -0
  284. primarysource/zim/collection.py +41 -0
  285. primarysource/zim/errors.py +100 -0
  286. primarysource/zim/groups.py +30 -0
  287. primarysource/zim/library.py +668 -0
  288. primarysource/zim/manage.py +636 -0
  289. primarysource/zim/markdown.py +1247 -0
  290. primarysource/zim/mcp.py +201 -0
  291. primarysource/zim/purl.py +130 -0
  292. primarysource/zim/py.typed +0 -0
  293. primarysource/zim/refs.py +90 -0
  294. primarysource/zim/service.py +1182 -0
  295. primarysource/zim/snapshot.py +197 -0
  296. primarysource/zim/sync.py +269 -0
  297. primarysource/zim/views.py +230 -0
  298. primarysource-0.1.0.dist-info/METADATA +118 -0
  299. primarysource-0.1.0.dist-info/RECORD +303 -0
  300. primarysource-0.1.0.dist-info/WHEEL +4 -0
  301. primarysource-0.1.0.dist-info/entry_points.txt +2 -0
  302. primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
  303. primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
@@ -0,0 +1,565 @@
1
+ """ITU-T formal descriptions: a catalogue, a bounded importer, and modules served as documents.
2
+
3
+ ITU publishes the machine-processable parts of its Recommendations (ASN.1 modules, MIBs, XSD, and others) as
4
+ files under `https://www.itu.int/ITU-T/formal-language/`. The web server lists those directories, so the set can
5
+ be enumerated without the Recommendation database. What it does not do is serve every listed file: the `.asn`
6
+ source is listed but answers 404, and the `.html` rendering beside it is what can be fetched (measured
7
+ 2026-11-13). The rendering is the source text inside a `<pre>`, one comment line linking to the Recommendation,
8
+ and nothing else, so `asn1mod.unwrap_html` recovers it exactly.
9
+
10
+ Terms: ITU's website terms permit downloading, copying and using content for personal, educational or
11
+ non-commercial purposes with the source acknowledged and notices kept, and forbid modification, reproduction,
12
+ distribution and derivative works without written permission. They say that publications "may be governed by
13
+ specific licensing terms", and no separate licence is stated for this database. So the policy is the one the
14
+ rest of this package keeps: **the user fetches, into `ITULIB_HOME`; nothing is vendored, and the only thing
15
+ committed is a digest and a size per artifact.** `tests/test_license.py` still applies to this tree.
16
+
17
+ Nothing here touches the network on its own. Every function that reads a URL takes a `Transport`, so the tests
18
+ run offline and a caller can supply a caching or rate-limiting one. `urllib_transport` is the real one.
19
+
20
+ ## Refs
21
+
22
+ <Rec>#<year>[-<month>]/<Module>[::<section>]
23
+
24
+ X.509#2019-10/AuthenticationFramework the module as a document
25
+ X.509#2019-10/AuthenticationFramework::Certificate one assignment
26
+ X.509#2019/AuthenticationFramework::IMPORTS a reserved section: IMPORTS, EXPORTS or HEADER
27
+
28
+ `<Rec>` is the Recommendation as ITU names it (`X.509`, `X.1080.0`; also `G.7712`, `RFC2578` style folders).
29
+ `<year>` is the folder the artifact sits in, which is the edition's year. The month is optional because the
30
+ folder does not carry it; it is read from the header comment each file opens with, and when a ref gives it
31
+ and it disagrees with the file, the ref does not resolve. An edition folder may carry a suffix (`2017-Cor1`),
32
+ written after the year: `X.1080.0#2017-Cor1/Module`. A bare `::` section matches an assignment name exactly;
33
+ IMPORTS, EXPORTS and HEADER cannot collide with one because they are reserved words in ASN.1.
34
+ """
35
+
36
+ from __future__ import annotations
37
+
38
+ import hashlib
39
+ import html
40
+ import json
41
+ import os
42
+ import re
43
+ import time
44
+ import urllib.error
45
+ import urllib.parse
46
+ import urllib.request
47
+ from collections.abc import Callable
48
+ from dataclasses import asdict, dataclass
49
+ from pathlib import Path
50
+
51
+ from . import __version__, asn1mod, corpus
52
+
53
+ BASE = "https://www.itu.int"
54
+ ROOT_PATH = "/ITU-T/formal-language/"
55
+ HOST = "www.itu.int"
56
+ USER_AGENT = f"itulib-formal/{__version__} (+https://github.com/b1tninja/itulib)"
57
+ MIN_INTERVAL = 1.0
58
+
59
+ MAX_ARTIFACT_OCTETS = 2 * 1024 * 1024
60
+ MAX_LISTING_OCTETS = 1024 * 1024
61
+ RESERVED_SECTIONS = ("IMPORTS", "EXPORTS", "HEADER")
62
+
63
+ # Languages by file extension as the listings show them. Only what has been seen is claimed.
64
+ LANGUAGES = {".asn": "ASN.1 (listed, not served)", ".html": "ASN.1 or other, rendered as HTML"}
65
+
66
+ _SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+() -]{0,127}")
67
+
68
+
69
+ class FormalError(RuntimeError):
70
+ """A refusal, with the reason. Never an empty result standing in for one."""
71
+
72
+
73
+ @dataclass(frozen=True)
74
+ class Fetched:
75
+ status: int
76
+ body: bytes
77
+ url: str
78
+
79
+
80
+ Transport = Callable[[str, int], Fetched]
81
+ """`(url, max_octets) -> Fetched`. Must not read more than max_octets + 1 and must report the final URL."""
82
+
83
+
84
+ def urllib_transport(url: str, max_octets: int) -> Fetched:
85
+ """The real transport: https to itu.int only, an identifying User-Agent, a bounded read."""
86
+ parts = urllib.parse.urlsplit(url)
87
+ if parts.scheme != "https" or parts.hostname != HOST:
88
+ raise FormalError(f"refusing {url}: only https://{HOST}/ is fetched")
89
+ request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) # noqa: S310
90
+ try:
91
+ with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310
92
+ final = response.geturl()
93
+ body = response.read(max_octets + 1)
94
+ return Fetched(response.status, body, final)
95
+ except urllib.error.HTTPError as error:
96
+ return Fetched(error.code, b"", url)
97
+
98
+
99
+ # --------------------------------------------------------------------------------------------------- pins
100
+
101
+
102
+ @dataclass(frozen=True)
103
+ class Pin:
104
+ sha256: str
105
+ octets: int
106
+ month: str = "" # "10/2019" as the file's header names it
107
+
108
+
109
+ # Digests of the `.html` renderings as served on 2026-11-13. ITU re-renders these files (the x509/2019
110
+ # folder carries text from a 2026 corrigendum), so a mismatch means the file changed, not that it was
111
+ # tampered with; the remedy is to re-read the change and re-pin, never to skip the check.
112
+ PINS: dict[str, Pin] = {
113
+ "itu-t/x/x509/2019/AuthenticationFramework.html": Pin(
114
+ "44a22fa7e191ca9072c479e87388131ac21e727eb1ddaf2c03fc620055f707d2", 21_148, "10/2019"
115
+ ),
116
+ "itu-t/x/x509/2019/CertificateExtensions.html": Pin(
117
+ "b4c921ae4be61043add9aa6b58873dffeebac84ed585e8909779d7a1c5ef97a7", 29_763, "10/2019"
118
+ ),
119
+ }
120
+
121
+
122
+ def artifact_url(path: str) -> str:
123
+ return f"{BASE}{ROOT_PATH}{urllib.parse.quote(path)}"
124
+
125
+
126
+ def _check_path(path: str) -> list[str]:
127
+ """Validated segments of a path relative to the formal-language root. Refuses anything odd."""
128
+ segments = path.strip("/").split("/")
129
+ if not path or len(segments) > 8:
130
+ raise FormalError(f"bad artifact path {path!r}")
131
+ for segment in segments:
132
+ if not _SEGMENT.fullmatch(segment) or ".." in segment:
133
+ raise FormalError(f"bad path segment {segment!r} in {path!r}")
134
+ return segments
135
+
136
+
137
+ # ----------------------------------------------------------------------------------------------- listing
138
+
139
+ _ROW = re.compile(
140
+ r"(?P<date>\d{1,2}/\d{1,2}/\d{4})\s+(?P<time>\d{1,2}:\d{2}\s*[AP]M)\s+"
141
+ r"(?P<size>&lt;dir&gt;|\d+)\s*<A\s+HREF=\"(?P<href>[^\"]{1,512})\">(?P<name>[^<]{1,256})</A>",
142
+ re.IGNORECASE,
143
+ )
144
+
145
+
146
+ @dataclass(frozen=True)
147
+ class Entry:
148
+ name: str
149
+ directory: bool
150
+ size: int
151
+ stamp: str # as the server prints it: "10/22/2026 7:44 PM"
152
+
153
+
154
+ def parse_listing(page: str, directory: str) -> tuple[list[Entry], int]:
155
+ """Entries of an IIS directory listing, and how many rows were dropped as unsafe.
156
+
157
+ `directory` is the path the listing was fetched from, e.g. `/ITU-T/formal-language/itu-t/x/`. A row whose
158
+ link leaves that directory, whose name is not a plain segment, or that repeats is dropped and counted.
159
+ """
160
+ directory = directory if directory.endswith("/") else directory + "/"
161
+ entries: list[Entry] = []
162
+ seen: set[str] = set()
163
+ dropped = 0
164
+ for match in _ROW.finditer(page):
165
+ name = html.unescape(match["name"]).strip()
166
+ href = html.unescape(match["href"])
167
+ is_dir = match["size"].lower() == "&lt;dir&gt;"
168
+ expected = directory + name + ("/" if is_dir else "")
169
+ if urllib.parse.unquote(href) != expected or not _SEGMENT.fullmatch(name) or ".." in name or name in seen:
170
+ dropped += 1
171
+ continue
172
+ seen.add(name)
173
+ entries.append(Entry(name, is_dir, 0 if is_dir else int(match["size"]), f"{match['date']} {match['time']}"))
174
+ return entries, dropped
175
+
176
+
177
+ @dataclass
178
+ class Budget:
179
+ """Bounds on one crawl: requests, bytes, depth and entries. Exhaustion is reported, never hidden."""
180
+
181
+ max_requests: int = 400
182
+ max_octets: int = 64 * 1024 * 1024
183
+ max_depth: int = 6
184
+ max_entries: int = 20_000
185
+ requests: int = 0
186
+ octets: int = 0
187
+ stopped: str = ""
188
+
189
+ def spend(self, octets: int) -> bool:
190
+ self.requests += 1
191
+ self.octets += octets
192
+ if self.requests >= self.max_requests:
193
+ self.stopped = f"request budget {self.max_requests} used"
194
+ elif self.octets >= self.max_octets:
195
+ self.stopped = f"byte budget {self.max_octets} used"
196
+ return not self.stopped
197
+
198
+
199
+ @dataclass(frozen=True)
200
+ class Artifact:
201
+ path: str # relative to the formal-language root: itu-t/x/x509/2019/AuthenticationFramework.html
202
+ size: int
203
+ stamp: str
204
+
205
+ @property
206
+ def folder(self) -> str:
207
+ return self.path.rsplit("/", 1)[0]
208
+
209
+ @property
210
+ def name(self) -> str:
211
+ return self.path.rsplit("/", 1)[1]
212
+
213
+
214
+ def enumerate_tree(
215
+ transport: Transport,
216
+ start: str = "itu-t/",
217
+ *,
218
+ budget: Budget | None = None,
219
+ pause: Callable[[float], None] = time.sleep,
220
+ ) -> tuple[list[Artifact], Budget, int]:
221
+ """Walk directory listings below `start`. Returns artifacts, the budget (check `.stopped`) and drops."""
222
+ budget = budget or Budget()
223
+ root = _check_path(start)
224
+ queue: list[tuple[str, int]] = [("/".join(root) + "/", 0)]
225
+ found: list[Artifact] = []
226
+ dropped = 0
227
+ while queue and not budget.stopped:
228
+ relative, depth = queue.pop(0)
229
+ if budget.requests:
230
+ pause(MIN_INTERVAL)
231
+ response = transport(artifact_url(relative), MAX_LISTING_OCTETS)
232
+ budget.spend(len(response.body))
233
+ if response.status != 200 or len(response.body) > MAX_LISTING_OCTETS:
234
+ dropped += 1
235
+ continue
236
+ if urllib.parse.urlsplit(response.url).hostname != HOST:
237
+ raise FormalError(f"listing redirected away from {HOST}: {response.url}")
238
+ entries, bad = parse_listing(response.body.decode("utf-8", errors="replace"), ROOT_PATH + relative)
239
+ dropped += bad
240
+ for entry in entries:
241
+ if entry.directory:
242
+ if depth + 1 <= budget.max_depth:
243
+ queue.append((relative + entry.name + "/", depth + 1))
244
+ elif len(found) < budget.max_entries:
245
+ found.append(Artifact(relative + entry.name, entry.size, entry.stamp))
246
+ return found, budget, dropped
247
+
248
+
249
+ # ----------------------------------------------------------------------------------------------- storage
250
+
251
+
252
+ def formal_home() -> Path:
253
+ return corpus.home() / "formal"
254
+
255
+
256
+ def artifact_file(path: str) -> Path:
257
+ return formal_home().joinpath("files", *_check_path(path))
258
+
259
+
260
+ def _manifest_file() -> Path:
261
+ return formal_home() / "manifest.json"
262
+
263
+
264
+ def load_manifest() -> dict[str, dict[str, object]]:
265
+ file = _manifest_file()
266
+ if not file.is_file():
267
+ return {}
268
+ loaded = json.loads(file.read_text(encoding="utf-8"))
269
+ return loaded if isinstance(loaded, dict) else {}
270
+
271
+
272
+ def _atomic_write(file: Path, data: bytes) -> None:
273
+ file.parent.mkdir(parents=True, exist_ok=True)
274
+ temporary = file.with_name(f"{file.name}.{os.getpid()}.part")
275
+ temporary.write_bytes(data)
276
+ os.replace(temporary, file)
277
+
278
+
279
+ def _save_manifest(manifest: dict[str, dict[str, object]]) -> None:
280
+ _atomic_write(_manifest_file(), json.dumps(manifest, indent=1, sort_keys=True).encode("utf-8"))
281
+
282
+
283
+ @dataclass(frozen=True)
284
+ class Imported:
285
+ path: str
286
+ sha256: str
287
+ octets: int
288
+ pinned: bool
289
+ modules: tuple[str, ...]
290
+
291
+
292
+ def import_artifact(
293
+ path: str,
294
+ transport: Transport,
295
+ *,
296
+ trust_first_use: bool = False,
297
+ pins: dict[str, Pin] | None = None,
298
+ ) -> Imported:
299
+ """Fetch one artifact into ITULIB_HOME, verified, parsed, and recorded.
300
+
301
+ A pinned artifact must hash to its pin. An unpinned one is refused unless the caller says
302
+ `trust_first_use`, in which case it is stored marked unpinned with its digest reported so it can be pinned.
303
+ Nothing is written until the bytes have been size-checked, hashed and parsed.
304
+ """
305
+ pins = PINS if pins is None else pins
306
+ _check_path(path)
307
+ if not path.endswith(".html"):
308
+ raise FormalError(f"{path}: only the .html rendering is served by ITU (the .asn listing entry is a 404)")
309
+ pin = pins.get(path)
310
+ if pin is None and not trust_first_use:
311
+ raise FormalError(f"{path} has no pinned digest. Re-run with trust_first_use to store it unpinned.")
312
+ response = transport(artifact_url(path), MAX_ARTIFACT_OCTETS)
313
+ if urllib.parse.urlsplit(response.url).hostname != HOST:
314
+ raise FormalError(f"{path}: redirected away from {HOST} to {response.url}")
315
+ if response.status != 200:
316
+ raise FormalError(f"{path}: HTTP {response.status}")
317
+ body = response.body
318
+ if len(body) > MAX_ARTIFACT_OCTETS:
319
+ raise FormalError(f"{path}: larger than the {MAX_ARTIFACT_OCTETS} octet bound")
320
+ digest = hashlib.sha256(body).hexdigest()
321
+ if pin is not None and (digest != pin.sha256 or len(body) != pin.octets):
322
+ raise FormalError(
323
+ f"{path}: digest {digest} ({len(body)} octets) does not match the pinned "
324
+ f"{pin.sha256} ({pin.octets}). ITU re-renders these files; read the change before re-pinning."
325
+ )
326
+ try:
327
+ text, note = asn1mod.unwrap_html(body)
328
+ modules = asn1mod.parse_modules(text)
329
+ except asn1mod.ModuleError as error:
330
+ raise FormalError(f"{path}: not readable as ASN.1: {error}") from error
331
+ _atomic_write(artifact_file(path), body)
332
+ manifest = load_manifest()
333
+ manifest[path] = {
334
+ "sha256": digest,
335
+ "octets": len(body),
336
+ "pinned": pin is not None,
337
+ "fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
338
+ "source": artifact_url(path),
339
+ "header_edition": note,
340
+ "modules": [module.name for module in modules],
341
+ "form": (
342
+ "Asnp-rendered HTML, markup stripped"
343
+ if b"Generated by Asnp" in body
344
+ else "source text in <pre>, wrapper stripped"
345
+ ),
346
+ }
347
+ _save_manifest(manifest)
348
+ return Imported(path, digest, len(body), pin is not None, tuple(m.name for m in modules))
349
+
350
+
351
+ def save_catalogue(artifacts: list[Artifact]) -> Path:
352
+ """Record an enumeration (paths, sizes, listing dates) beside the files. Metadata only."""
353
+ file = formal_home() / "catalogue.json"
354
+ rows = [asdict(a) for a in sorted(artifacts, key=lambda a: a.path)]
355
+ _atomic_write(
356
+ file,
357
+ json.dumps(
358
+ {"enumerated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "artifacts": rows}, indent=0
359
+ ).encode("utf-8"),
360
+ )
361
+ return file
362
+
363
+
364
+ # ------------------------------------------------------------------------------------------------- refs
365
+
366
+
367
+ @dataclass(frozen=True)
368
+ class FormalRef:
369
+ rec: str
370
+ year: str
371
+ month: str | None
372
+ suffix: str
373
+ module: str
374
+ section: str | None
375
+
376
+ @property
377
+ def edition(self) -> str:
378
+ return self.year + (f"-{self.month}" if self.month else "") + (f"-{self.suffix}" if self.suffix else "")
379
+
380
+
381
+ _REF = re.compile(
382
+ r"(?P<rec>[A-Za-z]{1,6}\.?[0-9][A-Za-z0-9.]{0,15})#(?P<year>\d{4})(?:-(?P<month>(?:0[1-9]|1[0-2])))?"
383
+ r"(?:-(?P<suffix>[A-Za-z][A-Za-z0-9]{0,15}))?/(?P<module>[A-Za-z][A-Za-z0-9-]{0,127})"
384
+ r"(?:::(?P<section>[A-Za-z][A-Za-z0-9-]{0,127}))?"
385
+ )
386
+
387
+
388
+ def parse_formal_ref(ref: str) -> FormalRef:
389
+ match = _REF.fullmatch(ref.strip())
390
+ if not match:
391
+ raise FormalError(f"{ref!r} is not a formal-description ref: expected Rec#year[-mm]/Module[::section]")
392
+ return FormalRef(
393
+ match["rec"], match["year"], match["month"], match["suffix"] or "", match["module"], match["section"]
394
+ )
395
+
396
+
397
+ def _folder_matches(folder: str, ref: FormalRef) -> bool:
398
+ parts = folder.split("/")
399
+ rec_folder = ref.rec.lower().replace(".", "")
400
+ edition_folder = ref.year + (f"-{ref.suffix}" if ref.suffix else "")
401
+ return len(parts) >= 2 and parts[-2].lower().replace(".", "") == rec_folder and parts[-1] == edition_folder
402
+
403
+
404
+ def _month_year(note: str | None) -> tuple[str, str] | None:
405
+ found = re.search(r"\((\d{2})/(\d{4})\)", note or "")
406
+ return (found.group(2), found.group(1)) if found else None
407
+
408
+
409
+ def resolve(ref: FormalRef, manifest: dict[str, dict[str, object]] | None = None) -> str:
410
+ """The artifact path a ref names among the imported ones, or a refusal that says why."""
411
+ manifest = load_manifest() if manifest is None else manifest
412
+ candidates = [
413
+ path
414
+ for path, info in manifest.items()
415
+ if path.endswith(".html")
416
+ and _folder_matches(path.rsplit("/", 1)[0], ref)
417
+ and ref.module in info.get("modules", []) # type: ignore[operator]
418
+ ]
419
+ if not candidates:
420
+ raise FormalError(
421
+ f"{ref.rec}#{ref.edition}/{ref.module} is not imported. Import it with "
422
+ "`python -m primarysource.itu.formal import <path>`; list what is imported with "
423
+ "`python -m primarysource.itu.formal list`."
424
+ )
425
+ if len(candidates) > 1:
426
+ raise FormalError(f"ambiguous: {sorted(candidates)}")
427
+ path = candidates[0]
428
+ stated = _month_year(str(manifest[path].get("header_edition")))
429
+ if ref.month and stated and (stated[0], stated[1]) != (ref.year, ref.month):
430
+ raise FormalError(
431
+ f"{path} states edition {stated[1]}/{stated[0]}, not {ref.month}/{ref.year}; "
432
+ f"the ref does not resolve rather than citing a different edition"
433
+ )
434
+ return path
435
+
436
+
437
+ # ------------------------------------------------------------------------------------------- documents
438
+
439
+
440
+ def read(ref_text: str, manifest: dict[str, dict[str, object]] | None = None) -> dict[str, object]:
441
+ """Serve a ref as a document: the module as a whole, or one section of it."""
442
+ ref = parse_formal_ref(ref_text)
443
+ manifest = load_manifest() if manifest is None else manifest
444
+ path = resolve(ref, manifest)
445
+ info = manifest[path]
446
+ text, _ = asn1mod.unwrap_html(artifact_file(path).read_bytes())
447
+ if hashlib.sha256(artifact_file(path).read_bytes()).hexdigest() != info["sha256"]:
448
+ raise FormalError(f"{path}: stored bytes no longer match the recorded digest; re-import")
449
+ modules = {m.name: m for m in asn1mod.parse_modules(text)}
450
+ module = modules.get(ref.module)
451
+ if module is None:
452
+ raise FormalError(f"{path} has no module {ref.module}")
453
+ stated = str(info.get("header_edition") or "") or None
454
+ cite = (
455
+ f"ITU-T {stated} formal description, module {module.name}"
456
+ if stated
457
+ else f"ITU-T {ref.rec} ({ref.year}) formal description, module {module.name}"
458
+ )
459
+ result: dict[str, object] = {
460
+ "ref": ref_text.strip(),
461
+ "module": module.name,
462
+ "oid": module.oid,
463
+ "oid_text": module.oid_text,
464
+ "source": info.get("source"),
465
+ "sha256": info["sha256"],
466
+ "pinned": info.get("pinned"),
467
+ "form": info.get("form"),
468
+ "cite": cite,
469
+ }
470
+ section = ref.section
471
+ if section is None:
472
+ result.update(
473
+ tagging=module.tagging,
474
+ extensibility_implied=module.extensibility_implied,
475
+ exports=module.exports
476
+ if module.exports is None or isinstance(module.exports, str)
477
+ else list(module.exports),
478
+ imports=[
479
+ {"module": i.module, "oid_text": i.oid_text, "symbols": list(i.symbols)} for i in module.imports
480
+ ],
481
+ sections=[{"name": a.name, "kind": a.kind, "line": a.line} for a in module.assignments],
482
+ )
483
+ return result
484
+ if section == "IMPORTS":
485
+ result["imports"] = [
486
+ {"module": i.module, "oid_text": i.oid_text, "symbols": list(i.symbols)} for i in module.imports
487
+ ]
488
+ return result
489
+ if section == "EXPORTS":
490
+ result["exports"] = (
491
+ module.exports if module.exports is None or isinstance(module.exports, str) else list(module.exports)
492
+ )
493
+ return result
494
+ if section == "HEADER":
495
+ result["text"] = text[: text.find("BEGIN") + 5] if "BEGIN" in text else ""
496
+ return result
497
+ found = module.assignment(section)
498
+ if found is None:
499
+ raise FormalError(f"{module.name} has no assignment {section}")
500
+ result.update(
501
+ name=found.name, kind=found.kind, parameterised=found.parameterised, line=found.line, text=found.text
502
+ )
503
+ return result
504
+
505
+
506
+ def listing() -> list[dict[str, object]]:
507
+ return [
508
+ {
509
+ "path": p,
510
+ **{k: v for k, v in i.items() if k in {"sha256", "octets", "pinned", "modules", "header_edition"}},
511
+ }
512
+ for p, i in sorted(load_manifest().items())
513
+ ]
514
+
515
+
516
+ def main(argv: list[str] | None = None) -> int:
517
+ import argparse
518
+ import sys
519
+
520
+ parser = argparse.ArgumentParser(prog="python -m primarysource.itu.formal", description=__doc__.splitlines()[0])
521
+ sub = parser.add_subparsers(dest="command", required=True)
522
+ enum = sub.add_parser("enumerate", help="walk the directory listings and record the catalogue")
523
+ enum.add_argument("start", nargs="?", default="itu-t/x/")
524
+ enum.add_argument("--max-requests", type=int, default=400)
525
+ imp = sub.add_parser("import", help="fetch artifacts (the .html path under formal-language/) into ITULIB_HOME")
526
+ imp.add_argument("paths", nargs="+")
527
+ imp.add_argument("--trust-first-use", action="store_true")
528
+ sub.add_parser("list", help="what is imported")
529
+ rd = sub.add_parser("read", help="serve a ref: Rec#year[-mm]/Module[::section]")
530
+ rd.add_argument("ref")
531
+ args = parser.parse_args(argv)
532
+ try:
533
+ if args.command == "enumerate":
534
+ artifacts, budget, dropped = enumerate_tree(
535
+ urllib_transport, args.start, budget=Budget(max_requests=args.max_requests)
536
+ )
537
+ file = save_catalogue(artifacts)
538
+ print(
539
+ json.dumps(
540
+ {
541
+ "artifacts": len(artifacts),
542
+ "requests": budget.requests,
543
+ "stopped": budget.stopped,
544
+ "dropped": dropped,
545
+ "catalogue": str(file),
546
+ }
547
+ )
548
+ )
549
+ elif args.command == "import":
550
+ for path in args.paths:
551
+ done = import_artifact(path, urllib_transport, trust_first_use=args.trust_first_use)
552
+ print(json.dumps(asdict(done)))
553
+ time.sleep(MIN_INTERVAL)
554
+ elif args.command == "list":
555
+ print(json.dumps(listing(), indent=1))
556
+ else:
557
+ print(json.dumps(read(args.ref), indent=1))
558
+ except FormalError as error:
559
+ print(f"error: {error}", file=sys.stderr)
560
+ return 2
561
+ return 0
562
+
563
+
564
+ if __name__ == "__main__":
565
+ raise SystemExit(main())
@@ -0,0 +1,26 @@
1
+ """How the itu collection recognises an ITU-T Recommendation in a supplied PDF (`registry.Collection.identify`).
2
+
3
+ A cover reads `Recommendation ITU-T X.690` and pages carry `Rec. ITU-T X.690 (02/2021)`; ITU's own download is
4
+ named `T-REC-X.690-202102-I!!PDF-E.pdf`. The itu collection fetches its own pinned copy, so the command it
5
+ suggests is the fetch, not an import of this file.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import re
11
+
12
+ from primarysource.identify import Candidate, Probe, candidates
13
+
14
+ _TEXT = re.compile(r"\b(?:Rec(?:ommendation)?\.?\s+)?ITU-T\s+(?P<rec>[A-Z]\.\d{1,4}(?:\.\d{1,3})?)\b")
15
+ _FILE = re.compile(r"T-REC-(?P<rec>[A-Z]\.\d{1,4}(?:\.\d{1,3})?)-")
16
+
17
+
18
+ def _rec(match: re.Match[str]) -> str:
19
+ return f"Rec. ITU-T {match['rec']}"
20
+
21
+
22
+ def identify(probe: Probe) -> list[Candidate]:
23
+ def command(label: str) -> str:
24
+ return f"primarysource itu fetch {label.rsplit(' ', 1)[-1]} (itu fetches its own pinned copy)"
25
+
26
+ return candidates(probe, "itu", _TEXT, _rec, command, file_pattern=_FILE)