primarysource 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (303) hide show
  1. primarysource/__init__.py +5 -0
  2. primarysource/bbf/__init__.py +56 -0
  3. primarysource/bbf/_xml.py +63 -0
  4. primarysource/bbf/catalogue.json +777 -0
  5. primarysource/bbf/catalogue.py +137 -0
  6. primarysource/bbf/cli.py +79 -0
  7. primarysource/bbf/datamodel.py +281 -0
  8. primarysource/bbf/importer.py +151 -0
  9. primarysource/bbf/library.py +380 -0
  10. primarysource/bbf/server.py +72 -0
  11. primarysource/budget.py +63 -0
  12. primarysource/c2sp/__init__.py +53 -0
  13. primarysource/c2sp/_fetch.py +5 -0
  14. primarysource/c2sp/catalogue.json +231 -0
  15. primarysource/c2sp/catalogue.py +95 -0
  16. primarysource/c2sp/cli.py +62 -0
  17. primarysource/c2sp/importer.py +90 -0
  18. primarysource/c2sp/library.py +309 -0
  19. primarysource/c2sp/markdown.py +31 -0
  20. primarysource/c2sp/server.py +68 -0
  21. primarysource/cavp/__init__.py +42 -0
  22. primarysource/cavp/cli.py +84 -0
  23. primarysource/cavp/data/vectors.toml +47 -0
  24. primarysource/cavp/importer.py +156 -0
  25. primarysource/cavp/model.py +279 -0
  26. primarysource/cavp/parse.py +269 -0
  27. primarysource/cavp/paths.py +17 -0
  28. primarysource/cavp/server.py +67 -0
  29. primarysource/cavp/tools.py +169 -0
  30. primarysource/cli.py +107 -0
  31. primarysource/cursor.py +82 -0
  32. primarysource/cve/__init__.py +5 -0
  33. primarysource/cve/__main__.py +6 -0
  34. primarysource/cve/cli.py +144 -0
  35. primarysource/cve/collection.py +20 -0
  36. primarysource/cve/config.py +58 -0
  37. primarysource/cve/errors.py +16 -0
  38. primarysource/cve/kernel.py +232 -0
  39. primarysource/cve/mcp.py +139 -0
  40. primarysource/cve/parse.py +318 -0
  41. primarysource/cve/py.typed +0 -0
  42. primarysource/cve/service.py +760 -0
  43. primarysource/cve/store.py +214 -0
  44. primarysource/cve/sync.py +438 -0
  45. primarysource/cve/views.py +304 -0
  46. primarysource/debpolicy/__init__.py +16 -0
  47. primarysource/debpolicy/catalogue.py +263 -0
  48. primarysource/debpolicy/cli.py +12 -0
  49. primarysource/debpolicy/server.py +11 -0
  50. primarysource/draft/__init__.py +11 -0
  51. primarysource/draft/catalogue.py +117 -0
  52. primarysource/draft/cli.py +149 -0
  53. primarysource/draft/collection.py +21 -0
  54. primarysource/draft/datatracker.py +158 -0
  55. primarysource/draft/importer.py +163 -0
  56. primarysource/draft/licence.py +89 -0
  57. primarysource/draft/names.py +64 -0
  58. primarysource/draft/net.py +133 -0
  59. primarysource/draft/segment.py +192 -0
  60. primarysource/draft/server.py +91 -0
  61. primarysource/draft/status.py +146 -0
  62. primarysource/draft/store.py +199 -0
  63. primarysource/draft/tracked.json +104 -0
  64. primarysource/draft/views.py +430 -0
  65. primarysource/draft/xmlmeta.py +74 -0
  66. primarysource/ecma/__init__.py +52 -0
  67. primarysource/ecma/_pdf.py +143 -0
  68. primarysource/ecma/catalogue.json +22 -0
  69. primarysource/ecma/catalogue.py +100 -0
  70. primarysource/ecma/cli.py +67 -0
  71. primarysource/ecma/identify.py +25 -0
  72. primarysource/ecma/importer.py +108 -0
  73. primarysource/ecma/library.py +240 -0
  74. primarysource/ecma/server.py +67 -0
  75. primarysource/errors.py +29 -0
  76. primarysource/fetch.py +360 -0
  77. primarysource/fsutil.py +150 -0
  78. primarysource/gitdocs/__init__.py +15 -0
  79. primarysource/gitdocs/catalogue.py +116 -0
  80. primarysource/gitdocs/cli.py +12 -0
  81. primarysource/gitdocs/server.py +8 -0
  82. primarysource/home.py +65 -0
  83. primarysource/iana/__init__.py +32 -0
  84. primarysource/iana/catalogue.json +244 -0
  85. primarysource/iana/catalogue.py +69 -0
  86. primarysource/iana/cli.py +69 -0
  87. primarysource/iana/importer.py +157 -0
  88. primarysource/iana/model.py +325 -0
  89. primarysource/iana/reader.py +315 -0
  90. primarysource/iana/server.py +67 -0
  91. primarysource/iana/store.py +83 -0
  92. primarysource/iana/xmlsafe.py +47 -0
  93. primarysource/identify.py +186 -0
  94. primarysource/ieee/__init__.py +58 -0
  95. primarysource/ieee/_html.py +49 -0
  96. primarysource/ieee/catalogue.json +74 -0
  97. primarysource/ieee/catalogue.py +123 -0
  98. primarysource/ieee/cli.py +79 -0
  99. primarysource/ieee/identify.py +24 -0
  100. primarysource/ieee/ieee802.py +47 -0
  101. primarysource/ieee/importer.py +305 -0
  102. primarysource/ieee/posix.py +233 -0
  103. primarysource/ieee/server.py +79 -0
  104. primarysource/index.py +446 -0
  105. primarysource/iso/__init__.py +51 -0
  106. primarysource/iso/cli.py +98 -0
  107. primarysource/iso/designation.py +95 -0
  108. primarysource/iso/identify.py +100 -0
  109. primarysource/iso/importer.py +224 -0
  110. primarysource/iso/library.py +249 -0
  111. primarysource/iso/server.py +69 -0
  112. primarysource/itu/__init__.py +41 -0
  113. primarysource/itu/__main__.py +8 -0
  114. primarysource/itu/asn1mod.py +478 -0
  115. primarysource/itu/catalog.py +472 -0
  116. primarysource/itu/cli.py +445 -0
  117. primarysource/itu/collection.py +20 -0
  118. primarysource/itu/corpus.py +118 -0
  119. primarysource/itu/errata.py +16 -0
  120. primarysource/itu/formal.py +565 -0
  121. primarysource/itu/identify.py +26 -0
  122. primarysource/itu/index.py +490 -0
  123. primarysource/itu/mcp.py +474 -0
  124. primarysource/itu/py.typed +0 -0
  125. primarysource/itu/sources.py +600 -0
  126. primarysource/itu/text.py +609 -0
  127. primarysource/kconfig/__init__.py +16 -0
  128. primarysource/kconfig/catalogue.py +98 -0
  129. primarysource/kconfig/cli.py +12 -0
  130. primarysource/kconfig/server.py +8 -0
  131. primarysource/model.py +63 -0
  132. primarysource/nist/__init__.py +5 -0
  133. primarysource/nist/collection.py +46 -0
  134. primarysource/nist/data/acquired-nist.json +1657 -0
  135. primarysource/nist/data/nist-catalog.json +12090 -0
  136. primarysource/nist/data/sources.toml +1996 -0
  137. primarysource/nist/identify.py +41 -0
  138. primarysource/nist/ingest/__init__.py +1 -0
  139. primarysource/nist/ingest/acquire_nist.py +128 -0
  140. primarysource/nist/ingest/controls.py +82 -0
  141. primarysource/nist/ingest/discover_nist.py +218 -0
  142. primarysource/nist/ingest/extract.py +48 -0
  143. primarysource/nist/ingest/fetch.py +211 -0
  144. primarysource/nist/ingest/index.py +14 -0
  145. primarysource/nist/ingest/oscal.py +480 -0
  146. primarysource/nist/ingest/prose_layouts.py +236 -0
  147. primarysource/nist/ingest/sources.py +94 -0
  148. primarysource/nist/ingest/status.py +255 -0
  149. primarysource/nist/models.py +7 -0
  150. primarysource/nist/paths.py +33 -0
  151. primarysource/nist/server/__init__.py +3 -0
  152. primarysource/nist/server/corpus.py +38 -0
  153. primarysource/nist/server/mcp.py +19 -0
  154. primarysource/nist/server/tools.py +35 -0
  155. primarysource/nist/spec.py +38 -0
  156. primarysource/pdfcorpus/__init__.py +16 -0
  157. primarysource/pdfcorpus/corpus.py +159 -0
  158. primarysource/pdfcorpus/fts.py +175 -0
  159. primarysource/pdfcorpus/index.py +70 -0
  160. primarysource/pdfcorpus/models.py +30 -0
  161. primarysource/pdfcorpus/server.py +96 -0
  162. primarysource/pdfcorpus/spec.py +37 -0
  163. primarysource/pdfcorpus/split.py +391 -0
  164. primarysource/pdfcorpus/tools.py +244 -0
  165. primarysource/pep/__init__.py +15 -0
  166. primarysource/pep/catalogue.py +191 -0
  167. primarysource/pep/cli.py +12 -0
  168. primarysource/pep/server.py +8 -0
  169. primarysource/pkgdocs/__init__.py +19 -0
  170. primarysource/pkgdocs/fetch.py +356 -0
  171. primarysource/pkgdocs/index.py +89 -0
  172. primarysource/pkgdocs/model.py +76 -0
  173. primarysource/pkgdocs/readers/__init__.py +1 -0
  174. primarysource/pkgdocs/readers/_common.py +134 -0
  175. primarysource/pkgdocs/readers/asciidoc.py +160 -0
  176. primarysource/pkgdocs/readers/latex.py +315 -0
  177. primarysource/pkgdocs/readers/markdown.py +173 -0
  178. primarysource/pkgdocs/readers/pod.py +132 -0
  179. primarysource/pkgdocs/readers/rst.py +239 -0
  180. primarysource/pkgdocs/readers/scdoc.py +108 -0
  181. primarysource/pkgdocs/runtime.py +552 -0
  182. primarysource/pkgdocs/serving.py +158 -0
  183. primarysource/pkgdocs/store.py +167 -0
  184. primarysource/pms/__init__.py +16 -0
  185. primarysource/pms/catalogue.py +88 -0
  186. primarysource/pms/cli.py +12 -0
  187. primarysource/pms/server.py +8 -0
  188. primarysource/py.typed +0 -0
  189. primarysource/readers/__init__.py +10 -0
  190. primarysource/readers/bookmarks.py +192 -0
  191. primarysource/readers/html.py +1312 -0
  192. primarysource/readers/markdown.py +189 -0
  193. primarysource/readers/pdf.py +187 -0
  194. primarysource/readers/xml.py +230 -0
  195. primarysource/refs.py +87 -0
  196. primarysource/registry.py +161 -0
  197. primarysource/render.py +49 -0
  198. primarysource/rfc/__init__.py +3 -0
  199. primarysource/rfc/__main__.py +6 -0
  200. primarysource/rfc/cite.py +204 -0
  201. primarysource/rfc/cli.py +903 -0
  202. primarysource/rfc/collection.py +19 -0
  203. primarysource/rfc/config.py +138 -0
  204. primarysource/rfc/errata.py +203 -0
  205. primarysource/rfc/fsutil.py +82 -0
  206. primarysource/rfc/graph.py +213 -0
  207. primarysource/rfc/index/__init__.py +1 -0
  208. primarysource/rfc/index/chunker.py +281 -0
  209. primarysource/rfc/index/embed.py +166 -0
  210. primarysource/rfc/index/hybrid.py +305 -0
  211. primarysource/rfc/index/lexical.py +223 -0
  212. primarysource/rfc/index/rfcxml.py +673 -0
  213. primarysource/rfc/index/search.py +508 -0
  214. primarysource/rfc/index/txtseg.py +585 -0
  215. primarysource/rfc/library.py +294 -0
  216. primarysource/rfc/mcp/__init__.py +1 -0
  217. primarysource/rfc/mcp/server.py +381 -0
  218. primarysource/rfc/packs/__init__.py +1 -0
  219. primarysource/rfc/packs/builder.py +228 -0
  220. primarysource/rfc/packs/collections/all-current-txt.json +10 -0
  221. primarysource/rfc/packs/collections/essentials.json +19 -0
  222. primarysource/rfc/packs/collections/fyi-current.json +10 -0
  223. primarysource/rfc/packs/collections/standards-current.json +10 -0
  224. primarysource/rfc/packs/collections/year-2025.json +10 -0
  225. primarysource/rfc/packs/collections/year-2026.json +10 -0
  226. primarysource/rfc/packs/collections.py +163 -0
  227. primarysource/rfc/packs/docsetexport.py +163 -0
  228. primarysource/rfc/packs/epub.py +159 -0
  229. primarysource/rfc/packs/export.py +306 -0
  230. primarysource/rfc/packs/htmlexport.py +82 -0
  231. primarysource/rfc/packs/serve.py +143 -0
  232. primarysource/rfc/packs/zimexport.py +171 -0
  233. primarysource/rfc/query.py +466 -0
  234. primarysource/rfc/subjects.py +457 -0
  235. primarysource/rfc/sync/__init__.py +1 -0
  236. primarysource/rfc/sync/downloader.py +132 -0
  237. primarysource/rfc/sync/manifest.py +162 -0
  238. primarysource/rfc/sync/scanner.py +49 -0
  239. primarysource/rfc/sync/state.py +464 -0
  240. primarysource/rfc/views.py +1067 -0
  241. primarysource/rpmdocs/__init__.py +15 -0
  242. primarysource/rpmdocs/catalogue.py +117 -0
  243. primarysource/rpmdocs/cli.py +12 -0
  244. primarysource/rpmdocs/server.py +8 -0
  245. primarysource/scaffold.py +223 -0
  246. primarysource/serve.py +75 -0
  247. primarysource/settings.py +152 -0
  248. primarysource/usb/__init__.py +5 -0
  249. primarysource/usb/collection.py +37 -0
  250. primarysource/usb/data/acquired.json +2025 -0
  251. primarysource/usb/data/sources-usb.toml +852 -0
  252. primarysource/usb/data/sources.toml +10 -0
  253. primarysource/usb/data/usb-catalog.json +1663 -0
  254. primarysource/usb/data/usb-classification.json +1986 -0
  255. primarysource/usb/data/usb-corpus-build.json +608 -0
  256. primarysource/usb/data/usb-pdf-choice.json +839 -0
  257. primarysource/usb/data/usb-split-measure.json +1803 -0
  258. primarysource/usb/ingest/__init__.py +1 -0
  259. primarysource/usb/ingest/acquire.py +130 -0
  260. primarysource/usb/ingest/discover_usb.py +88 -0
  261. primarysource/usb/ingest/extract.py +48 -0
  262. primarysource/usb/ingest/fetch.py +197 -0
  263. primarysource/usb/ingest/index.py +14 -0
  264. primarysource/usb/ingest/sources.py +85 -0
  265. primarysource/usb/models.py +7 -0
  266. primarysource/usb/paths.py +33 -0
  267. primarysource/usb/server/__init__.py +3 -0
  268. primarysource/usb/server/corpus.py +38 -0
  269. primarysource/usb/server/mcp.py +19 -0
  270. primarysource/usb/server/tools.py +35 -0
  271. primarysource/usb/spec.py +20 -0
  272. primarysource/usbids/__init__.py +56 -0
  273. primarysource/usbids/cli.py +65 -0
  274. primarysource/usbids/importer.py +272 -0
  275. primarysource/usbids/parse.py +283 -0
  276. primarysource/usbids/server.py +55 -0
  277. primarysource/usbids/store.py +80 -0
  278. primarysource/usbids/tools.py +174 -0
  279. primarysource/zim/__init__.py +5 -0
  280. primarysource/zim/__main__.py +8 -0
  281. primarysource/zim/_libzim.py +37 -0
  282. primarysource/zim/catalog.py +594 -0
  283. primarysource/zim/cli.py +555 -0
  284. primarysource/zim/collection.py +41 -0
  285. primarysource/zim/errors.py +100 -0
  286. primarysource/zim/groups.py +30 -0
  287. primarysource/zim/library.py +668 -0
  288. primarysource/zim/manage.py +636 -0
  289. primarysource/zim/markdown.py +1247 -0
  290. primarysource/zim/mcp.py +201 -0
  291. primarysource/zim/purl.py +130 -0
  292. primarysource/zim/py.typed +0 -0
  293. primarysource/zim/refs.py +90 -0
  294. primarysource/zim/service.py +1182 -0
  295. primarysource/zim/snapshot.py +197 -0
  296. primarysource/zim/sync.py +269 -0
  297. primarysource/zim/views.py +230 -0
  298. primarysource-0.1.0.dist-info/METADATA +118 -0
  299. primarysource-0.1.0.dist-info/RECORD +303 -0
  300. primarysource-0.1.0.dist-info/WHEEL +4 -0
  301. primarysource-0.1.0.dist-info/entry_points.txt +2 -0
  302. primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
  303. primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
@@ -0,0 +1,269 @@
1
+ """Parse CAVP `.rsp` response files and ACVP vector-set JSON into `VectorSet`s, within fixed bounds.
2
+
3
+ Bytes in, a `VectorSet` out: nothing is written to disk and no archive is unpacked (a zip member is read into
4
+ memory, capped at `MAX_SOURCE_BYTES`).
5
+
6
+ An `.rsp` file is a header of `#` comments, then `[...]` lines that open a group, then blocks of `Key = Value`
7
+ lines separated by blank lines. Rules this parser applies, because real files need them:
8
+
9
+ - consecutive `[...]` lines form one group (label joined with `,`); a `[...]` line after cases opens the next;
10
+ - a repeated key inside a block starts a new block, so a file with no blank lines still splits;
11
+ - RSA `SigVer` files put the modulus `n` in a block of its own, ahead of the cases that use it. A block whose
12
+ keys are disjoint from the group's most common block shape is *context*: it is merged into every following
13
+ case of the group and is not a case itself;
14
+ - a case's id is its `COUNT` when every case of the group has a unique one, else its 1-based position.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import io
20
+ import json
21
+ import re
22
+ import zipfile
23
+ from collections import Counter
24
+ from typing import Any
25
+
26
+ from primarysource.cavp.model import (
27
+ KINDS,
28
+ MAX_CASES,
29
+ MAX_FIELDS_PER_CASE,
30
+ MAX_GROUPS,
31
+ MAX_HEADER_LINES,
32
+ MAX_LINE_BYTES,
33
+ MAX_SOURCE_BYTES,
34
+ SET_ID,
35
+ Case,
36
+ Group,
37
+ VectorSet,
38
+ sha256_bytes,
39
+ )
40
+ from primarysource.errors import LibError
41
+
42
+ _MODE = re.compile(r'#\s*"([^"]+)"\s+information', re.IGNORECASE)
43
+
44
+
45
+ def read_member(archive: bytes, member: str) -> bytes:
46
+ """One file out of a zip held in memory, refused when it is larger than the cap (declared or actual)."""
47
+ try:
48
+ zf = zipfile.ZipFile(io.BytesIO(archive))
49
+ info = zf.getinfo(member)
50
+ except zipfile.BadZipFile:
51
+ raise LibError("the artifact is not a zip file", "check the url and the pinned sha256") from None
52
+ except KeyError:
53
+ names = ", ".join(n for n in zf.namelist() if not n.endswith("/"))[:400]
54
+ raise LibError(f"zip has no member {member!r} (members: {names})", "fix `member` in vectors.toml") from None
55
+ if info.file_size > MAX_SOURCE_BYTES:
56
+ raise LibError(f"{member}: {info.file_size} bytes is over the {MAX_SOURCE_BYTES} byte cap", "")
57
+ with zf.open(info) as stream:
58
+ data = stream.read(MAX_SOURCE_BYTES + 1)
59
+ if len(data) > MAX_SOURCE_BYTES:
60
+ raise LibError(f"{member}: larger than the {MAX_SOURCE_BYTES} byte cap", "")
61
+ return data
62
+
63
+
64
+ def _decode(data: bytes, what: str) -> str:
65
+ if len(data) > MAX_SOURCE_BYTES:
66
+ raise LibError(f"{what}: {len(data)} bytes is over the {MAX_SOURCE_BYTES} byte cap", "")
67
+ try:
68
+ return data.decode("utf-8-sig")
69
+ except UnicodeDecodeError as exc:
70
+ raise LibError(f"{what} is not UTF-8 ({exc})", "") from None
71
+
72
+
73
+ def _group_params(labels: list[str]) -> dict[str, Any]:
74
+ """`[L = 1024, N = 160, SHA-1]` -> {"L": "1024", "N": "160", "tokens": ["SHA-1"]}."""
75
+ params: dict[str, Any] = {}
76
+ tokens: list[str] = []
77
+ for label in labels:
78
+ for piece in label.split(","):
79
+ key, eq, value = piece.partition("=")
80
+ if eq:
81
+ params[key.strip()] = value.strip()
82
+ elif piece.strip():
83
+ tokens.append(piece.strip())
84
+ if tokens:
85
+ params["tokens"] = tokens
86
+ return params
87
+
88
+
89
+ def _cases(blocks: list[dict[str, str]]) -> tuple[Case, ...]:
90
+ if not blocks:
91
+ return ()
92
+ dominant = Counter(frozenset(b) for b in blocks).most_common(1)[0][0]
93
+ context: dict[str, str] = {}
94
+ rows: list[dict[str, str]] = []
95
+ for block in blocks:
96
+ if dominant.isdisjoint(block):
97
+ context = block
98
+ else:
99
+ rows.append({**context, **block})
100
+ counts = [row.get("COUNT") for row in rows]
101
+ use_count = bool(rows) and all(c is not None for c in counts) and len(set(counts)) == len(counts)
102
+ return tuple(Case(str(row["COUNT"]) if use_count else str(i), row) for i, row in enumerate(rows, 1))
103
+
104
+
105
+ def parse_rsp(
106
+ data: bytes,
107
+ *,
108
+ set_id: str,
109
+ algorithm: str,
110
+ source_url: str,
111
+ sha256: str,
112
+ member_sha256: str,
113
+ mode: str = "",
114
+ ) -> VectorSet:
115
+ """Parse one `.rsp`. `mode` defaults to the quoted name in the file's header (`# "SigVer" information`)."""
116
+ text = _decode(data, set_id)
117
+ header: list[str] = []
118
+ groups: list[tuple[list[str], list[dict[str, str]]]] = []
119
+ labels: list[str] | None = None
120
+ blocks: list[dict[str, str]] = []
121
+ block: dict[str, str] = {}
122
+
123
+ def close_block() -> None:
124
+ nonlocal block
125
+ if block:
126
+ blocks.append(block)
127
+ block = {}
128
+
129
+ def close_group() -> None:
130
+ nonlocal labels, blocks
131
+ close_block()
132
+ if labels is not None or blocks:
133
+ groups.append((labels if labels is not None else ["all"], blocks))
134
+ labels, blocks = None, []
135
+
136
+ for number, raw in enumerate(text.replace("\r\n", "\n").replace("\r", "\n").split("\n"), 1):
137
+ if len(raw) > MAX_LINE_BYTES:
138
+ raise LibError(f"{set_id} line {number}: over {MAX_LINE_BYTES} characters", "")
139
+ line = raw.strip()
140
+ if not line:
141
+ close_block()
142
+ elif line.startswith("#"):
143
+ if not groups and labels is None and not blocks and len(header) < MAX_HEADER_LINES:
144
+ header.append(line)
145
+ elif line.startswith("[") and line.endswith("]"):
146
+ close_block()
147
+ if labels is None or blocks:
148
+ close_group()
149
+ labels = []
150
+ labels.append(line[1:-1].strip())
151
+ if len(groups) >= MAX_GROUPS:
152
+ raise LibError(f"{set_id}: more than {MAX_GROUPS} groups", "")
153
+ else:
154
+ key, eq, value = line.partition("=")
155
+ if not eq or not key.strip():
156
+ raise LibError(f"{set_id} line {number}: not 'Key = Value' or '[group]': {line[:60]!r}", "")
157
+ key = key.strip()
158
+ if key in block:
159
+ close_block()
160
+ block[key] = value.strip()
161
+ if len(block) > MAX_FIELDS_PER_CASE:
162
+ raise LibError(f"{set_id} line {number}: more than {MAX_FIELDS_PER_CASE} fields in one case", "")
163
+ close_group()
164
+
165
+ seen: Counter[str] = Counter()
166
+ out: list[Group] = []
167
+ total = 0
168
+ for group_labels, group_blocks in groups:
169
+ label = ",".join(group_labels)
170
+ seen[label] += 1
171
+ if seen[label] > 1:
172
+ label = f"{label}~{seen[label]}"
173
+ cases = _cases(group_blocks)
174
+ total += len(cases)
175
+ if total > MAX_CASES:
176
+ raise LibError(f"{set_id}: more than {MAX_CASES} cases", "")
177
+ out.append(Group(label, _group_params(group_labels), cases))
178
+ if not out:
179
+ raise LibError(f"{set_id}: no groups or cases found", "is this a CAVP .rsp file?")
180
+ found = _MODE.search("\n".join(header))
181
+ return VectorSet(
182
+ id=set_id,
183
+ kind="cavp",
184
+ algorithm=algorithm,
185
+ mode=mode or (found.group(1) if found else ""),
186
+ source_url=source_url,
187
+ sha256=sha256,
188
+ member_sha256=member_sha256,
189
+ header=tuple(header),
190
+ groups=tuple(out),
191
+ )
192
+
193
+
194
+ def _acvp_document(data: bytes, what: str) -> dict[str, Any]:
195
+ try:
196
+ doc = json.loads(_decode(data, what))
197
+ except (ValueError, RecursionError) as exc:
198
+ raise LibError(f"{what} is not valid JSON ({exc})", "") from None
199
+ if isinstance(doc, list): # a server response: [{"acvVersion": ...}, {the vector set}]
200
+ doc = next((d for d in doc if isinstance(d, dict) and "testGroups" in d), None)
201
+ if not isinstance(doc, dict) or not isinstance(doc.get("testGroups"), list):
202
+ raise LibError(f"{what}: no testGroups", "is this an ACVP prompt.json or expectedResults.json?")
203
+ return doc
204
+
205
+
206
+ def parse_acvp(
207
+ prompt: bytes,
208
+ expected: bytes | None,
209
+ *,
210
+ set_id: str,
211
+ source_url: str,
212
+ sha256: str,
213
+ member_sha256: str,
214
+ ) -> VectorSet:
215
+ """One ACVP vector set: `prompt.json` (the inputs), joined by (tgId, tcId) with `expectedResults.json`."""
216
+ doc = _acvp_document(prompt, f"{set_id} prompt")
217
+ answers: dict[tuple[str, str], dict[str, Any]] = {}
218
+ if expected is not None:
219
+ for g in _acvp_document(expected, f"{set_id} expectedResults")["testGroups"]:
220
+ for t in g.get("tests", []):
221
+ answers[(str(g.get("tgId")), str(t.get("tcId")))] = {k: v for k, v in t.items() if k != "tcId"}
222
+ groups: list[Group] = []
223
+ total = 0
224
+ used = 0
225
+ for g in doc["testGroups"]:
226
+ tg = str(g.get("tgId"))
227
+ tests = g.get("tests")
228
+ if not isinstance(tests, list):
229
+ raise LibError(f"{set_id} group {tg}: no tests list", "")
230
+ cases = []
231
+ for t in tests:
232
+ tc = str(t.get("tcId"))
233
+ fields = {k: v for k, v in t.items() if k != "tcId"}
234
+ if len(fields) > MAX_FIELDS_PER_CASE:
235
+ raise LibError(f"{set_id} case {tg}/{tc}: more than {MAX_FIELDS_PER_CASE} fields", "")
236
+ answer = answers.get((tg, tc), {})
237
+ used += bool(answer)
238
+ cases.append(Case(tc, fields, answer))
239
+ total += len(cases)
240
+ if total > MAX_CASES or len(groups) >= MAX_GROUPS:
241
+ raise LibError(f"{set_id}: over {MAX_CASES} cases or {MAX_GROUPS} groups", "")
242
+ groups.append(Group(tg, {k: v for k, v in g.items() if k not in ("tests", "tgId")}, tuple(cases)))
243
+ if expected is not None and used != len(answers):
244
+ raise LibError(
245
+ f"{set_id}: expectedResults has {len(answers) - used} cases that prompt does not",
246
+ "pair the files of one vector set",
247
+ )
248
+ return VectorSet(
249
+ id=set_id,
250
+ kind="acvp",
251
+ algorithm=str(doc.get("algorithm", "")),
252
+ mode=str(doc.get("mode", "")),
253
+ source_url=source_url,
254
+ sha256=sha256,
255
+ member_sha256=member_sha256,
256
+ header=(),
257
+ groups=tuple(groups),
258
+ revision=str(doc.get("revision", "")),
259
+ )
260
+
261
+
262
+ def check_set_id(set_id: str, kind: str) -> None:
263
+ if kind not in KINDS or not SET_ID.fullmatch(set_id) or not set_id.startswith(kind + "/"):
264
+ raise LibError(
265
+ f"bad set id {set_id!r} for kind {kind!r}", "ids look like cavp/ecdsa/SigVer.rsp or acvp/ECDSA-SigVer-1.0"
266
+ )
267
+
268
+
269
+ __all__ = ["check_set_id", "parse_acvp", "parse_rsp", "read_member", "sha256_bytes"]
@@ -0,0 +1,17 @@
1
+ """Where the cavp collection keeps imported vector sets: `$PRIMARYSOURCE_HOME/cavp`, else the per-user
2
+ `primarysource/cavp` (`primarysource.home.collection_home`). Never the working directory, never the package."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from pathlib import Path
7
+
8
+ from primarysource.cavp.model import Store
9
+ from primarysource.home import collection_home
10
+
11
+
12
+ def store_dir() -> Path:
13
+ return collection_home("cavp")
14
+
15
+
16
+ def open_store() -> Store:
17
+ return Store(store_dir())
@@ -0,0 +1,67 @@
1
+ """MCP server for the cavp collection: `primarysource mcp --collection cavp`.
2
+
3
+ Tools are prefixed `cavp_`, answer in text with the fixed footer, and turn a `LibError` into a tool error naming
4
+ the remedy. Read-only, local files only.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ from mcp.server.mcpserver import MCPServer
10
+ from mcp.types import ToolAnnotations
11
+
12
+ from primarysource import __version__
13
+ from primarysource.cavp import tools
14
+ from primarysource.cavp.paths import open_store
15
+ from primarysource.scaffold import add_tools
16
+
17
+ READ_ONLY = ToolAnnotations(read_only_hint=True, idempotent_hint=True, destructive_hint=False, open_world_hint=False)
18
+ INSTRUCTIONS = (
19
+ "NIST cryptographic test vectors (CAVP .rsp, ACVP JSON) the user imported, each pinned by sha256. "
20
+ "cavp_sets() lists them as refs; cavp_groups('nist:cavp/ecdsa/SigVer.rsp') lists a set's groups; "
21
+ "cavp_read('nist:cavp/ecdsa/SigVer.rsp#[P-256,SHA-256]/3') reads one case in full; cavp_find looks a case up "
22
+ "by a field value (a message, a key, a signature). ACVP refs use the tgId and tcId: "
23
+ "'nist:acvp/ECDSA-SigVer-1.0#1/3'. The cite: footer line is the citation. If nothing is imported the error "
24
+ "says how."
25
+ )
26
+
27
+
28
+ def cavp_sets(query: str | None = None, limit: int = 50) -> str:
29
+ """List imported vector sets, one ref per line with algorithm, mode and counts. query filters by id,
30
+ algorithm or mode."""
31
+ return tools.sets(open_store(), query, limit)
32
+
33
+
34
+ def cavp_groups(set_ref: str, start: int = 0, max_tokens: int = 2000) -> str:
35
+ """List a set's groups as refs with their case counts and parameters. set_ref: 'nist:cavp/ecdsa/SigVer.rsp'.
36
+ A 'next: start=N' footer line means call again with start=N (a 0-based offset)."""
37
+ return tools.groups(open_store(), set_ref, start, max_tokens)
38
+
39
+
40
+ def cavp_read(ref: str, max_tokens: int = 2000) -> str:
41
+ """Read by ref: a set ('nist:cavp/ecdsa/SigVer.rsp'), a group ('...#[P-256,SHA-256]', values shortened) or one
42
+ case in full ('...#[P-256,SHA-256]/3'). A 'next: <ref>' footer line is the ref that reads the next page: pass
43
+ it back as ref."""
44
+ return tools.read(open_store(), ref, max_tokens)
45
+
46
+
47
+ def cavp_find(
48
+ set_ref: str,
49
+ field: str,
50
+ value: str,
51
+ group: str | None = None,
52
+ start: int = 0,
53
+ max_tokens: int = 2000,
54
+ ) -> str:
55
+ """Find cases in a set whose field equals value (hex compared without case or 0x), e.g. field='Msg'.
56
+ Returns refs. field may be 'expected.<name>' for an ACVP answer. A 'next: start=N' footer line means call
57
+ again with start=N (a 0-based offset)."""
58
+ return tools.find(open_store(), set_ref, field, value, group, start, max_tokens)
59
+
60
+
61
+ def cavp_cite(ref: str) -> str:
62
+ """The citation for a ref: algorithm, mode, source URL, pinned sha256, and NIST's credit line."""
63
+ return tools.cite(open_store(), ref)
64
+
65
+
66
+ cavp_server = MCPServer("primarysource-cavp", version=__version__, instructions=INSTRUCTIONS)
67
+ add_tools(cavp_server, (cavp_sets, cavp_groups, cavp_read, cavp_find, cavp_cite), annotations=READ_ONLY)
@@ -0,0 +1,169 @@
1
+ """The vector tools as plain functions over a `Store`, returning text with the core's fixed footer.
2
+
3
+ Each result is a body (one line per item, or one case in full), then footer lines: `cite:` names the set, its
4
+ source file and pinned sha256, `span:` is the hash of the exact text returned (case reads). When the token budget
5
+ cuts a page, `next:` is exactly what to pass back: for `read`, the ref that reads on (`<set or group ref>¶<n>`,
6
+ the nth line of that listing); for `groups` and `find`, `start=<n>`, a 0-based line offset.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ from typing import Any
13
+
14
+ from primarysource import cursor
15
+ from primarysource.budget import page
16
+ from primarysource.cavp.model import (
17
+ CREDIT,
18
+ Case,
19
+ Group,
20
+ Store,
21
+ VectorSet,
22
+ find_case,
23
+ find_group,
24
+ format_ref,
25
+ norm_value,
26
+ parse_ref,
27
+ )
28
+ from primarysource.errors import LibError
29
+ from primarysource.model import span_hash
30
+ from primarysource.render import next_start, render
31
+
32
+ SHOWN = 60 # characters of a field value in a listing; a case read is never shortened
33
+
34
+
35
+ def _value(v: Any) -> str:
36
+ return v if isinstance(v, str) else json.dumps(v, separators=(",", ":"))
37
+
38
+
39
+ def _short(v: Any) -> str:
40
+ s = _value(v)
41
+ return s if len(s) <= SHOWN else f"{s[:SHOWN]}...({len(s)} chars)"
42
+
43
+
44
+ def _cite(s: VectorSet, where: str = "") -> str:
45
+ kind = "ACVP" if s.kind == "acvp" else "CAVP"
46
+ head = f"NIST {kind} {s.algorithm} {s.mode}".strip()
47
+ rev = f" rev {s.revision}" if s.revision else ""
48
+ return f"{head}{rev}{where}; {s.source_url}; sha256:{s.sha256}"
49
+
50
+
51
+ def _listed(lines: list[str], start: int, max_tokens: int, cite: str) -> str:
52
+ """A list cut at the token budget from the 0-based line `start`; `next: start=<n>` continues."""
53
+ if start < 0 or max_tokens < 1:
54
+ raise LibError(f"start is a 0-based offset and max_tokens positive (got {start}, {max_tokens})", "")
55
+ got = page(lines, start + 1, max_tokens)
56
+ body = "\n".join(got.units) if got.units else f"nothing at start={start} ({got.total} lines)"
57
+ resume = None if got.next_start is None else got.next_start - 1
58
+ return render(body, {"cite": cite, "next": next_start(resume)})
59
+
60
+
61
+ def _read_page(lines: list[str], scope: str, line: int | None, max_tokens: int, cite: str) -> str:
62
+ """A read cut at the token budget from line `line` (1-based); `next:` is the ref that reads on."""
63
+ if max_tokens < 1:
64
+ raise LibError("max_tokens must be positive", "")
65
+ got = page(lines, cursor.first_unit(line, len(lines), "lines", f"{scope}¶{line}"), max_tokens)
66
+ return render("\n".join(got.units), {"cite": cite, "next": cursor.read_next(scope, got)})
67
+
68
+
69
+ def sets(store: Store, query: str | None = None, limit: int = 50) -> str:
70
+ """One line per imported set. `query` keeps the sets whose id, algorithm or mode contains it."""
71
+ entries = store.require()
72
+ needle = (query or "").lower()
73
+ hits = [e for e in entries if needle in f"{e.id} {e.algorithm} {e.mode}".lower()]
74
+ if not hits:
75
+ return f"no vector set matches {query!r} ({len(entries)} imported)\n"
76
+ lines = [
77
+ f"{format_ref(e.id)}: {e.algorithm} {e.mode}{' rev ' + e.revision if e.revision else ''}, "
78
+ f"{e.groups} groups, {e.cases} cases"
79
+ for e in hits[: max(1, limit)]
80
+ ]
81
+ if len(hits) > limit:
82
+ lines.append(f"... {len(hits) - limit} more; narrow the query")
83
+ return "\n".join(lines) + "\n"
84
+
85
+
86
+ def _group_line(s: VectorSet, g: Group) -> str:
87
+ extras = " ".join(f"{k}={_short(v)}" for k, v in g.params.items())
88
+ return f"{format_ref(s.id, g.label)}: {len(g.cases)} cases {extras}".rstrip()
89
+
90
+
91
+ def _case_body(s: VectorSet, g: Group, c: Case) -> str:
92
+ lines = [format_ref(s.id, g.label, c.id)]
93
+ lines += [f"{k} = {_value(v)}" for k, v in c.fields.items()]
94
+ lines += [f"expected.{k} = {_value(v)}" for k, v in c.expected.items()]
95
+ return "\n".join(lines)
96
+
97
+
98
+ def groups(store: Store, set_ref: str, start: int = 0, max_tokens: int = 2000) -> str:
99
+ """The groups of a set, one line each, as refs."""
100
+ ref = parse_ref(set_ref)
101
+ s = store.load(ref.set_id)
102
+ return _listed([_group_line(s, g) for g in s.groups], start, max_tokens, _cite(s))
103
+
104
+
105
+ def read(store: Store, ref_text: str, max_tokens: int = 2000) -> str:
106
+ """A set (its header and groups), a group (its cases, values shortened) or one case in full. `<ref>¶n` reads a
107
+ set or a group from line n of its listing."""
108
+ text, line = cursor.split_para(ref_text)
109
+ ref = parse_ref(text)
110
+ s = store.load(ref.set_id)
111
+ if ref.group is None:
112
+ header = [f"# {h.lstrip('# ')}" for h in s.header[:5]]
113
+ lines = header + [_group_line(s, g) for g in s.groups]
114
+ return _read_page(lines, format_ref(s.id), line, max_tokens, _cite(s))
115
+ g = find_group(s, ref.group)
116
+ if ref.case is None:
117
+ lines = [
118
+ f"{format_ref(s.id, g.label, c.id)}: "
119
+ + " ".join(
120
+ f"{k}={_short(v)}"
121
+ for k, v in {**c.fields, **{f"expected.{k}": v for k, v in c.expected.items()}}.items()
122
+ )
123
+ for c in g.cases
124
+ ]
125
+ return _read_page(lines, format_ref(s.id, g.label), line, max_tokens, _cite(s, f", group {g.label}"))
126
+ if line is not None:
127
+ raise LibError(f"{ref_text}: a case is read whole", f"read {format_ref(s.id, g.label, ref.case)}")
128
+ c = find_case(g, ref.case)
129
+ body = _case_body(s, g, c)
130
+ return render(body, {"cite": _cite(s, f", group {g.label}, case {c.id}"), "span": span_hash(body)})
131
+
132
+
133
+ def find(
134
+ store: Store,
135
+ set_ref: str,
136
+ field: str,
137
+ value: str,
138
+ group: str | None = None,
139
+ start: int = 0,
140
+ max_tokens: int = 2000,
141
+ ) -> str:
142
+ """The cases of a set whose `field` equals `value` (compared as hex: case and `0x` ignored), as refs with the
143
+ case's other short fields. `field` may be `expected.<name>` for an ACVP answer. Zero matches is said, not an
144
+ error."""
145
+ s = store.load(parse_ref(set_ref).set_id)
146
+ chosen = [find_group(s, group)] if group else list(s.groups)
147
+ want = norm_value(value)
148
+ lines: list[str] = []
149
+ for g in chosen:
150
+ for c in g.cases:
151
+ held = c.expected.get(field[9:]) if field.startswith("expected.") else c.fields.get(field)
152
+ if held is not None and norm_value(_value(held)) == want:
153
+ lines.append(f"{format_ref(s.id, g.label, c.id)}: {_short(held)}")
154
+ if not lines:
155
+ return f"no case in {format_ref(s.id)} has {field} = {_short(value)}\n"
156
+ return _listed(lines, start, max_tokens, _cite(s))
157
+
158
+
159
+ def cite(store: Store, ref_text: str) -> str:
160
+ """The citation for a ref, with its source, pinned digest and NIST's credit line."""
161
+ ref = parse_ref(ref_text)
162
+ s = store.load(ref.set_id)
163
+ where = ""
164
+ if ref.group is not None:
165
+ g = find_group(s, ref.group)
166
+ where = f", group {g.label}"
167
+ if ref.case is not None:
168
+ where += f", case {find_case(g, ref.case).id}"
169
+ return f"{format_ref(s.id, ref.group, ref.case)}\n{_cite(s, where)}\n{CREDIT}\n"
primarysource/cli.py ADDED
@@ -0,0 +1,107 @@
1
+ """The `primarysource` command.
2
+
3
+ primarysource collections list what is registered
4
+ primarysource identify <pdf or folder> ... which collection knows each PDF, and how to import it
5
+ primarysource mcp --collection rfc [--collection nist ...] [--bare-names]
6
+ [--transport http] [--host H] [--port P] [--allowed-host H] [--read-only]
7
+ primarysource rfc <command> ... the collection's own commands (sync, index, status, ...)
8
+
9
+ `mcp` serves every named collection from one process, each with its own database and its tools named
10
+ `<collection>_<tool>` (PLAN.md §5.12). `--bare-names` keeps a collection's own tool names instead; it is how the
11
+ old single-collection commands (`rfclib mcp`) stay compatible, and two collections with the same bare names are
12
+ refused at startup.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import argparse
18
+ import os
19
+ import sys
20
+ from collections.abc import Sequence
21
+ from pathlib import Path
22
+ from typing import Any
23
+
24
+ from primarysource import registry, scaffold, serve
25
+ from primarysource.errors import LibError
26
+
27
+
28
+ def _parser() -> argparse.ArgumentParser:
29
+ parser = argparse.ArgumentParser(prog="primarysource", description="Search, read and cite primary sources.")
30
+ commands = parser.add_subparsers(dest="command", required=True)
31
+ commands.add_parser("collections", help="list the registered collections")
32
+ ident = commands.add_parser("identify", help="say which collection knows each PDF, and how to import it")
33
+ ident.add_argument("paths", nargs="+", type=Path, help="PDFs, or folders of PDFs")
34
+ mcp = commands.add_parser("mcp", help="run the MCP server for one or more collections")
35
+ mcp.add_argument(
36
+ "--collection", action="append", required=True, help="a collection to serve, e.g. rfc (repeatable)"
37
+ )
38
+ mcp.add_argument(
39
+ "--bare-names",
40
+ action="store_true",
41
+ help="keep each collection's own tool names (search, read) instead of <collection>_<tool>",
42
+ )
43
+ mcp.add_argument("--transport", default="stdio", help="stdio (one server per client) or http (one shared server)")
44
+ mcp.add_argument("--host", default="127.0.0.1", help="http: address to bind; there is no authentication")
45
+ mcp.add_argument("--port", type=int, default=serve.MCP_HTTP_PORT, help="http: TCP port")
46
+ mcp.add_argument("--allowed-host", action="append", default=[], help="http: Host header to accept (repeatable)")
47
+ mcp.add_argument("--read-only", action="store_true", help="never write the library and never fetch")
48
+ return parser
49
+
50
+
51
+ def collection_server(collection: registry.Collection) -> Any:
52
+ """The `MCPServer` a collection's own module builds; a factory such as itu's `build()` is called."""
53
+ server = registry.attribute(collection, collection.server, "server")
54
+ if not hasattr(server, "run") and callable(server):
55
+ server = server()
56
+ return server
57
+
58
+
59
+ def build_server(names: Sequence[str], *, bare: bool = False) -> Any:
60
+ """The one server for `names`, in the order given; a name given twice is served once."""
61
+ served = []
62
+ for name in dict.fromkeys(names):
63
+ collection = registry.load(name)
64
+ served.append(scaffold.Served(collection.name, collection.title, collection_server(collection)))
65
+ return scaffold.compose(served, bare=bare)
66
+
67
+
68
+ def main(argv: Sequence[str] | None = None) -> int:
69
+ args_in = list(sys.argv[1:] if argv is None else argv)
70
+ try:
71
+ # `primarysource <collection> ...` hands the rest of the line to that collection's own command
72
+ if args_in and args_in[0] in registry.available():
73
+ collection = registry.load(args_in[0])
74
+ entry = registry.attribute(collection, collection.cli, "command line")
75
+ result = entry(args_in[1:])
76
+ return int(result or 0)
77
+ args = _parser().parse_args(args_in)
78
+ if args.command == "collections":
79
+ for name in registry.available():
80
+ print(name)
81
+ return 0
82
+ if args.command == "identify":
83
+ from primarysource import identify
84
+
85
+ hooks = identify.registered_hooks()
86
+ for path in identify.pdfs(args.paths):
87
+ try:
88
+ print(identify.describe(path, identify.identify(identify.probe_pdf(path), hooks)))
89
+ except LibError as exc:
90
+ print(f"{path.name}: {exc.message}")
91
+ return 0
92
+ transport = serve.normalise_transport(args.transport)
93
+ if args.read_only:
94
+ os.environ["PRIMARYSOURCE_READONLY"] = "1"
95
+ server = build_server(args.collection, bare=args.bare_names)
96
+ if transport == "http":
97
+ names = ", ".join(dict.fromkeys(args.collection))
98
+ print(f"primarysource mcp: {names} at http://{args.host}:{args.port}/mcp", file=sys.stderr)
99
+ serve.run(server, transport, host=args.host, port=args.port, allowed_hosts=args.allowed_host)
100
+ return 0
101
+ except LibError as exc:
102
+ print(exc.as_text(), file=sys.stderr)
103
+ return 2
104
+
105
+
106
+ if __name__ == "__main__":
107
+ raise SystemExit(main())