primarysource 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (303) hide show
  1. primarysource/__init__.py +5 -0
  2. primarysource/bbf/__init__.py +56 -0
  3. primarysource/bbf/_xml.py +63 -0
  4. primarysource/bbf/catalogue.json +777 -0
  5. primarysource/bbf/catalogue.py +137 -0
  6. primarysource/bbf/cli.py +79 -0
  7. primarysource/bbf/datamodel.py +281 -0
  8. primarysource/bbf/importer.py +151 -0
  9. primarysource/bbf/library.py +380 -0
  10. primarysource/bbf/server.py +72 -0
  11. primarysource/budget.py +63 -0
  12. primarysource/c2sp/__init__.py +53 -0
  13. primarysource/c2sp/_fetch.py +5 -0
  14. primarysource/c2sp/catalogue.json +231 -0
  15. primarysource/c2sp/catalogue.py +95 -0
  16. primarysource/c2sp/cli.py +62 -0
  17. primarysource/c2sp/importer.py +90 -0
  18. primarysource/c2sp/library.py +309 -0
  19. primarysource/c2sp/markdown.py +31 -0
  20. primarysource/c2sp/server.py +68 -0
  21. primarysource/cavp/__init__.py +42 -0
  22. primarysource/cavp/cli.py +84 -0
  23. primarysource/cavp/data/vectors.toml +47 -0
  24. primarysource/cavp/importer.py +156 -0
  25. primarysource/cavp/model.py +279 -0
  26. primarysource/cavp/parse.py +269 -0
  27. primarysource/cavp/paths.py +17 -0
  28. primarysource/cavp/server.py +67 -0
  29. primarysource/cavp/tools.py +169 -0
  30. primarysource/cli.py +107 -0
  31. primarysource/cursor.py +82 -0
  32. primarysource/cve/__init__.py +5 -0
  33. primarysource/cve/__main__.py +6 -0
  34. primarysource/cve/cli.py +144 -0
  35. primarysource/cve/collection.py +20 -0
  36. primarysource/cve/config.py +58 -0
  37. primarysource/cve/errors.py +16 -0
  38. primarysource/cve/kernel.py +232 -0
  39. primarysource/cve/mcp.py +139 -0
  40. primarysource/cve/parse.py +318 -0
  41. primarysource/cve/py.typed +0 -0
  42. primarysource/cve/service.py +760 -0
  43. primarysource/cve/store.py +214 -0
  44. primarysource/cve/sync.py +438 -0
  45. primarysource/cve/views.py +304 -0
  46. primarysource/debpolicy/__init__.py +16 -0
  47. primarysource/debpolicy/catalogue.py +263 -0
  48. primarysource/debpolicy/cli.py +12 -0
  49. primarysource/debpolicy/server.py +11 -0
  50. primarysource/draft/__init__.py +11 -0
  51. primarysource/draft/catalogue.py +117 -0
  52. primarysource/draft/cli.py +149 -0
  53. primarysource/draft/collection.py +21 -0
  54. primarysource/draft/datatracker.py +158 -0
  55. primarysource/draft/importer.py +163 -0
  56. primarysource/draft/licence.py +89 -0
  57. primarysource/draft/names.py +64 -0
  58. primarysource/draft/net.py +133 -0
  59. primarysource/draft/segment.py +192 -0
  60. primarysource/draft/server.py +91 -0
  61. primarysource/draft/status.py +146 -0
  62. primarysource/draft/store.py +199 -0
  63. primarysource/draft/tracked.json +104 -0
  64. primarysource/draft/views.py +430 -0
  65. primarysource/draft/xmlmeta.py +74 -0
  66. primarysource/ecma/__init__.py +52 -0
  67. primarysource/ecma/_pdf.py +143 -0
  68. primarysource/ecma/catalogue.json +22 -0
  69. primarysource/ecma/catalogue.py +100 -0
  70. primarysource/ecma/cli.py +67 -0
  71. primarysource/ecma/identify.py +25 -0
  72. primarysource/ecma/importer.py +108 -0
  73. primarysource/ecma/library.py +240 -0
  74. primarysource/ecma/server.py +67 -0
  75. primarysource/errors.py +29 -0
  76. primarysource/fetch.py +360 -0
  77. primarysource/fsutil.py +150 -0
  78. primarysource/gitdocs/__init__.py +15 -0
  79. primarysource/gitdocs/catalogue.py +116 -0
  80. primarysource/gitdocs/cli.py +12 -0
  81. primarysource/gitdocs/server.py +8 -0
  82. primarysource/home.py +65 -0
  83. primarysource/iana/__init__.py +32 -0
  84. primarysource/iana/catalogue.json +244 -0
  85. primarysource/iana/catalogue.py +69 -0
  86. primarysource/iana/cli.py +69 -0
  87. primarysource/iana/importer.py +157 -0
  88. primarysource/iana/model.py +325 -0
  89. primarysource/iana/reader.py +315 -0
  90. primarysource/iana/server.py +67 -0
  91. primarysource/iana/store.py +83 -0
  92. primarysource/iana/xmlsafe.py +47 -0
  93. primarysource/identify.py +186 -0
  94. primarysource/ieee/__init__.py +58 -0
  95. primarysource/ieee/_html.py +49 -0
  96. primarysource/ieee/catalogue.json +74 -0
  97. primarysource/ieee/catalogue.py +123 -0
  98. primarysource/ieee/cli.py +79 -0
  99. primarysource/ieee/identify.py +24 -0
  100. primarysource/ieee/ieee802.py +47 -0
  101. primarysource/ieee/importer.py +305 -0
  102. primarysource/ieee/posix.py +233 -0
  103. primarysource/ieee/server.py +79 -0
  104. primarysource/index.py +446 -0
  105. primarysource/iso/__init__.py +51 -0
  106. primarysource/iso/cli.py +98 -0
  107. primarysource/iso/designation.py +95 -0
  108. primarysource/iso/identify.py +100 -0
  109. primarysource/iso/importer.py +224 -0
  110. primarysource/iso/library.py +249 -0
  111. primarysource/iso/server.py +69 -0
  112. primarysource/itu/__init__.py +41 -0
  113. primarysource/itu/__main__.py +8 -0
  114. primarysource/itu/asn1mod.py +478 -0
  115. primarysource/itu/catalog.py +472 -0
  116. primarysource/itu/cli.py +445 -0
  117. primarysource/itu/collection.py +20 -0
  118. primarysource/itu/corpus.py +118 -0
  119. primarysource/itu/errata.py +16 -0
  120. primarysource/itu/formal.py +565 -0
  121. primarysource/itu/identify.py +26 -0
  122. primarysource/itu/index.py +490 -0
  123. primarysource/itu/mcp.py +474 -0
  124. primarysource/itu/py.typed +0 -0
  125. primarysource/itu/sources.py +600 -0
  126. primarysource/itu/text.py +609 -0
  127. primarysource/kconfig/__init__.py +16 -0
  128. primarysource/kconfig/catalogue.py +98 -0
  129. primarysource/kconfig/cli.py +12 -0
  130. primarysource/kconfig/server.py +8 -0
  131. primarysource/model.py +63 -0
  132. primarysource/nist/__init__.py +5 -0
  133. primarysource/nist/collection.py +46 -0
  134. primarysource/nist/data/acquired-nist.json +1657 -0
  135. primarysource/nist/data/nist-catalog.json +12090 -0
  136. primarysource/nist/data/sources.toml +1996 -0
  137. primarysource/nist/identify.py +41 -0
  138. primarysource/nist/ingest/__init__.py +1 -0
  139. primarysource/nist/ingest/acquire_nist.py +128 -0
  140. primarysource/nist/ingest/controls.py +82 -0
  141. primarysource/nist/ingest/discover_nist.py +218 -0
  142. primarysource/nist/ingest/extract.py +48 -0
  143. primarysource/nist/ingest/fetch.py +211 -0
  144. primarysource/nist/ingest/index.py +14 -0
  145. primarysource/nist/ingest/oscal.py +480 -0
  146. primarysource/nist/ingest/prose_layouts.py +236 -0
  147. primarysource/nist/ingest/sources.py +94 -0
  148. primarysource/nist/ingest/status.py +255 -0
  149. primarysource/nist/models.py +7 -0
  150. primarysource/nist/paths.py +33 -0
  151. primarysource/nist/server/__init__.py +3 -0
  152. primarysource/nist/server/corpus.py +38 -0
  153. primarysource/nist/server/mcp.py +19 -0
  154. primarysource/nist/server/tools.py +35 -0
  155. primarysource/nist/spec.py +38 -0
  156. primarysource/pdfcorpus/__init__.py +16 -0
  157. primarysource/pdfcorpus/corpus.py +159 -0
  158. primarysource/pdfcorpus/fts.py +175 -0
  159. primarysource/pdfcorpus/index.py +70 -0
  160. primarysource/pdfcorpus/models.py +30 -0
  161. primarysource/pdfcorpus/server.py +96 -0
  162. primarysource/pdfcorpus/spec.py +37 -0
  163. primarysource/pdfcorpus/split.py +391 -0
  164. primarysource/pdfcorpus/tools.py +244 -0
  165. primarysource/pep/__init__.py +15 -0
  166. primarysource/pep/catalogue.py +191 -0
  167. primarysource/pep/cli.py +12 -0
  168. primarysource/pep/server.py +8 -0
  169. primarysource/pkgdocs/__init__.py +19 -0
  170. primarysource/pkgdocs/fetch.py +356 -0
  171. primarysource/pkgdocs/index.py +89 -0
  172. primarysource/pkgdocs/model.py +76 -0
  173. primarysource/pkgdocs/readers/__init__.py +1 -0
  174. primarysource/pkgdocs/readers/_common.py +134 -0
  175. primarysource/pkgdocs/readers/asciidoc.py +160 -0
  176. primarysource/pkgdocs/readers/latex.py +315 -0
  177. primarysource/pkgdocs/readers/markdown.py +173 -0
  178. primarysource/pkgdocs/readers/pod.py +132 -0
  179. primarysource/pkgdocs/readers/rst.py +239 -0
  180. primarysource/pkgdocs/readers/scdoc.py +108 -0
  181. primarysource/pkgdocs/runtime.py +552 -0
  182. primarysource/pkgdocs/serving.py +158 -0
  183. primarysource/pkgdocs/store.py +167 -0
  184. primarysource/pms/__init__.py +16 -0
  185. primarysource/pms/catalogue.py +88 -0
  186. primarysource/pms/cli.py +12 -0
  187. primarysource/pms/server.py +8 -0
  188. primarysource/py.typed +0 -0
  189. primarysource/readers/__init__.py +10 -0
  190. primarysource/readers/bookmarks.py +192 -0
  191. primarysource/readers/html.py +1312 -0
  192. primarysource/readers/markdown.py +189 -0
  193. primarysource/readers/pdf.py +187 -0
  194. primarysource/readers/xml.py +230 -0
  195. primarysource/refs.py +87 -0
  196. primarysource/registry.py +161 -0
  197. primarysource/render.py +49 -0
  198. primarysource/rfc/__init__.py +3 -0
  199. primarysource/rfc/__main__.py +6 -0
  200. primarysource/rfc/cite.py +204 -0
  201. primarysource/rfc/cli.py +903 -0
  202. primarysource/rfc/collection.py +19 -0
  203. primarysource/rfc/config.py +138 -0
  204. primarysource/rfc/errata.py +203 -0
  205. primarysource/rfc/fsutil.py +82 -0
  206. primarysource/rfc/graph.py +213 -0
  207. primarysource/rfc/index/__init__.py +1 -0
  208. primarysource/rfc/index/chunker.py +281 -0
  209. primarysource/rfc/index/embed.py +166 -0
  210. primarysource/rfc/index/hybrid.py +305 -0
  211. primarysource/rfc/index/lexical.py +223 -0
  212. primarysource/rfc/index/rfcxml.py +673 -0
  213. primarysource/rfc/index/search.py +508 -0
  214. primarysource/rfc/index/txtseg.py +585 -0
  215. primarysource/rfc/library.py +294 -0
  216. primarysource/rfc/mcp/__init__.py +1 -0
  217. primarysource/rfc/mcp/server.py +381 -0
  218. primarysource/rfc/packs/__init__.py +1 -0
  219. primarysource/rfc/packs/builder.py +228 -0
  220. primarysource/rfc/packs/collections/all-current-txt.json +10 -0
  221. primarysource/rfc/packs/collections/essentials.json +19 -0
  222. primarysource/rfc/packs/collections/fyi-current.json +10 -0
  223. primarysource/rfc/packs/collections/standards-current.json +10 -0
  224. primarysource/rfc/packs/collections/year-2025.json +10 -0
  225. primarysource/rfc/packs/collections/year-2026.json +10 -0
  226. primarysource/rfc/packs/collections.py +163 -0
  227. primarysource/rfc/packs/docsetexport.py +163 -0
  228. primarysource/rfc/packs/epub.py +159 -0
  229. primarysource/rfc/packs/export.py +306 -0
  230. primarysource/rfc/packs/htmlexport.py +82 -0
  231. primarysource/rfc/packs/serve.py +143 -0
  232. primarysource/rfc/packs/zimexport.py +171 -0
  233. primarysource/rfc/query.py +466 -0
  234. primarysource/rfc/subjects.py +457 -0
  235. primarysource/rfc/sync/__init__.py +1 -0
  236. primarysource/rfc/sync/downloader.py +132 -0
  237. primarysource/rfc/sync/manifest.py +162 -0
  238. primarysource/rfc/sync/scanner.py +49 -0
  239. primarysource/rfc/sync/state.py +464 -0
  240. primarysource/rfc/views.py +1067 -0
  241. primarysource/rpmdocs/__init__.py +15 -0
  242. primarysource/rpmdocs/catalogue.py +117 -0
  243. primarysource/rpmdocs/cli.py +12 -0
  244. primarysource/rpmdocs/server.py +8 -0
  245. primarysource/scaffold.py +223 -0
  246. primarysource/serve.py +75 -0
  247. primarysource/settings.py +152 -0
  248. primarysource/usb/__init__.py +5 -0
  249. primarysource/usb/collection.py +37 -0
  250. primarysource/usb/data/acquired.json +2025 -0
  251. primarysource/usb/data/sources-usb.toml +852 -0
  252. primarysource/usb/data/sources.toml +10 -0
  253. primarysource/usb/data/usb-catalog.json +1663 -0
  254. primarysource/usb/data/usb-classification.json +1986 -0
  255. primarysource/usb/data/usb-corpus-build.json +608 -0
  256. primarysource/usb/data/usb-pdf-choice.json +839 -0
  257. primarysource/usb/data/usb-split-measure.json +1803 -0
  258. primarysource/usb/ingest/__init__.py +1 -0
  259. primarysource/usb/ingest/acquire.py +130 -0
  260. primarysource/usb/ingest/discover_usb.py +88 -0
  261. primarysource/usb/ingest/extract.py +48 -0
  262. primarysource/usb/ingest/fetch.py +197 -0
  263. primarysource/usb/ingest/index.py +14 -0
  264. primarysource/usb/ingest/sources.py +85 -0
  265. primarysource/usb/models.py +7 -0
  266. primarysource/usb/paths.py +33 -0
  267. primarysource/usb/server/__init__.py +3 -0
  268. primarysource/usb/server/corpus.py +38 -0
  269. primarysource/usb/server/mcp.py +19 -0
  270. primarysource/usb/server/tools.py +35 -0
  271. primarysource/usb/spec.py +20 -0
  272. primarysource/usbids/__init__.py +56 -0
  273. primarysource/usbids/cli.py +65 -0
  274. primarysource/usbids/importer.py +272 -0
  275. primarysource/usbids/parse.py +283 -0
  276. primarysource/usbids/server.py +55 -0
  277. primarysource/usbids/store.py +80 -0
  278. primarysource/usbids/tools.py +174 -0
  279. primarysource/zim/__init__.py +5 -0
  280. primarysource/zim/__main__.py +8 -0
  281. primarysource/zim/_libzim.py +37 -0
  282. primarysource/zim/catalog.py +594 -0
  283. primarysource/zim/cli.py +555 -0
  284. primarysource/zim/collection.py +41 -0
  285. primarysource/zim/errors.py +100 -0
  286. primarysource/zim/groups.py +30 -0
  287. primarysource/zim/library.py +668 -0
  288. primarysource/zim/manage.py +636 -0
  289. primarysource/zim/markdown.py +1247 -0
  290. primarysource/zim/mcp.py +201 -0
  291. primarysource/zim/purl.py +130 -0
  292. primarysource/zim/py.typed +0 -0
  293. primarysource/zim/refs.py +90 -0
  294. primarysource/zim/service.py +1182 -0
  295. primarysource/zim/snapshot.py +197 -0
  296. primarysource/zim/sync.py +269 -0
  297. primarysource/zim/views.py +230 -0
  298. primarysource-0.1.0.dist-info/METADATA +118 -0
  299. primarysource-0.1.0.dist-info/RECORD +303 -0
  300. primarysource-0.1.0.dist-info/WHEEL +4 -0
  301. primarysource-0.1.0.dist-info/entry_points.txt +2 -0
  302. primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
  303. primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
@@ -0,0 +1,555 @@
1
+ """zimlore: a Kiwix library (ZIM files) as token-budgeted, citable documentation.
2
+
3
+ primarysource zim info where the library is, what is installed, whether libzim is available
4
+ primarysource zim list [name] installed libraries (or those matching a name), with edition dates
5
+ primarysource zim resolve flask the one library a name means, or the candidates when it is ambiguous
6
+ primarysource zim search "blueprint" -l flask search one library; without -l, search every library
7
+ primarysource zim read flask§blueprints/index#why-blueprints
8
+ primarysource zim outline python§library/os ; primarysource zim outline python --prefix library/a
9
+ primarysource zim links flask§blueprints/index ; primarysource zim cite flask§blueprints/index
10
+ primarysource zim catalog flask search the public Kiwix catalogue (network)
11
+ primarysource zim catalog --refresh store the catalogue as an offline snapshot (network)
12
+ primarysource zim catalog --offline [flask] read the snapshot; ranks like `available` (no network)
13
+ primarysource zim available flask what could be installed, from the snapshot (no network)
14
+ primarysource zim get devdocs_en_flask [--replace] download a ZIM, verified by SHA-256 (network)
15
+ primarysource zim outdated installed ZIMs with a newer edition in the catalogue (network)
16
+ primarysource zim add PATH [--link] [--force] install a local ZIM, checked by libzim, with a provenance sidecar
17
+ primarysource zim sync [--manifest F] [--group G ...] [--dry-run] [--update] match a manifest (network)
18
+ primarysource zim verify [name] [--adopt] check files against their provenance sidecar and libzim's checksum
19
+ primarysource zim remove NAME [--edition D | --all] [--yes] delete editions (dry run without --yes)
20
+ primarysource zim prune [--keep N] [--include-unmanaged] [--yes] drop old editions (dry run without --yes)
21
+ primarysource zim du disk use per library
22
+ primarysource zim mcp serve the tools over stdio (needs the mcp extra)
23
+
24
+ A reference is `library§path[#anchor][¶n]`; `::` is the ASCII spelling of `§`. Every command takes `--json`.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import argparse
30
+ import json
31
+ import sys
32
+ import time
33
+ from collections.abc import Callable
34
+ from pathlib import Path
35
+ from typing import Any
36
+
37
+ from primarysource.errors import LibError
38
+
39
+ from . import __version__, _libzim, catalog, groups, library, manage, service, snapshot, sync, views
40
+ from .errors import ZimloreError
41
+
42
+
43
+ def _print_json(data: object) -> None:
44
+ print(json.dumps(data, indent=2, ensure_ascii=False))
45
+
46
+
47
+ def _emit(args: argparse.Namespace, data: dict[str, Any], human: Callable[[dict[str, Any]], None]) -> int:
48
+ if args.json:
49
+ _print_json(data)
50
+ else:
51
+ human(data)
52
+ return 0
53
+
54
+
55
+ def _fmt_freshness(row: dict[str, Any]) -> str:
56
+ fresh = row["freshness"]
57
+ if fresh["age_days"] is None:
58
+ return "undated"
59
+ return f"{fresh['date']} ({fresh['age_days']} days old{', STALE' if fresh['stale'] else ''})"
60
+
61
+
62
+ def cmd_info(args: argparse.Namespace) -> int:
63
+ scanned = library.scan()
64
+ data = {
65
+ "version": __version__,
66
+ "home": str(library.home()),
67
+ "zim_directories": [str(d) for d in scanned.directories],
68
+ "libzim": _libzim.available(),
69
+ "installed": len(scanned.books),
70
+ "problems": scanned.problems,
71
+ }
72
+
73
+ def human(d: dict[str, Any]) -> None:
74
+ print(
75
+ f"primarysource zim {d['version']}\nhome: {d['home']}\nZIM directories: {', '.join(d['zim_directories'])}"
76
+ )
77
+ print(f"libzim: {'available' if d['libzim'] else 'MISSING - ' + _libzim.REMEDY}")
78
+ print(f"installed: {d['installed']}")
79
+ for problem in d["problems"]:
80
+ print(f" unreadable: {problem['path']}: {problem['error']}")
81
+
82
+ return _emit(args, data, human)
83
+
84
+
85
+ def cmd_list(args: argparse.Namespace) -> int:
86
+ data = service.libraries(args.name or "", args.limit, args.verbose, args.all_versions, args.start)
87
+ return _emit(args, data, lambda d: print(views.libraries(d), end=""))
88
+
89
+
90
+ def cmd_resolve(args: argparse.Namespace) -> int:
91
+ return _emit(args, service.resolve(args.name), lambda d: print(views.resolve(d), end=""))
92
+
93
+
94
+ def cmd_search(args: argparse.Namespace) -> int:
95
+ data = service.search(args.query, args.library, args.limit, args.max_tokens, args.language, args.start)
96
+ return _emit(args, data, lambda d: print(views.search(d), end=""))
97
+
98
+
99
+ def cmd_read(args: argparse.Namespace) -> int:
100
+ data = service.read(args.ref, args.library, args.max_tokens, args.format)
101
+ return _emit(args, data, lambda d: print(views.read(d), end=""))
102
+
103
+
104
+ def cmd_outline(args: argparse.Namespace) -> int:
105
+ data = service.outline(args.ref, args.library, args.start, args.limit, args.prefix)
106
+ return _emit(args, data, lambda d: print(views.outline(d), end=""))
107
+
108
+
109
+ def cmd_links(args: argparse.Namespace) -> int:
110
+ data = service.links(args.ref, args.library, args.limit, start=args.start)
111
+ return _emit(args, data, lambda d: print(views.links(d), end=""))
112
+
113
+
114
+ def cmd_cite(args: argparse.Namespace) -> int:
115
+ return _emit(args, service.cite(args.ref, args.library), lambda d: print(views.cite(d), end=""))
116
+
117
+
118
+ def cmd_available(args: argparse.Namespace) -> int:
119
+ data = service.available(args.query, args.limit, args.start)
120
+ return _emit(args, data, lambda d: print(views.available(d), end=""))
121
+
122
+
123
+ def _catalog_refresh(args: argparse.Namespace) -> int:
124
+ started = time.monotonic()
125
+ data = snapshot.refresh(language=args.lang)
126
+ data["seconds"] = round(time.monotonic() - started, 1)
127
+
128
+ def human(d: dict[str, Any]) -> None:
129
+ print(
130
+ f"snapshot written: {d['entries']} entries of {d['total_results']} in {d['pages']} page(s), "
131
+ f"{d['bytes'] / 1e6:.2f} MB, {d['seconds']} s -> {d['path']}"
132
+ )
133
+
134
+ return _emit(args, data, human)
135
+
136
+
137
+ def _catalog_offline(args: argparse.Namespace) -> int:
138
+ if args.query:
139
+ return _emit(args, service.available(args.query, args.count), lambda d: print(views.available(d), end=""))
140
+ entries, fetched_at, age = snapshot.load()
141
+ data = {"entries": len(entries), "fetched_at": fetched_at, "age_days": age, "path": str(snapshot.path())}
142
+
143
+ def human(d: dict[str, Any]) -> None:
144
+ print(f"snapshot: {d['entries']} entries, fetched {d['fetched_at']} ({d['age_days']} days old), {d['path']}")
145
+
146
+ return _emit(args, data, human)
147
+
148
+
149
+ def cmd_catalog(args: argparse.Namespace) -> int:
150
+ if args.refresh and args.offline:
151
+ raise ZimloreError("--refresh uses the network and --offline forbids it; choose one")
152
+ if args.refresh:
153
+ return _catalog_refresh(args)
154
+ if args.offline:
155
+ return _catalog_offline(args)
156
+ if not (args.query or args.tag):
157
+ raise ZimloreError(
158
+ "give a search query or --tag",
159
+ "e.g. primarysource zim catalog flask ; primarysource zim catalog --tag devdocs",
160
+ )
161
+ entries, total = catalog.query(
162
+ args.query or "",
163
+ language=args.lang,
164
+ count=args.count,
165
+ tag=args.tag,
166
+ max_bytes=int(args.max_mb * 1_000_000),
167
+ )
168
+ data = {
169
+ "total": total,
170
+ "entries": [
171
+ {
172
+ "name": e.name,
173
+ "title": e.title,
174
+ "date": e.date,
175
+ "approx_mb": round(e.approx_bytes / 1e6, 1),
176
+ "articles": e.articles,
177
+ "summary": e.summary,
178
+ }
179
+ for e in catalog.latest(entries)
180
+ ],
181
+ }
182
+
183
+ def human(d: dict[str, Any]) -> None:
184
+ for row in d["entries"]:
185
+ print(
186
+ f"{row['name']} {row['title']} [{row['date']}; ~{row['approx_mb']} MB; {row['articles']} articles]"
187
+ )
188
+ print(f"({len(d['entries'])} shown of {d['total']} catalogue entries; primarysource zim get <name>)")
189
+
190
+ return _emit(args, data, human)
191
+
192
+
193
+ def cmd_get(args: argparse.Namespace) -> int:
194
+ found = catalog.latest(catalog.editions_of(args.name))
195
+ if not found:
196
+ raise ZimloreError(
197
+ f"the catalogue has no ZIM named {args.name!r}", "primarysource zim catalog <words> finds names"
198
+ )
199
+ entry = found[0]
200
+ directory = library.zim_dirs()[0]
201
+ print(f"{entry.name} {entry.date} (~{entry.approx_bytes / 1e6:.1f} MB) -> {directory}", file=sys.stderr)
202
+ last = [0]
203
+
204
+ def progress(done: int, total: int) -> None:
205
+ if done - last[0] >= 5_000_000 or done == total:
206
+ last[0] = done
207
+ print(f" {done / 1e6:.1f} / {total / 1e6:.1f} MB", file=sys.stderr)
208
+
209
+ result = catalog.download_result(entry, directory, progress=progress, replace=args.replace)
210
+ book = library.read_book(result.path)
211
+ if result.status == "already-installed":
212
+ print(f"already installed, identical to the catalogue's file (SHA-256 matches): {result.path}")
213
+ else:
214
+ verb = "replaced" if result.status == "replaced" else "installed"
215
+ print(f"verified (SHA-256) and {verb}: {result.path}")
216
+ print(f"licence: {service.licence_note(book)}")
217
+ return 0
218
+
219
+
220
+ def cmd_add(args: argparse.Namespace) -> int:
221
+ data = manage.add_local(Path(args.path), link=args.link, force=args.force)
222
+
223
+ def human(d: dict[str, Any]) -> None:
224
+ if d["status"] == "already-installed":
225
+ print(f"already installed (identical file): {d['file']}")
226
+ else:
227
+ how = {"copy": "copied", "hardlink": "hard-linked", "symlink": "symbolic-linked"}[d["mode"]]
228
+ print(f"{how} and recorded (SHA-256 computed locally): {d['file']}")
229
+ if d["forced"]:
230
+ print("WARNING: libzim's checksum fails; installed because of --force. verify will flag it")
231
+ print(f"licence: {service.licence_note(library.read_book(Path(d['file'])))}")
232
+
233
+ return _emit(args, data, human)
234
+
235
+
236
+ def _print_plan(plan: sync.Plan) -> None:
237
+ print(f"manifest: {plan.manifest or 'none (groups from the command line)'}")
238
+ for item in plan.items:
239
+ size = f" ~{_mb(item.approx_bytes)}" if item.approx_bytes else ""
240
+ have = f" installed {item.installed}" if item.installed else ""
241
+ age = " OVER max_age_days" if item.over_max_age else ""
242
+ note = f" ({item.note})" if item.note else ""
243
+ print(f"{item.action:11} {item.name} catalogue {item.available or '-'}{have}{size}{age}{note}")
244
+ for extra in plan.extras:
245
+ print(f"extra {extra['name']} installed, not in the manifest; never removed by sync")
246
+ print(
247
+ f"to download: ~{_mb(plan.download_bytes)} "
248
+ "(approximate, from the catalogue's rounded lengths; the .meta4 has the true size)"
249
+ )
250
+ if plan.budget is not None:
251
+ verdict = "EXCEEDS the budget" if plan.over_budget else "within the budget"
252
+ total = plan.installed_bytes + plan.download_bytes
253
+ print(f"budget: {_mb(plan.budget)}; library after sync ~{_mb(total)}: {verdict}")
254
+
255
+
256
+ def cmd_sync(args: argparse.Namespace) -> int:
257
+ path = sync.find_manifest(args.manifest)
258
+ manifest = sync.load_manifest(path) if path else sync.Manifest()
259
+ named = tuple(args.group or ())
260
+ if not (manifest.libraries or manifest.groups or named):
261
+ raise ZimloreError(
262
+ "nothing to sync: " + (f"{path} names no libraries or groups" if path else "no manifest and no --group"),
263
+ f"write ./{sync.MANIFEST_NAME} or {library.home() / sync.HOME_MANIFEST} "
264
+ '(libraries = ["devdocs_en_python"]), or run: '
265
+ f"primarysource zim sync --group {sorted(groups.GROUPS)[0]} --dry-run",
266
+ )
267
+ plan = sync.make_plan(manifest, named, update=args.update)
268
+ results: list[dict[str, Any]] = []
269
+ if args.json:
270
+ if not args.dry_run:
271
+ results = sync.execute(plan)
272
+ _print_json({**plan.as_dict(), "dry_run": args.dry_run, "results": results})
273
+ else:
274
+ _print_plan(plan)
275
+ if not args.dry_run:
276
+ last: dict[str, int] = {}
277
+
278
+ def progress(name: str, done: int, total: int) -> None:
279
+ if done - last.get(name, 0) >= 5_000_000 or done == total:
280
+ last[name] = done
281
+ print(f" {name}: {done / 1e6:.1f} / {total / 1e6:.1f} MB", file=sys.stderr)
282
+
283
+ results = sync.execute(plan, progress=progress)
284
+ for row in results:
285
+ print(f"{'installed' if row['ok'] else 'FAILED'} {row['name']} {row.get('file') or row['error']}")
286
+ else:
287
+ print("dry run: nothing was downloaded or changed")
288
+ failed = [r for r in results if not r["ok"]]
289
+ return 1 if failed or plan.problems or (args.dry_run and plan.over_budget) else 0
290
+
291
+
292
+ def cmd_outdated(args: argparse.Namespace) -> int:
293
+ scanned = library.scan()
294
+ rows = []
295
+ for name in sorted({b.name for b in scanned.books}):
296
+ mine = library.newest([b for b in scanned.books if b.name == name])
297
+ if mine is None:
298
+ continue
299
+ newest = catalog.latest(catalog.editions_of(name))
300
+ rows.append(
301
+ {
302
+ "name": name,
303
+ "installed": mine.date,
304
+ "catalogue": newest[0].date if newest else None,
305
+ "outdated": library.is_newer(newest[0].date, mine.date) if newest else False,
306
+ }
307
+ )
308
+
309
+ def human(d: dict[str, Any]) -> None:
310
+ for row in d["rows"]:
311
+ state = (
312
+ "NEWER AVAILABLE"
313
+ if row["outdated"]
314
+ else (
315
+ "not in catalogue"
316
+ if row["catalogue"] is None
317
+ else ("date unknown, cannot compare" if row["outdated"] is None else "current")
318
+ )
319
+ )
320
+ print(f"{row['name']} installed {row['installed']} catalogue {row['catalogue']} {state}")
321
+
322
+ return _emit(args, {"rows": rows}, human)
323
+
324
+
325
+ def _mb(size: int) -> str:
326
+ return f"{size / 1e6:.1f} MB"
327
+
328
+
329
+ def cmd_verify(args: argparse.Namespace) -> int:
330
+ data = manage.verify(args.name or "", hash_files=not args.no_hash, adopt=args.adopt)
331
+
332
+ def human(d: dict[str, Any]) -> None:
333
+ for row in d["files"]:
334
+ extra = row.get("why") or row.get("note") or row.get("provenance_problem") or ""
335
+ print(f"{row['status']:10} {row['file']} {extra}".rstrip())
336
+ for problem in d["problems"]:
337
+ print(f"unreadable {problem['path']}: {problem['error']}")
338
+ unrecorded = sum(r["status"] == "unrecorded" for r in d["files"])
339
+ if unrecorded:
340
+ print(
341
+ f"({unrecorded} file(s) have no provenance record; --adopt records what they are, not their origin)"
342
+ )
343
+
344
+ _emit(args, data, human)
345
+ return 1 if data["failed"] else 0
346
+
347
+
348
+ def _report_deletion(data: dict[str, Any]) -> None:
349
+ if data["applied"]:
350
+ for item in data["deleted"]:
351
+ gone = " (already gone)" if item.get("already_gone") else ""
352
+ print(f"deleted {item['file']} ({_mb(item['bytes'])}){gone}")
353
+ for item in data["failed"]:
354
+ print(f"FAILED {item['file']}: {item['error']}")
355
+ if item["hint"]:
356
+ print(f" fix: {item['hint']}")
357
+ else:
358
+ for item in data["files"]:
359
+ print(f"would delete {item['file']} ({_mb(item['bytes'])})")
360
+ for item in data["refused"]:
361
+ print(f"REFUSED {item['name']}: {item['reason']}")
362
+ for path in item["files"]:
363
+ print(f" {path}")
364
+ for item in data["skipped"]:
365
+ print(f"kept {item['file']} ({item['reason']})")
366
+ verb = "freed" if data["applied"] else "would free"
367
+ print(f"{verb} {_mb(data['freed'] if data['applied'] else data['bytes'])}")
368
+ if data["note"]:
369
+ print(f"({data['note']})")
370
+
371
+
372
+ def _deletion_exit(data: dict[str, Any]) -> int:
373
+ """1 when a file could not be deleted or a library was refused under --yes: partial work is never exit 0."""
374
+ return 0 if data["ok"] else 1
375
+
376
+
377
+ def cmd_remove(args: argparse.Namespace) -> int:
378
+ data = manage.remove(
379
+ args.name, edition=args.edition, all_editions=args.all, all_versions=args.all_versions, apply=args.yes
380
+ )
381
+ _emit(args, data, _report_deletion)
382
+ return _deletion_exit(data)
383
+
384
+
385
+ def cmd_prune(args: argparse.Namespace) -> int:
386
+ data = manage.prune(keep=args.keep, apply=args.yes, include_unmanaged=args.include_unmanaged)
387
+ _emit(args, data, _report_deletion)
388
+ return _deletion_exit(data)
389
+
390
+
391
+ def cmd_du(args: argparse.Namespace) -> int:
392
+ data = manage.du()
393
+
394
+ def human(d: dict[str, Any]) -> None:
395
+ for row in d["libraries"]:
396
+ newest = row["newest"] or "undated"
397
+ print(f"{_mb(row['bytes']):>10} {row['name']} ({row['editions']} edition(s), newest {newest})")
398
+ print(f"{_mb(d['total_bytes']):>10} total in {', '.join(d['directories'])}")
399
+ if d["partial_bytes"]:
400
+ print(f"{_mb(d['partial_bytes']):>10} partial downloads (primarysource zim prune --yes removes them)")
401
+ if d["free_bytes"] is not None:
402
+ print(f"{_mb(d['free_bytes']):>10} free on the volume")
403
+
404
+ return _emit(args, data, human)
405
+
406
+
407
+ def cmd_mcp(_args: argparse.Namespace) -> int:
408
+ from . import mcp
409
+
410
+ try:
411
+ mcp.require_extra()
412
+ except ImportError as error:
413
+ print(error, file=sys.stderr)
414
+ return 2
415
+ mcp.main()
416
+ return 0
417
+
418
+
419
+ def build_parser() -> argparse.ArgumentParser:
420
+ parser = argparse.ArgumentParser(
421
+ prog="primarysource zim", description=__doc__, formatter_class=argparse.RawTextHelpFormatter
422
+ )
423
+ parser.add_argument("--version", action="version", version=f"primarysource zim {__version__}")
424
+ sub = parser.add_subparsers(dest="command", required=True)
425
+
426
+ def add(name: str, handler: Callable[[argparse.Namespace], int], help_text: str, json_flag: bool = True) -> Any:
427
+ command = sub.add_parser(name, help=help_text, description=help_text)
428
+ command.set_defaults(handler=handler)
429
+ if json_flag:
430
+ command.add_argument("--json", action="store_true", help="print the structured result")
431
+ return command
432
+
433
+ add("info", cmd_info, "where the library is and whether libzim is available")
434
+ listing = add("list", cmd_list, "installed libraries, with edition dates")
435
+ listing.add_argument("name", nargs="?")
436
+ listing.add_argument("--limit", type=int, default=50, help="at most this many libraries (default 50)")
437
+ listing.add_argument("--verbose", action="store_true", help="add description, creator, tags, licence, provenance")
438
+ listing.add_argument("--all-versions", action="store_true", help="one row per ZIM Name instead of per package")
439
+ listing.add_argument("--start", type=int, default=0, help="first library to show (the `next: start=` value)")
440
+ add("resolve", cmd_resolve, "the one library a name means").add_argument("name")
441
+
442
+ search = add("search", cmd_search, "search a library, or all libraries")
443
+ search.add_argument("query")
444
+ search.add_argument("--library", "-l", default="", help="restrict to one library (any alias); default: all")
445
+ search.add_argument("--limit", type=int, default=8)
446
+ search.add_argument("--max-tokens", type=int, default=1500)
447
+ search.add_argument("--start", type=int, default=0, help="first hit to show (the `next: start=` value)")
448
+ search.add_argument(
449
+ "--language", default="", help="'' ranks translations last, 'any' keeps engine order, 'ru' only that language"
450
+ )
451
+
452
+ read = add("read", cmd_read, "read an article or section (#anchor), from block ¶n on, paged by token budget")
453
+ read.add_argument("ref")
454
+ read.add_argument("--library", default="", help="the library, when ref is a bare path")
455
+ read.add_argument("--max-tokens", type=int, default=2000)
456
+ read.add_argument("--format", choices=["markdown", "text"], default="markdown")
457
+
458
+ outline = add("outline", cmd_outline, "headings of an article, or the articles of a library")
459
+ outline.add_argument("ref")
460
+ outline.add_argument("--library", default="")
461
+ outline.add_argument("--start", type=int, default=0)
462
+ outline.add_argument("--limit", type=int, default=100)
463
+ outline.add_argument("--prefix", default="", help="only entries whose path starts with this")
464
+
465
+ links = add("links", cmd_links, "links out of an article, resolved to refs")
466
+ links.add_argument("ref")
467
+ links.add_argument("--library", default="")
468
+ links.add_argument("--limit", type=int, default=100)
469
+ links.add_argument("--start", type=int, default=0, help="first link to show (the `next: start=` value)")
470
+
471
+ cite = add("cite", cmd_cite, "the citation string, edition and licence note for a ref")
472
+ cite.add_argument("ref")
473
+ cite.add_argument("--library", default="")
474
+
475
+ catalogue = add("catalog", cmd_catalog, "search the public Kiwix catalogue (network)")
476
+ catalogue.add_argument("query", nargs="?")
477
+ catalogue.add_argument("--lang", default="eng", help="ISO 639-3 code, or '' for any")
478
+ catalogue.add_argument("--count", type=int, default=20)
479
+ catalogue.add_argument("--tag", default="", help="only ZIMs with this tag, e.g. devdocs or stackoverflow")
480
+ catalogue.add_argument("--max-mb", type=float, default=0, help="only ZIMs of at most this many megabytes")
481
+ catalogue.add_argument(
482
+ "--refresh", action="store_true", help="fetch the whole catalogue into the offline snapshot"
483
+ )
484
+ catalogue.add_argument("--offline", action="store_true", help="read the snapshot; never touch the network")
485
+ avail = add("available", cmd_available, "what could be installed, from the offline snapshot (no network)")
486
+ avail.add_argument("query")
487
+ avail.add_argument("--limit", type=int, default=8)
488
+ avail.add_argument("--start", type=int, default=0, help="first match to show (the `next: start=` value)")
489
+
490
+ get = add("get", cmd_get, "download a ZIM from the catalogue, verified by SHA-256 (network)")
491
+ get.add_argument("name")
492
+ get.add_argument(
493
+ "--replace", action="store_true", help="replace an existing file of the same name (otherwise it is refused)"
494
+ )
495
+ add_cmd = add(
496
+ "add", cmd_add, "install a local ZIM file into the library, checked first, with a provenance sidecar"
497
+ )
498
+ add_cmd.add_argument("path")
499
+ add_cmd.add_argument("--link", action="store_true", help="hard-link (else symlink) instead of copying")
500
+ add_cmd.add_argument("--force", action="store_true", help="install even though libzim's checksum fails")
501
+
502
+ sync_cmd = add("sync", cmd_sync, "install the libraries a manifest names; never removes anything (network)")
503
+ sync_cmd.add_argument(
504
+ "--manifest", default="", help=f"default: ./{sync.MANIFEST_NAME}, then $ZIMLORE_HOME/libraries.toml"
505
+ )
506
+ sync_cmd.add_argument(
507
+ "--group", nargs="+", action="extend", help=f"named sets: {', '.join(sorted(groups.GROUPS))}"
508
+ )
509
+ sync_cmd.add_argument("--dry-run", action="store_true", help="print the plan and sizes; download nothing")
510
+ sync_cmd.add_argument("--update", action="store_true", help="also fetch newer editions, beside the installed one")
511
+ add("outdated", cmd_outdated, "installed ZIMs that have a newer catalogue edition (network)")
512
+ verify = add("verify", cmd_verify, "check installed files against their provenance and libzim's checksum")
513
+ verify.add_argument("name", nargs="?")
514
+ verify.add_argument("--no-hash", action="store_true", help="compare sizes only; do not recompute SHA-256")
515
+ verify.add_argument("--adopt", action="store_true", help="record a provenance sidecar for files that lack one")
516
+
517
+ remove = add("remove", cmd_remove, "delete installed editions of a library (dry run unless --yes)")
518
+ remove.add_argument("name")
519
+ remove.add_argument("--edition", default="", help="only this edition date")
520
+ remove.add_argument("--all", action="store_true", help="every edition")
521
+ remove.add_argument("--all-versions", action="store_true", help="every version of a package, all editions")
522
+ remove.add_argument("--yes", action="store_true", help="actually delete")
523
+
524
+ prune = add("prune", cmd_prune, "delete old editions and partial downloads (dry run unless --yes)")
525
+ prune.add_argument("--keep", type=int, default=2, help="editions to keep per library (default 2)")
526
+ prune.add_argument(
527
+ "--include-unmanaged",
528
+ action="store_true",
529
+ help="also delete files this library did not install (no get/sync/add sidecar: a shared Kiwix library)",
530
+ )
531
+ prune.add_argument("--yes", action="store_true", help="actually delete")
532
+
533
+ add("du", cmd_du, "disk use per library, partial downloads, free space")
534
+ add("mcp", cmd_mcp, "serve the MCP tools over stdio (needs the mcp extra)", json_flag=False)
535
+ return parser
536
+
537
+
538
+ def main(argv: list[str] | None = None) -> int:
539
+ # ZIM text is Unicode throughout (the reference grammar itself uses § and ¶), and a Windows console's code
540
+ # page cannot print it: a documentation tool must not die on a curly quote.
541
+ sys.stdout.reconfigure(encoding="utf-8", errors="replace") # type: ignore[union-attr]
542
+ args = build_parser().parse_args(argv)
543
+ try:
544
+ return int(args.handler(args))
545
+ except LibError as error:
546
+ if not isinstance(error, ZimloreError):
547
+ print(error.as_text(), file=sys.stderr)
548
+ return 2
549
+ if getattr(args, "json", False):
550
+ _print_json(error.as_dict())
551
+ else:
552
+ print(error.cli_text(), file=sys.stderr)
553
+ if error.detail():
554
+ print(error.detail(), file=sys.stderr)
555
+ return 2
@@ -0,0 +1,41 @@
1
+ """A Kiwix library (ZIM files) as a primarysource collection. Importing this needs nothing beyond the core: the
2
+ native `libzim` binding is imported only when a ZIM is opened (`_libzim.load`)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from primarysource.errors import LibError
7
+ from primarysource.model import Unit
8
+ from primarysource.refs import Ref
9
+ from primarysource.registry import Collection
10
+
11
+ from . import refs
12
+ from .errors import ZimloreError
13
+
14
+
15
+ class ZimRefs:
16
+ """zimlore's refs (`devdocs_en_flask§blueprints/index#why-blueprints¶3`) as the core's `Ref`: the library is the
17
+ `doc`, the entry path the `section`. A path is a slash-separated URL path, not a dotted number, so the core's
18
+ `RefGrammar` cannot read it; `refs.parse_ref` can."""
19
+
20
+ @staticmethod
21
+ def parse(text: str) -> Ref:
22
+ try:
23
+ parsed = refs.parse_ref(text)
24
+ except ZimloreError as exc:
25
+ raise LibError(exc.message, exc.hint) from exc
26
+ return Ref(parsed.book, parsed.path or None, parsed.para, parsed.anchor or None)
27
+
28
+ @staticmethod
29
+ def format(ref: Ref) -> str:
30
+ return refs.format_ref(ref.doc or "", ref.section or "", ref.anchor or "", ref.para)
31
+
32
+
33
+ COLLECTION = Collection(
34
+ name="zim",
35
+ title="Kiwix libraries (ZIM files): DevDocs, Stack Exchange, language and tool documentation",
36
+ prefix="zim_",
37
+ unit=Unit.BLOCK,
38
+ grammar=ZimRefs(),
39
+ server="primarysource.zim.mcp:build",
40
+ cli="primarysource.zim.cli:main",
41
+ )