primarysource 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- primarysource/__init__.py +5 -0
- primarysource/bbf/__init__.py +56 -0
- primarysource/bbf/_xml.py +63 -0
- primarysource/bbf/catalogue.json +777 -0
- primarysource/bbf/catalogue.py +137 -0
- primarysource/bbf/cli.py +79 -0
- primarysource/bbf/datamodel.py +281 -0
- primarysource/bbf/importer.py +151 -0
- primarysource/bbf/library.py +380 -0
- primarysource/bbf/server.py +72 -0
- primarysource/budget.py +63 -0
- primarysource/c2sp/__init__.py +53 -0
- primarysource/c2sp/_fetch.py +5 -0
- primarysource/c2sp/catalogue.json +231 -0
- primarysource/c2sp/catalogue.py +95 -0
- primarysource/c2sp/cli.py +62 -0
- primarysource/c2sp/importer.py +90 -0
- primarysource/c2sp/library.py +309 -0
- primarysource/c2sp/markdown.py +31 -0
- primarysource/c2sp/server.py +68 -0
- primarysource/cavp/__init__.py +42 -0
- primarysource/cavp/cli.py +84 -0
- primarysource/cavp/data/vectors.toml +47 -0
- primarysource/cavp/importer.py +156 -0
- primarysource/cavp/model.py +279 -0
- primarysource/cavp/parse.py +269 -0
- primarysource/cavp/paths.py +17 -0
- primarysource/cavp/server.py +67 -0
- primarysource/cavp/tools.py +169 -0
- primarysource/cli.py +107 -0
- primarysource/cursor.py +82 -0
- primarysource/cve/__init__.py +5 -0
- primarysource/cve/__main__.py +6 -0
- primarysource/cve/cli.py +144 -0
- primarysource/cve/collection.py +20 -0
- primarysource/cve/config.py +58 -0
- primarysource/cve/errors.py +16 -0
- primarysource/cve/kernel.py +232 -0
- primarysource/cve/mcp.py +139 -0
- primarysource/cve/parse.py +318 -0
- primarysource/cve/py.typed +0 -0
- primarysource/cve/service.py +760 -0
- primarysource/cve/store.py +214 -0
- primarysource/cve/sync.py +438 -0
- primarysource/cve/views.py +304 -0
- primarysource/debpolicy/__init__.py +16 -0
- primarysource/debpolicy/catalogue.py +263 -0
- primarysource/debpolicy/cli.py +12 -0
- primarysource/debpolicy/server.py +11 -0
- primarysource/draft/__init__.py +11 -0
- primarysource/draft/catalogue.py +117 -0
- primarysource/draft/cli.py +149 -0
- primarysource/draft/collection.py +21 -0
- primarysource/draft/datatracker.py +158 -0
- primarysource/draft/importer.py +163 -0
- primarysource/draft/licence.py +89 -0
- primarysource/draft/names.py +64 -0
- primarysource/draft/net.py +133 -0
- primarysource/draft/segment.py +192 -0
- primarysource/draft/server.py +91 -0
- primarysource/draft/status.py +146 -0
- primarysource/draft/store.py +199 -0
- primarysource/draft/tracked.json +104 -0
- primarysource/draft/views.py +430 -0
- primarysource/draft/xmlmeta.py +74 -0
- primarysource/ecma/__init__.py +52 -0
- primarysource/ecma/_pdf.py +143 -0
- primarysource/ecma/catalogue.json +22 -0
- primarysource/ecma/catalogue.py +100 -0
- primarysource/ecma/cli.py +67 -0
- primarysource/ecma/identify.py +25 -0
- primarysource/ecma/importer.py +108 -0
- primarysource/ecma/library.py +240 -0
- primarysource/ecma/server.py +67 -0
- primarysource/errors.py +29 -0
- primarysource/fetch.py +360 -0
- primarysource/fsutil.py +150 -0
- primarysource/gitdocs/__init__.py +15 -0
- primarysource/gitdocs/catalogue.py +116 -0
- primarysource/gitdocs/cli.py +12 -0
- primarysource/gitdocs/server.py +8 -0
- primarysource/home.py +65 -0
- primarysource/iana/__init__.py +32 -0
- primarysource/iana/catalogue.json +244 -0
- primarysource/iana/catalogue.py +69 -0
- primarysource/iana/cli.py +69 -0
- primarysource/iana/importer.py +157 -0
- primarysource/iana/model.py +325 -0
- primarysource/iana/reader.py +315 -0
- primarysource/iana/server.py +67 -0
- primarysource/iana/store.py +83 -0
- primarysource/iana/xmlsafe.py +47 -0
- primarysource/identify.py +186 -0
- primarysource/ieee/__init__.py +58 -0
- primarysource/ieee/_html.py +49 -0
- primarysource/ieee/catalogue.json +74 -0
- primarysource/ieee/catalogue.py +123 -0
- primarysource/ieee/cli.py +79 -0
- primarysource/ieee/identify.py +24 -0
- primarysource/ieee/ieee802.py +47 -0
- primarysource/ieee/importer.py +305 -0
- primarysource/ieee/posix.py +233 -0
- primarysource/ieee/server.py +79 -0
- primarysource/index.py +446 -0
- primarysource/iso/__init__.py +51 -0
- primarysource/iso/cli.py +98 -0
- primarysource/iso/designation.py +95 -0
- primarysource/iso/identify.py +100 -0
- primarysource/iso/importer.py +224 -0
- primarysource/iso/library.py +249 -0
- primarysource/iso/server.py +69 -0
- primarysource/itu/__init__.py +41 -0
- primarysource/itu/__main__.py +8 -0
- primarysource/itu/asn1mod.py +478 -0
- primarysource/itu/catalog.py +472 -0
- primarysource/itu/cli.py +445 -0
- primarysource/itu/collection.py +20 -0
- primarysource/itu/corpus.py +118 -0
- primarysource/itu/errata.py +16 -0
- primarysource/itu/formal.py +565 -0
- primarysource/itu/identify.py +26 -0
- primarysource/itu/index.py +490 -0
- primarysource/itu/mcp.py +474 -0
- primarysource/itu/py.typed +0 -0
- primarysource/itu/sources.py +600 -0
- primarysource/itu/text.py +609 -0
- primarysource/kconfig/__init__.py +16 -0
- primarysource/kconfig/catalogue.py +98 -0
- primarysource/kconfig/cli.py +12 -0
- primarysource/kconfig/server.py +8 -0
- primarysource/model.py +63 -0
- primarysource/nist/__init__.py +5 -0
- primarysource/nist/collection.py +46 -0
- primarysource/nist/data/acquired-nist.json +1657 -0
- primarysource/nist/data/nist-catalog.json +12090 -0
- primarysource/nist/data/sources.toml +1996 -0
- primarysource/nist/identify.py +41 -0
- primarysource/nist/ingest/__init__.py +1 -0
- primarysource/nist/ingest/acquire_nist.py +128 -0
- primarysource/nist/ingest/controls.py +82 -0
- primarysource/nist/ingest/discover_nist.py +218 -0
- primarysource/nist/ingest/extract.py +48 -0
- primarysource/nist/ingest/fetch.py +211 -0
- primarysource/nist/ingest/index.py +14 -0
- primarysource/nist/ingest/oscal.py +480 -0
- primarysource/nist/ingest/prose_layouts.py +236 -0
- primarysource/nist/ingest/sources.py +94 -0
- primarysource/nist/ingest/status.py +255 -0
- primarysource/nist/models.py +7 -0
- primarysource/nist/paths.py +33 -0
- primarysource/nist/server/__init__.py +3 -0
- primarysource/nist/server/corpus.py +38 -0
- primarysource/nist/server/mcp.py +19 -0
- primarysource/nist/server/tools.py +35 -0
- primarysource/nist/spec.py +38 -0
- primarysource/pdfcorpus/__init__.py +16 -0
- primarysource/pdfcorpus/corpus.py +159 -0
- primarysource/pdfcorpus/fts.py +175 -0
- primarysource/pdfcorpus/index.py +70 -0
- primarysource/pdfcorpus/models.py +30 -0
- primarysource/pdfcorpus/server.py +96 -0
- primarysource/pdfcorpus/spec.py +37 -0
- primarysource/pdfcorpus/split.py +391 -0
- primarysource/pdfcorpus/tools.py +244 -0
- primarysource/pep/__init__.py +15 -0
- primarysource/pep/catalogue.py +191 -0
- primarysource/pep/cli.py +12 -0
- primarysource/pep/server.py +8 -0
- primarysource/pkgdocs/__init__.py +19 -0
- primarysource/pkgdocs/fetch.py +356 -0
- primarysource/pkgdocs/index.py +89 -0
- primarysource/pkgdocs/model.py +76 -0
- primarysource/pkgdocs/readers/__init__.py +1 -0
- primarysource/pkgdocs/readers/_common.py +134 -0
- primarysource/pkgdocs/readers/asciidoc.py +160 -0
- primarysource/pkgdocs/readers/latex.py +315 -0
- primarysource/pkgdocs/readers/markdown.py +173 -0
- primarysource/pkgdocs/readers/pod.py +132 -0
- primarysource/pkgdocs/readers/rst.py +239 -0
- primarysource/pkgdocs/readers/scdoc.py +108 -0
- primarysource/pkgdocs/runtime.py +552 -0
- primarysource/pkgdocs/serving.py +158 -0
- primarysource/pkgdocs/store.py +167 -0
- primarysource/pms/__init__.py +16 -0
- primarysource/pms/catalogue.py +88 -0
- primarysource/pms/cli.py +12 -0
- primarysource/pms/server.py +8 -0
- primarysource/py.typed +0 -0
- primarysource/readers/__init__.py +10 -0
- primarysource/readers/bookmarks.py +192 -0
- primarysource/readers/html.py +1312 -0
- primarysource/readers/markdown.py +189 -0
- primarysource/readers/pdf.py +187 -0
- primarysource/readers/xml.py +230 -0
- primarysource/refs.py +87 -0
- primarysource/registry.py +161 -0
- primarysource/render.py +49 -0
- primarysource/rfc/__init__.py +3 -0
- primarysource/rfc/__main__.py +6 -0
- primarysource/rfc/cite.py +204 -0
- primarysource/rfc/cli.py +903 -0
- primarysource/rfc/collection.py +19 -0
- primarysource/rfc/config.py +138 -0
- primarysource/rfc/errata.py +203 -0
- primarysource/rfc/fsutil.py +82 -0
- primarysource/rfc/graph.py +213 -0
- primarysource/rfc/index/__init__.py +1 -0
- primarysource/rfc/index/chunker.py +281 -0
- primarysource/rfc/index/embed.py +166 -0
- primarysource/rfc/index/hybrid.py +305 -0
- primarysource/rfc/index/lexical.py +223 -0
- primarysource/rfc/index/rfcxml.py +673 -0
- primarysource/rfc/index/search.py +508 -0
- primarysource/rfc/index/txtseg.py +585 -0
- primarysource/rfc/library.py +294 -0
- primarysource/rfc/mcp/__init__.py +1 -0
- primarysource/rfc/mcp/server.py +381 -0
- primarysource/rfc/packs/__init__.py +1 -0
- primarysource/rfc/packs/builder.py +228 -0
- primarysource/rfc/packs/collections/all-current-txt.json +10 -0
- primarysource/rfc/packs/collections/essentials.json +19 -0
- primarysource/rfc/packs/collections/fyi-current.json +10 -0
- primarysource/rfc/packs/collections/standards-current.json +10 -0
- primarysource/rfc/packs/collections/year-2025.json +10 -0
- primarysource/rfc/packs/collections/year-2026.json +10 -0
- primarysource/rfc/packs/collections.py +163 -0
- primarysource/rfc/packs/docsetexport.py +163 -0
- primarysource/rfc/packs/epub.py +159 -0
- primarysource/rfc/packs/export.py +306 -0
- primarysource/rfc/packs/htmlexport.py +82 -0
- primarysource/rfc/packs/serve.py +143 -0
- primarysource/rfc/packs/zimexport.py +171 -0
- primarysource/rfc/query.py +466 -0
- primarysource/rfc/subjects.py +457 -0
- primarysource/rfc/sync/__init__.py +1 -0
- primarysource/rfc/sync/downloader.py +132 -0
- primarysource/rfc/sync/manifest.py +162 -0
- primarysource/rfc/sync/scanner.py +49 -0
- primarysource/rfc/sync/state.py +464 -0
- primarysource/rfc/views.py +1067 -0
- primarysource/rpmdocs/__init__.py +15 -0
- primarysource/rpmdocs/catalogue.py +117 -0
- primarysource/rpmdocs/cli.py +12 -0
- primarysource/rpmdocs/server.py +8 -0
- primarysource/scaffold.py +223 -0
- primarysource/serve.py +75 -0
- primarysource/settings.py +152 -0
- primarysource/usb/__init__.py +5 -0
- primarysource/usb/collection.py +37 -0
- primarysource/usb/data/acquired.json +2025 -0
- primarysource/usb/data/sources-usb.toml +852 -0
- primarysource/usb/data/sources.toml +10 -0
- primarysource/usb/data/usb-catalog.json +1663 -0
- primarysource/usb/data/usb-classification.json +1986 -0
- primarysource/usb/data/usb-corpus-build.json +608 -0
- primarysource/usb/data/usb-pdf-choice.json +839 -0
- primarysource/usb/data/usb-split-measure.json +1803 -0
- primarysource/usb/ingest/__init__.py +1 -0
- primarysource/usb/ingest/acquire.py +130 -0
- primarysource/usb/ingest/discover_usb.py +88 -0
- primarysource/usb/ingest/extract.py +48 -0
- primarysource/usb/ingest/fetch.py +197 -0
- primarysource/usb/ingest/index.py +14 -0
- primarysource/usb/ingest/sources.py +85 -0
- primarysource/usb/models.py +7 -0
- primarysource/usb/paths.py +33 -0
- primarysource/usb/server/__init__.py +3 -0
- primarysource/usb/server/corpus.py +38 -0
- primarysource/usb/server/mcp.py +19 -0
- primarysource/usb/server/tools.py +35 -0
- primarysource/usb/spec.py +20 -0
- primarysource/usbids/__init__.py +56 -0
- primarysource/usbids/cli.py +65 -0
- primarysource/usbids/importer.py +272 -0
- primarysource/usbids/parse.py +283 -0
- primarysource/usbids/server.py +55 -0
- primarysource/usbids/store.py +80 -0
- primarysource/usbids/tools.py +174 -0
- primarysource/zim/__init__.py +5 -0
- primarysource/zim/__main__.py +8 -0
- primarysource/zim/_libzim.py +37 -0
- primarysource/zim/catalog.py +594 -0
- primarysource/zim/cli.py +555 -0
- primarysource/zim/collection.py +41 -0
- primarysource/zim/errors.py +100 -0
- primarysource/zim/groups.py +30 -0
- primarysource/zim/library.py +668 -0
- primarysource/zim/manage.py +636 -0
- primarysource/zim/markdown.py +1247 -0
- primarysource/zim/mcp.py +201 -0
- primarysource/zim/purl.py +130 -0
- primarysource/zim/py.typed +0 -0
- primarysource/zim/refs.py +90 -0
- primarysource/zim/service.py +1182 -0
- primarysource/zim/snapshot.py +197 -0
- primarysource/zim/sync.py +269 -0
- primarysource/zim/views.py +230 -0
- primarysource-0.1.0.dist-info/METADATA +118 -0
- primarysource-0.1.0.dist-info/RECORD +303 -0
- primarysource-0.1.0.dist-info/WHEEL +4 -0
- primarysource-0.1.0.dist-info/entry_points.txt +2 -0
- primarysource-0.1.0.dist-info/licenses/LICENSE +203 -0
- primarysource-0.1.0.dist-info/licenses/NOTICE +51 -0
|
@@ -0,0 +1,565 @@
|
|
|
1
|
+
"""ITU-T formal descriptions: a catalogue, a bounded importer, and modules served as documents.
|
|
2
|
+
|
|
3
|
+
ITU publishes the machine-processable parts of its Recommendations (ASN.1 modules, MIBs, XSD, and others) as
|
|
4
|
+
files under `https://www.itu.int/ITU-T/formal-language/`. The web server lists those directories, so the set can
|
|
5
|
+
be enumerated without the Recommendation database. What it does not do is serve every listed file: the `.asn`
|
|
6
|
+
source is listed but answers 404, and the `.html` rendering beside it is what can be fetched (measured
|
|
7
|
+
2026-11-13). The rendering is the source text inside a `<pre>`, one comment line linking to the Recommendation,
|
|
8
|
+
and nothing else, so `asn1mod.unwrap_html` recovers it exactly.
|
|
9
|
+
|
|
10
|
+
Terms: ITU's website terms permit downloading, copying and using content for personal, educational or
|
|
11
|
+
non-commercial purposes with the source acknowledged and notices kept, and forbid modification, reproduction,
|
|
12
|
+
distribution and derivative works without written permission. They say that publications "may be governed by
|
|
13
|
+
specific licensing terms", and no separate licence is stated for this database. So the policy is the one the
|
|
14
|
+
rest of this package keeps: **the user fetches, into `ITULIB_HOME`; nothing is vendored, and the only thing
|
|
15
|
+
committed is a digest and a size per artifact.** `tests/test_license.py` still applies to this tree.
|
|
16
|
+
|
|
17
|
+
Nothing here touches the network on its own. Every function that reads a URL takes a `Transport`, so the tests
|
|
18
|
+
run offline and a caller can supply a caching or rate-limiting one. `urllib_transport` is the real one.
|
|
19
|
+
|
|
20
|
+
## Refs
|
|
21
|
+
|
|
22
|
+
<Rec>#<year>[-<month>]/<Module>[::<section>]
|
|
23
|
+
|
|
24
|
+
X.509#2019-10/AuthenticationFramework the module as a document
|
|
25
|
+
X.509#2019-10/AuthenticationFramework::Certificate one assignment
|
|
26
|
+
X.509#2019/AuthenticationFramework::IMPORTS a reserved section: IMPORTS, EXPORTS or HEADER
|
|
27
|
+
|
|
28
|
+
`<Rec>` is the Recommendation as ITU names it (`X.509`, `X.1080.0`; also `G.7712`, `RFC2578` style folders).
|
|
29
|
+
`<year>` is the folder the artifact sits in, which is the edition's year. The month is optional because the
|
|
30
|
+
folder does not carry it; it is read from the header comment each file opens with, and when a ref gives it
|
|
31
|
+
and it disagrees with the file, the ref does not resolve. An edition folder may carry a suffix (`2017-Cor1`),
|
|
32
|
+
written after the year: `X.1080.0#2017-Cor1/Module`. A bare `::` section matches an assignment name exactly;
|
|
33
|
+
IMPORTS, EXPORTS and HEADER cannot collide with one because they are reserved words in ASN.1.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
import hashlib
|
|
39
|
+
import html
|
|
40
|
+
import json
|
|
41
|
+
import os
|
|
42
|
+
import re
|
|
43
|
+
import time
|
|
44
|
+
import urllib.error
|
|
45
|
+
import urllib.parse
|
|
46
|
+
import urllib.request
|
|
47
|
+
from collections.abc import Callable
|
|
48
|
+
from dataclasses import asdict, dataclass
|
|
49
|
+
from pathlib import Path
|
|
50
|
+
|
|
51
|
+
from . import __version__, asn1mod, corpus
|
|
52
|
+
|
|
53
|
+
BASE = "https://www.itu.int"
|
|
54
|
+
ROOT_PATH = "/ITU-T/formal-language/"
|
|
55
|
+
HOST = "www.itu.int"
|
|
56
|
+
USER_AGENT = f"itulib-formal/{__version__} (+https://github.com/b1tninja/itulib)"
|
|
57
|
+
MIN_INTERVAL = 1.0
|
|
58
|
+
|
|
59
|
+
MAX_ARTIFACT_OCTETS = 2 * 1024 * 1024
|
|
60
|
+
MAX_LISTING_OCTETS = 1024 * 1024
|
|
61
|
+
RESERVED_SECTIONS = ("IMPORTS", "EXPORTS", "HEADER")
|
|
62
|
+
|
|
63
|
+
# Languages by file extension as the listings show them. Only what has been seen is claimed.
|
|
64
|
+
LANGUAGES = {".asn": "ASN.1 (listed, not served)", ".html": "ASN.1 or other, rendered as HTML"}
|
|
65
|
+
|
|
66
|
+
_SEGMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+() -]{0,127}")
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class FormalError(RuntimeError):
|
|
70
|
+
"""A refusal, with the reason. Never an empty result standing in for one."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True)
|
|
74
|
+
class Fetched:
|
|
75
|
+
status: int
|
|
76
|
+
body: bytes
|
|
77
|
+
url: str
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
Transport = Callable[[str, int], Fetched]
|
|
81
|
+
"""`(url, max_octets) -> Fetched`. Must not read more than max_octets + 1 and must report the final URL."""
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def urllib_transport(url: str, max_octets: int) -> Fetched:
|
|
85
|
+
"""The real transport: https to itu.int only, an identifying User-Agent, a bounded read."""
|
|
86
|
+
parts = urllib.parse.urlsplit(url)
|
|
87
|
+
if parts.scheme != "https" or parts.hostname != HOST:
|
|
88
|
+
raise FormalError(f"refusing {url}: only https://{HOST}/ is fetched")
|
|
89
|
+
request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) # noqa: S310
|
|
90
|
+
try:
|
|
91
|
+
with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310
|
|
92
|
+
final = response.geturl()
|
|
93
|
+
body = response.read(max_octets + 1)
|
|
94
|
+
return Fetched(response.status, body, final)
|
|
95
|
+
except urllib.error.HTTPError as error:
|
|
96
|
+
return Fetched(error.code, b"", url)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
# --------------------------------------------------------------------------------------------------- pins
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True)
|
|
103
|
+
class Pin:
|
|
104
|
+
sha256: str
|
|
105
|
+
octets: int
|
|
106
|
+
month: str = "" # "10/2019" as the file's header names it
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# Digests of the `.html` renderings as served on 2026-11-13. ITU re-renders these files (the x509/2019
|
|
110
|
+
# folder carries text from a 2026 corrigendum), so a mismatch means the file changed, not that it was
|
|
111
|
+
# tampered with; the remedy is to re-read the change and re-pin, never to skip the check.
|
|
112
|
+
PINS: dict[str, Pin] = {
|
|
113
|
+
"itu-t/x/x509/2019/AuthenticationFramework.html": Pin(
|
|
114
|
+
"44a22fa7e191ca9072c479e87388131ac21e727eb1ddaf2c03fc620055f707d2", 21_148, "10/2019"
|
|
115
|
+
),
|
|
116
|
+
"itu-t/x/x509/2019/CertificateExtensions.html": Pin(
|
|
117
|
+
"b4c921ae4be61043add9aa6b58873dffeebac84ed585e8909779d7a1c5ef97a7", 29_763, "10/2019"
|
|
118
|
+
),
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def artifact_url(path: str) -> str:
|
|
123
|
+
return f"{BASE}{ROOT_PATH}{urllib.parse.quote(path)}"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _check_path(path: str) -> list[str]:
|
|
127
|
+
"""Validated segments of a path relative to the formal-language root. Refuses anything odd."""
|
|
128
|
+
segments = path.strip("/").split("/")
|
|
129
|
+
if not path or len(segments) > 8:
|
|
130
|
+
raise FormalError(f"bad artifact path {path!r}")
|
|
131
|
+
for segment in segments:
|
|
132
|
+
if not _SEGMENT.fullmatch(segment) or ".." in segment:
|
|
133
|
+
raise FormalError(f"bad path segment {segment!r} in {path!r}")
|
|
134
|
+
return segments
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
# ----------------------------------------------------------------------------------------------- listing
|
|
138
|
+
|
|
139
|
+
_ROW = re.compile(
|
|
140
|
+
r"(?P<date>\d{1,2}/\d{1,2}/\d{4})\s+(?P<time>\d{1,2}:\d{2}\s*[AP]M)\s+"
|
|
141
|
+
r"(?P<size><dir>|\d+)\s*<A\s+HREF=\"(?P<href>[^\"]{1,512})\">(?P<name>[^<]{1,256})</A>",
|
|
142
|
+
re.IGNORECASE,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass(frozen=True)
|
|
147
|
+
class Entry:
|
|
148
|
+
name: str
|
|
149
|
+
directory: bool
|
|
150
|
+
size: int
|
|
151
|
+
stamp: str # as the server prints it: "10/22/2026 7:44 PM"
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def parse_listing(page: str, directory: str) -> tuple[list[Entry], int]:
|
|
155
|
+
"""Entries of an IIS directory listing, and how many rows were dropped as unsafe.
|
|
156
|
+
|
|
157
|
+
`directory` is the path the listing was fetched from, e.g. `/ITU-T/formal-language/itu-t/x/`. A row whose
|
|
158
|
+
link leaves that directory, whose name is not a plain segment, or that repeats is dropped and counted.
|
|
159
|
+
"""
|
|
160
|
+
directory = directory if directory.endswith("/") else directory + "/"
|
|
161
|
+
entries: list[Entry] = []
|
|
162
|
+
seen: set[str] = set()
|
|
163
|
+
dropped = 0
|
|
164
|
+
for match in _ROW.finditer(page):
|
|
165
|
+
name = html.unescape(match["name"]).strip()
|
|
166
|
+
href = html.unescape(match["href"])
|
|
167
|
+
is_dir = match["size"].lower() == "<dir>"
|
|
168
|
+
expected = directory + name + ("/" if is_dir else "")
|
|
169
|
+
if urllib.parse.unquote(href) != expected or not _SEGMENT.fullmatch(name) or ".." in name or name in seen:
|
|
170
|
+
dropped += 1
|
|
171
|
+
continue
|
|
172
|
+
seen.add(name)
|
|
173
|
+
entries.append(Entry(name, is_dir, 0 if is_dir else int(match["size"]), f"{match['date']} {match['time']}"))
|
|
174
|
+
return entries, dropped
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@dataclass
|
|
178
|
+
class Budget:
|
|
179
|
+
"""Bounds on one crawl: requests, bytes, depth and entries. Exhaustion is reported, never hidden."""
|
|
180
|
+
|
|
181
|
+
max_requests: int = 400
|
|
182
|
+
max_octets: int = 64 * 1024 * 1024
|
|
183
|
+
max_depth: int = 6
|
|
184
|
+
max_entries: int = 20_000
|
|
185
|
+
requests: int = 0
|
|
186
|
+
octets: int = 0
|
|
187
|
+
stopped: str = ""
|
|
188
|
+
|
|
189
|
+
def spend(self, octets: int) -> bool:
|
|
190
|
+
self.requests += 1
|
|
191
|
+
self.octets += octets
|
|
192
|
+
if self.requests >= self.max_requests:
|
|
193
|
+
self.stopped = f"request budget {self.max_requests} used"
|
|
194
|
+
elif self.octets >= self.max_octets:
|
|
195
|
+
self.stopped = f"byte budget {self.max_octets} used"
|
|
196
|
+
return not self.stopped
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
@dataclass(frozen=True)
|
|
200
|
+
class Artifact:
|
|
201
|
+
path: str # relative to the formal-language root: itu-t/x/x509/2019/AuthenticationFramework.html
|
|
202
|
+
size: int
|
|
203
|
+
stamp: str
|
|
204
|
+
|
|
205
|
+
@property
|
|
206
|
+
def folder(self) -> str:
|
|
207
|
+
return self.path.rsplit("/", 1)[0]
|
|
208
|
+
|
|
209
|
+
@property
|
|
210
|
+
def name(self) -> str:
|
|
211
|
+
return self.path.rsplit("/", 1)[1]
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def enumerate_tree(
|
|
215
|
+
transport: Transport,
|
|
216
|
+
start: str = "itu-t/",
|
|
217
|
+
*,
|
|
218
|
+
budget: Budget | None = None,
|
|
219
|
+
pause: Callable[[float], None] = time.sleep,
|
|
220
|
+
) -> tuple[list[Artifact], Budget, int]:
|
|
221
|
+
"""Walk directory listings below `start`. Returns artifacts, the budget (check `.stopped`) and drops."""
|
|
222
|
+
budget = budget or Budget()
|
|
223
|
+
root = _check_path(start)
|
|
224
|
+
queue: list[tuple[str, int]] = [("/".join(root) + "/", 0)]
|
|
225
|
+
found: list[Artifact] = []
|
|
226
|
+
dropped = 0
|
|
227
|
+
while queue and not budget.stopped:
|
|
228
|
+
relative, depth = queue.pop(0)
|
|
229
|
+
if budget.requests:
|
|
230
|
+
pause(MIN_INTERVAL)
|
|
231
|
+
response = transport(artifact_url(relative), MAX_LISTING_OCTETS)
|
|
232
|
+
budget.spend(len(response.body))
|
|
233
|
+
if response.status != 200 or len(response.body) > MAX_LISTING_OCTETS:
|
|
234
|
+
dropped += 1
|
|
235
|
+
continue
|
|
236
|
+
if urllib.parse.urlsplit(response.url).hostname != HOST:
|
|
237
|
+
raise FormalError(f"listing redirected away from {HOST}: {response.url}")
|
|
238
|
+
entries, bad = parse_listing(response.body.decode("utf-8", errors="replace"), ROOT_PATH + relative)
|
|
239
|
+
dropped += bad
|
|
240
|
+
for entry in entries:
|
|
241
|
+
if entry.directory:
|
|
242
|
+
if depth + 1 <= budget.max_depth:
|
|
243
|
+
queue.append((relative + entry.name + "/", depth + 1))
|
|
244
|
+
elif len(found) < budget.max_entries:
|
|
245
|
+
found.append(Artifact(relative + entry.name, entry.size, entry.stamp))
|
|
246
|
+
return found, budget, dropped
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
# ----------------------------------------------------------------------------------------------- storage
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def formal_home() -> Path:
|
|
253
|
+
return corpus.home() / "formal"
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def artifact_file(path: str) -> Path:
|
|
257
|
+
return formal_home().joinpath("files", *_check_path(path))
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _manifest_file() -> Path:
|
|
261
|
+
return formal_home() / "manifest.json"
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def load_manifest() -> dict[str, dict[str, object]]:
|
|
265
|
+
file = _manifest_file()
|
|
266
|
+
if not file.is_file():
|
|
267
|
+
return {}
|
|
268
|
+
loaded = json.loads(file.read_text(encoding="utf-8"))
|
|
269
|
+
return loaded if isinstance(loaded, dict) else {}
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _atomic_write(file: Path, data: bytes) -> None:
|
|
273
|
+
file.parent.mkdir(parents=True, exist_ok=True)
|
|
274
|
+
temporary = file.with_name(f"{file.name}.{os.getpid()}.part")
|
|
275
|
+
temporary.write_bytes(data)
|
|
276
|
+
os.replace(temporary, file)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _save_manifest(manifest: dict[str, dict[str, object]]) -> None:
|
|
280
|
+
_atomic_write(_manifest_file(), json.dumps(manifest, indent=1, sort_keys=True).encode("utf-8"))
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@dataclass(frozen=True)
|
|
284
|
+
class Imported:
|
|
285
|
+
path: str
|
|
286
|
+
sha256: str
|
|
287
|
+
octets: int
|
|
288
|
+
pinned: bool
|
|
289
|
+
modules: tuple[str, ...]
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def import_artifact(
|
|
293
|
+
path: str,
|
|
294
|
+
transport: Transport,
|
|
295
|
+
*,
|
|
296
|
+
trust_first_use: bool = False,
|
|
297
|
+
pins: dict[str, Pin] | None = None,
|
|
298
|
+
) -> Imported:
|
|
299
|
+
"""Fetch one artifact into ITULIB_HOME, verified, parsed, and recorded.
|
|
300
|
+
|
|
301
|
+
A pinned artifact must hash to its pin. An unpinned one is refused unless the caller says
|
|
302
|
+
`trust_first_use`, in which case it is stored marked unpinned with its digest reported so it can be pinned.
|
|
303
|
+
Nothing is written until the bytes have been size-checked, hashed and parsed.
|
|
304
|
+
"""
|
|
305
|
+
pins = PINS if pins is None else pins
|
|
306
|
+
_check_path(path)
|
|
307
|
+
if not path.endswith(".html"):
|
|
308
|
+
raise FormalError(f"{path}: only the .html rendering is served by ITU (the .asn listing entry is a 404)")
|
|
309
|
+
pin = pins.get(path)
|
|
310
|
+
if pin is None and not trust_first_use:
|
|
311
|
+
raise FormalError(f"{path} has no pinned digest. Re-run with trust_first_use to store it unpinned.")
|
|
312
|
+
response = transport(artifact_url(path), MAX_ARTIFACT_OCTETS)
|
|
313
|
+
if urllib.parse.urlsplit(response.url).hostname != HOST:
|
|
314
|
+
raise FormalError(f"{path}: redirected away from {HOST} to {response.url}")
|
|
315
|
+
if response.status != 200:
|
|
316
|
+
raise FormalError(f"{path}: HTTP {response.status}")
|
|
317
|
+
body = response.body
|
|
318
|
+
if len(body) > MAX_ARTIFACT_OCTETS:
|
|
319
|
+
raise FormalError(f"{path}: larger than the {MAX_ARTIFACT_OCTETS} octet bound")
|
|
320
|
+
digest = hashlib.sha256(body).hexdigest()
|
|
321
|
+
if pin is not None and (digest != pin.sha256 or len(body) != pin.octets):
|
|
322
|
+
raise FormalError(
|
|
323
|
+
f"{path}: digest {digest} ({len(body)} octets) does not match the pinned "
|
|
324
|
+
f"{pin.sha256} ({pin.octets}). ITU re-renders these files; read the change before re-pinning."
|
|
325
|
+
)
|
|
326
|
+
try:
|
|
327
|
+
text, note = asn1mod.unwrap_html(body)
|
|
328
|
+
modules = asn1mod.parse_modules(text)
|
|
329
|
+
except asn1mod.ModuleError as error:
|
|
330
|
+
raise FormalError(f"{path}: not readable as ASN.1: {error}") from error
|
|
331
|
+
_atomic_write(artifact_file(path), body)
|
|
332
|
+
manifest = load_manifest()
|
|
333
|
+
manifest[path] = {
|
|
334
|
+
"sha256": digest,
|
|
335
|
+
"octets": len(body),
|
|
336
|
+
"pinned": pin is not None,
|
|
337
|
+
"fetched_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
338
|
+
"source": artifact_url(path),
|
|
339
|
+
"header_edition": note,
|
|
340
|
+
"modules": [module.name for module in modules],
|
|
341
|
+
"form": (
|
|
342
|
+
"Asnp-rendered HTML, markup stripped"
|
|
343
|
+
if b"Generated by Asnp" in body
|
|
344
|
+
else "source text in <pre>, wrapper stripped"
|
|
345
|
+
),
|
|
346
|
+
}
|
|
347
|
+
_save_manifest(manifest)
|
|
348
|
+
return Imported(path, digest, len(body), pin is not None, tuple(m.name for m in modules))
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def save_catalogue(artifacts: list[Artifact]) -> Path:
|
|
352
|
+
"""Record an enumeration (paths, sizes, listing dates) beside the files. Metadata only."""
|
|
353
|
+
file = formal_home() / "catalogue.json"
|
|
354
|
+
rows = [asdict(a) for a in sorted(artifacts, key=lambda a: a.path)]
|
|
355
|
+
_atomic_write(
|
|
356
|
+
file,
|
|
357
|
+
json.dumps(
|
|
358
|
+
{"enumerated_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "artifacts": rows}, indent=0
|
|
359
|
+
).encode("utf-8"),
|
|
360
|
+
)
|
|
361
|
+
return file
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
# ------------------------------------------------------------------------------------------------- refs
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
@dataclass(frozen=True)
|
|
368
|
+
class FormalRef:
|
|
369
|
+
rec: str
|
|
370
|
+
year: str
|
|
371
|
+
month: str | None
|
|
372
|
+
suffix: str
|
|
373
|
+
module: str
|
|
374
|
+
section: str | None
|
|
375
|
+
|
|
376
|
+
@property
|
|
377
|
+
def edition(self) -> str:
|
|
378
|
+
return self.year + (f"-{self.month}" if self.month else "") + (f"-{self.suffix}" if self.suffix else "")
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
_REF = re.compile(
|
|
382
|
+
r"(?P<rec>[A-Za-z]{1,6}\.?[0-9][A-Za-z0-9.]{0,15})#(?P<year>\d{4})(?:-(?P<month>(?:0[1-9]|1[0-2])))?"
|
|
383
|
+
r"(?:-(?P<suffix>[A-Za-z][A-Za-z0-9]{0,15}))?/(?P<module>[A-Za-z][A-Za-z0-9-]{0,127})"
|
|
384
|
+
r"(?:::(?P<section>[A-Za-z][A-Za-z0-9-]{0,127}))?"
|
|
385
|
+
)
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def parse_formal_ref(ref: str) -> FormalRef:
|
|
389
|
+
match = _REF.fullmatch(ref.strip())
|
|
390
|
+
if not match:
|
|
391
|
+
raise FormalError(f"{ref!r} is not a formal-description ref: expected Rec#year[-mm]/Module[::section]")
|
|
392
|
+
return FormalRef(
|
|
393
|
+
match["rec"], match["year"], match["month"], match["suffix"] or "", match["module"], match["section"]
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def _folder_matches(folder: str, ref: FormalRef) -> bool:
|
|
398
|
+
parts = folder.split("/")
|
|
399
|
+
rec_folder = ref.rec.lower().replace(".", "")
|
|
400
|
+
edition_folder = ref.year + (f"-{ref.suffix}" if ref.suffix else "")
|
|
401
|
+
return len(parts) >= 2 and parts[-2].lower().replace(".", "") == rec_folder and parts[-1] == edition_folder
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _month_year(note: str | None) -> tuple[str, str] | None:
|
|
405
|
+
found = re.search(r"\((\d{2})/(\d{4})\)", note or "")
|
|
406
|
+
return (found.group(2), found.group(1)) if found else None
|
|
407
|
+
|
|
408
|
+
|
|
409
|
+
def resolve(ref: FormalRef, manifest: dict[str, dict[str, object]] | None = None) -> str:
|
|
410
|
+
"""The artifact path a ref names among the imported ones, or a refusal that says why."""
|
|
411
|
+
manifest = load_manifest() if manifest is None else manifest
|
|
412
|
+
candidates = [
|
|
413
|
+
path
|
|
414
|
+
for path, info in manifest.items()
|
|
415
|
+
if path.endswith(".html")
|
|
416
|
+
and _folder_matches(path.rsplit("/", 1)[0], ref)
|
|
417
|
+
and ref.module in info.get("modules", []) # type: ignore[operator]
|
|
418
|
+
]
|
|
419
|
+
if not candidates:
|
|
420
|
+
raise FormalError(
|
|
421
|
+
f"{ref.rec}#{ref.edition}/{ref.module} is not imported. Import it with "
|
|
422
|
+
"`python -m primarysource.itu.formal import <path>`; list what is imported with "
|
|
423
|
+
"`python -m primarysource.itu.formal list`."
|
|
424
|
+
)
|
|
425
|
+
if len(candidates) > 1:
|
|
426
|
+
raise FormalError(f"ambiguous: {sorted(candidates)}")
|
|
427
|
+
path = candidates[0]
|
|
428
|
+
stated = _month_year(str(manifest[path].get("header_edition")))
|
|
429
|
+
if ref.month and stated and (stated[0], stated[1]) != (ref.year, ref.month):
|
|
430
|
+
raise FormalError(
|
|
431
|
+
f"{path} states edition {stated[1]}/{stated[0]}, not {ref.month}/{ref.year}; "
|
|
432
|
+
f"the ref does not resolve rather than citing a different edition"
|
|
433
|
+
)
|
|
434
|
+
return path
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
# ------------------------------------------------------------------------------------------- documents
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def read(ref_text: str, manifest: dict[str, dict[str, object]] | None = None) -> dict[str, object]:
|
|
441
|
+
"""Serve a ref as a document: the module as a whole, or one section of it."""
|
|
442
|
+
ref = parse_formal_ref(ref_text)
|
|
443
|
+
manifest = load_manifest() if manifest is None else manifest
|
|
444
|
+
path = resolve(ref, manifest)
|
|
445
|
+
info = manifest[path]
|
|
446
|
+
text, _ = asn1mod.unwrap_html(artifact_file(path).read_bytes())
|
|
447
|
+
if hashlib.sha256(artifact_file(path).read_bytes()).hexdigest() != info["sha256"]:
|
|
448
|
+
raise FormalError(f"{path}: stored bytes no longer match the recorded digest; re-import")
|
|
449
|
+
modules = {m.name: m for m in asn1mod.parse_modules(text)}
|
|
450
|
+
module = modules.get(ref.module)
|
|
451
|
+
if module is None:
|
|
452
|
+
raise FormalError(f"{path} has no module {ref.module}")
|
|
453
|
+
stated = str(info.get("header_edition") or "") or None
|
|
454
|
+
cite = (
|
|
455
|
+
f"ITU-T {stated} formal description, module {module.name}"
|
|
456
|
+
if stated
|
|
457
|
+
else f"ITU-T {ref.rec} ({ref.year}) formal description, module {module.name}"
|
|
458
|
+
)
|
|
459
|
+
result: dict[str, object] = {
|
|
460
|
+
"ref": ref_text.strip(),
|
|
461
|
+
"module": module.name,
|
|
462
|
+
"oid": module.oid,
|
|
463
|
+
"oid_text": module.oid_text,
|
|
464
|
+
"source": info.get("source"),
|
|
465
|
+
"sha256": info["sha256"],
|
|
466
|
+
"pinned": info.get("pinned"),
|
|
467
|
+
"form": info.get("form"),
|
|
468
|
+
"cite": cite,
|
|
469
|
+
}
|
|
470
|
+
section = ref.section
|
|
471
|
+
if section is None:
|
|
472
|
+
result.update(
|
|
473
|
+
tagging=module.tagging,
|
|
474
|
+
extensibility_implied=module.extensibility_implied,
|
|
475
|
+
exports=module.exports
|
|
476
|
+
if module.exports is None or isinstance(module.exports, str)
|
|
477
|
+
else list(module.exports),
|
|
478
|
+
imports=[
|
|
479
|
+
{"module": i.module, "oid_text": i.oid_text, "symbols": list(i.symbols)} for i in module.imports
|
|
480
|
+
],
|
|
481
|
+
sections=[{"name": a.name, "kind": a.kind, "line": a.line} for a in module.assignments],
|
|
482
|
+
)
|
|
483
|
+
return result
|
|
484
|
+
if section == "IMPORTS":
|
|
485
|
+
result["imports"] = [
|
|
486
|
+
{"module": i.module, "oid_text": i.oid_text, "symbols": list(i.symbols)} for i in module.imports
|
|
487
|
+
]
|
|
488
|
+
return result
|
|
489
|
+
if section == "EXPORTS":
|
|
490
|
+
result["exports"] = (
|
|
491
|
+
module.exports if module.exports is None or isinstance(module.exports, str) else list(module.exports)
|
|
492
|
+
)
|
|
493
|
+
return result
|
|
494
|
+
if section == "HEADER":
|
|
495
|
+
result["text"] = text[: text.find("BEGIN") + 5] if "BEGIN" in text else ""
|
|
496
|
+
return result
|
|
497
|
+
found = module.assignment(section)
|
|
498
|
+
if found is None:
|
|
499
|
+
raise FormalError(f"{module.name} has no assignment {section}")
|
|
500
|
+
result.update(
|
|
501
|
+
name=found.name, kind=found.kind, parameterised=found.parameterised, line=found.line, text=found.text
|
|
502
|
+
)
|
|
503
|
+
return result
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def listing() -> list[dict[str, object]]:
|
|
507
|
+
return [
|
|
508
|
+
{
|
|
509
|
+
"path": p,
|
|
510
|
+
**{k: v for k, v in i.items() if k in {"sha256", "octets", "pinned", "modules", "header_edition"}},
|
|
511
|
+
}
|
|
512
|
+
for p, i in sorted(load_manifest().items())
|
|
513
|
+
]
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def main(argv: list[str] | None = None) -> int:
|
|
517
|
+
import argparse
|
|
518
|
+
import sys
|
|
519
|
+
|
|
520
|
+
parser = argparse.ArgumentParser(prog="python -m primarysource.itu.formal", description=__doc__.splitlines()[0])
|
|
521
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
522
|
+
enum = sub.add_parser("enumerate", help="walk the directory listings and record the catalogue")
|
|
523
|
+
enum.add_argument("start", nargs="?", default="itu-t/x/")
|
|
524
|
+
enum.add_argument("--max-requests", type=int, default=400)
|
|
525
|
+
imp = sub.add_parser("import", help="fetch artifacts (the .html path under formal-language/) into ITULIB_HOME")
|
|
526
|
+
imp.add_argument("paths", nargs="+")
|
|
527
|
+
imp.add_argument("--trust-first-use", action="store_true")
|
|
528
|
+
sub.add_parser("list", help="what is imported")
|
|
529
|
+
rd = sub.add_parser("read", help="serve a ref: Rec#year[-mm]/Module[::section]")
|
|
530
|
+
rd.add_argument("ref")
|
|
531
|
+
args = parser.parse_args(argv)
|
|
532
|
+
try:
|
|
533
|
+
if args.command == "enumerate":
|
|
534
|
+
artifacts, budget, dropped = enumerate_tree(
|
|
535
|
+
urllib_transport, args.start, budget=Budget(max_requests=args.max_requests)
|
|
536
|
+
)
|
|
537
|
+
file = save_catalogue(artifacts)
|
|
538
|
+
print(
|
|
539
|
+
json.dumps(
|
|
540
|
+
{
|
|
541
|
+
"artifacts": len(artifacts),
|
|
542
|
+
"requests": budget.requests,
|
|
543
|
+
"stopped": budget.stopped,
|
|
544
|
+
"dropped": dropped,
|
|
545
|
+
"catalogue": str(file),
|
|
546
|
+
}
|
|
547
|
+
)
|
|
548
|
+
)
|
|
549
|
+
elif args.command == "import":
|
|
550
|
+
for path in args.paths:
|
|
551
|
+
done = import_artifact(path, urllib_transport, trust_first_use=args.trust_first_use)
|
|
552
|
+
print(json.dumps(asdict(done)))
|
|
553
|
+
time.sleep(MIN_INTERVAL)
|
|
554
|
+
elif args.command == "list":
|
|
555
|
+
print(json.dumps(listing(), indent=1))
|
|
556
|
+
else:
|
|
557
|
+
print(json.dumps(read(args.ref), indent=1))
|
|
558
|
+
except FormalError as error:
|
|
559
|
+
print(f"error: {error}", file=sys.stderr)
|
|
560
|
+
return 2
|
|
561
|
+
return 0
|
|
562
|
+
|
|
563
|
+
|
|
564
|
+
if __name__ == "__main__":
|
|
565
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""How the itu collection recognises an ITU-T Recommendation in a supplied PDF (`registry.Collection.identify`).
|
|
2
|
+
|
|
3
|
+
A cover reads `Recommendation ITU-T X.690` and pages carry `Rec. ITU-T X.690 (02/2021)`; ITU's own download is
|
|
4
|
+
named `T-REC-X.690-202102-I!!PDF-E.pdf`. The itu collection fetches its own pinned copy, so the command it
|
|
5
|
+
suggests is the fetch, not an import of this file.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import re
|
|
11
|
+
|
|
12
|
+
from primarysource.identify import Candidate, Probe, candidates
|
|
13
|
+
|
|
14
|
+
_TEXT = re.compile(r"\b(?:Rec(?:ommendation)?\.?\s+)?ITU-T\s+(?P<rec>[A-Z]\.\d{1,4}(?:\.\d{1,3})?)\b")
|
|
15
|
+
_FILE = re.compile(r"T-REC-(?P<rec>[A-Z]\.\d{1,4}(?:\.\d{1,3})?)-")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _rec(match: re.Match[str]) -> str:
|
|
19
|
+
return f"Rec. ITU-T {match['rec']}"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def identify(probe: Probe) -> list[Candidate]:
|
|
23
|
+
def command(label: str) -> str:
|
|
24
|
+
return f"primarysource itu fetch {label.rsplit(' ', 1)[-1]} (itu fetches its own pinned copy)"
|
|
25
|
+
|
|
26
|
+
return candidates(probe, "itu", _TEXT, _rec, command, file_pattern=_FILE)
|