iso-standard-check 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,27 @@
1
+ CC0 1.0 Universal — Public Domain Dedication
2
+
3
+ The person who associated a work with this deed has dedicated the work to the
4
+ public domain by waiving all of his or her rights to the work worldwide under
5
+ copyright law, including all related and neighboring rights, to the extent
6
+ allowed by law.
7
+
8
+ You can copy, modify, distribute and perform the work, even for commercial
9
+ purposes, all without asking permission.
10
+
11
+ Full text: https://creativecommons.org/publicdomain/zero/1.0/legalcode
12
+
13
+ ---
14
+
15
+ This dedication covers the compilation and the code in this package.
16
+
17
+ The underlying facts are taken from third-party sources, each of which is cited
18
+ by URL in the shipped data alongside the verbatim quote the value was read from:
19
+
20
+ * PyTorch's RELEASE.md and .ci/manywheel/build_cuda.sh — BSD-3-Clause
21
+ (pytorch/pytorch LICENSE)
22
+ * NVIDIA's CUDA Toolkit release notes and CUDA GPUs page — vendor technical
23
+ documentation
24
+
25
+ Facts are not copyrightable. What is reproduced from those sources is a fact
26
+ plus a short attributed quote linking back to it; neither document is
27
+ reproduced.
@@ -0,0 +1,148 @@
1
+ Metadata-Version: 2.4
2
+ Name: iso-standard-check
3
+ Version: 0.1.0
4
+ Summary: Find ISO and IEC standard citations in a repository and fail CI on the editions ISO's own register marks withdrawn — every verdict with the register row it was read from
5
+ Author: referencesource.org
6
+ License: CC0-1.0
7
+ Project-URL: Homepage, https://referencesource.org/iso-standard-supersessions/
8
+ Project-URL: Source, https://referencesource.org/iso-standard-supersessions/
9
+ Keywords: iso,iec,standards,withdrawn,superseded,iso-9001,iso-13485,iso-14001,iso-27001,compliance,ci,pre-commit,lint,quality-management,references
10
+ Classifier: Development Status :: 4 - Beta
11
+ Classifier: Intended Audience :: Developers
12
+ Classifier: Intended Audience :: Manufacturing
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Software Development :: Quality Assurance
15
+ Classifier: Topic :: Text Processing :: Linguistic
16
+ Requires-Python: >=3.8
17
+ Description-Content-Type: text/markdown
18
+ License-File: LICENSE
19
+ Dynamic: license-file
20
+
21
+ # iso-standard-check
22
+
23
+ Find the ISO and IEC standards your repository cites, and fail CI on the ones
24
+ ISO's own register marks **withdrawn or superseded** — each with the register row
25
+ it was read from.
26
+
27
+ ```bash
28
+ npx iso-standard-check . # scan a repository
29
+ npx iso-standard-check docs/ supplier-spec.md # or named paths
30
+ npx iso-standard-check --standard "ISO 9001:2008"
31
+ ```
32
+
33
+ ```
34
+ FAIL ISO 9001:2015 is withdrawn or superseded — docs/quality-plan.md:12, 48
35
+ Quality management systems — Requirements
36
+ the register names ISO 9001:2026 as its replacement
37
+ register row: 62085,IS,,ISO 9001:2015,Quality management systems — Requirements,…,2015-09-22,5,…
38
+ https://referencesource.org/iso-standard-supersessions/iso-9001-2015/
39
+
40
+ ? ISO 14001 names no edition year — docs/quality-plan.md:19
41
+ not judged: the register marks 1996, 2004, 2015 withdrawn. Cite the edition
42
+ and this becomes a check.
43
+ ```
44
+
45
+ Exit **1** if a cited edition is withdrawn. Exit **0** otherwise.
46
+
47
+ ## Why this is hard to do by hand
48
+
49
+ **ISO 9001:2015 is no longer current.** ISO published ISO 9001:2026 on
50
+ 2026-09-16, so every quality plan, supplier spec and audit checklist in the world
51
+ that cites the 2015 edition now cites a superseded one. That is the ordinary case
52
+ this checker exists for, and nothing in a repository tells you it happened.
53
+
54
+ **The edition year is the whole difficulty.** Every one of the 24,640
55
+ identifiers in ISO's register is of the form `ISO 9001:2008`. ISO 9001:2008 is
56
+ withdrawn; its replacement is not. So a document that writes just **"ISO 9001"**
57
+ cannot be judged withdrawn or current by anybody, this program included. Those
58
+ citations are printed as *unjudged*, with the editions of that designation the
59
+ register does mark withdrawn, and **they never fail your build**. Call it with
60
+ `--strict` if you would rather be made to write the edition down.
61
+
62
+ ## What the index covers, and what a silence means
63
+
64
+ The shipped data holds deliverables ISO's register marks withdrawn or
65
+ superseded, **and nothing else**.
66
+
67
+ So a hit is a measurement and a miss is not. A designation this program does not
68
+ report may be current, may be withdrawn in a later edition of the register than
69
+ the one we read, or may not exist. It is reported as *not in this extract*, and
70
+ it is never printed as "current". If you need "is this one current?", that is a
71
+ different question and this is not the tool that answers it.
72
+
73
+ Two things ISO's register does not give, so neither does this package:
74
+
75
+ - **The date a standard was withdrawn.** The only date in the file is the
76
+ deliverable's own publication date. It is printed as `published` and never as a
77
+ withdrawal date.
78
+ - **Any guarantee the named replacement exists as a document.** 2,507 of the
79
+ 24,640 records (10.2%) name at least one replacement the register gives no
80
+ publication date; on 416 (1.7%) *every* named replacement is undated — a
81
+ registered revision project you may not be able to obtain. Both cases say so
82
+ in the output rather than handing you a designation that cannot be bought.
83
+
84
+ ## Where the numbers come from
85
+
86
+ Every verdict is read from one file: ISO's own open-data deliverables register,
87
+
88
+ ```
89
+ https://isopublicstorageprod.blob.core.windows.net/opendata/_latest/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv
90
+ ```
91
+
92
+ published by ISO under the Open Data Commons Attribution License v1.0, which
93
+ permits reuse with attribution. Each of the 24,640 records carries the CSV row it
94
+ was read from, verbatim, and is published on its own page with that row visible:
95
+
96
+ **<https://referencesource.org/out/pkg/iso-standard-check/>**
97
+
98
+ Nothing in this package computes, normalises or infers a status. It reads what
99
+ the register states, and where the register states nothing it says so in words.
100
+ The records are re-checked against the live file on a schedule; the index here
101
+ carries the date it was built.
102
+
103
+ ## As a pre-commit hook
104
+
105
+ The Python twin on PyPI ships a hook:
106
+
107
+ ```yaml
108
+ repos:
109
+ - repo: https://github.com/referencesource/iso-standard-check
110
+ rev: v0.1.0
111
+ hooks:
112
+ - id: iso-standard-check
113
+ ```
114
+
115
+ Or `pipx run iso-standard-check .`
116
+
117
+ ## In a GitHub Actions workflow
118
+
119
+ ```yaml
120
+ - name: ISO citations
121
+ run: npx --yes iso-standard-check docs/ specs/
122
+ ```
123
+
124
+ ## Options
125
+
126
+ | | |
127
+ |---|---|
128
+ | `<paths>` | files or directories; defaults to `.` |
129
+ | `--standard "<id>"` | ask about one designation and exit |
130
+ | `--json` | machine-readable, including every line number |
131
+ | `--quiet` | withdrawn citations only, no unjudged ones |
132
+ | `--strict` | also exit 1 when a citation names no edition year |
133
+
134
+ Caches, virtualenvs, `node_modules` and build output are not walked: a citation
135
+ inside something you installed is a fact about somebody else's repository.
136
+
137
+ ## API
138
+
139
+ ```js
140
+ const iso = require('iso-standard-check');
141
+ iso.lookup('ISO 13485:2003'); // one designation
142
+ iso.scanText(readmeText); // findings with line numbers
143
+ iso.scanPaths(['docs']); // grouped by verdict
144
+ ```
145
+
146
+ CC0-1.0. Built by an automated pipeline at
147
+ [referencesource.org](https://referencesource.org/iso-standard-supersessions/);
148
+ the facts are ISO's and are attributed to ISO on every line.
@@ -0,0 +1,128 @@
1
+ # iso-standard-check
2
+
3
+ Find the ISO and IEC standards your repository cites, and fail CI on the ones
4
+ ISO's own register marks **withdrawn or superseded** — each with the register row
5
+ it was read from.
6
+
7
+ ```bash
8
+ npx iso-standard-check . # scan a repository
9
+ npx iso-standard-check docs/ supplier-spec.md # or named paths
10
+ npx iso-standard-check --standard "ISO 9001:2008"
11
+ ```
12
+
13
+ ```
14
+ FAIL ISO 9001:2015 is withdrawn or superseded — docs/quality-plan.md:12, 48
15
+ Quality management systems — Requirements
16
+ the register names ISO 9001:2026 as its replacement
17
+ register row: 62085,IS,,ISO 9001:2015,Quality management systems — Requirements,…,2015-09-22,5,…
18
+ https://referencesource.org/iso-standard-supersessions/iso-9001-2015/
19
+
20
+ ? ISO 14001 names no edition year — docs/quality-plan.md:19
21
+ not judged: the register marks 1996, 2004, 2015 withdrawn. Cite the edition
22
+ and this becomes a check.
23
+ ```
24
+
25
+ Exit **1** if a cited edition is withdrawn. Exit **0** otherwise.
26
+
27
+ ## Why this is hard to do by hand
28
+
29
+ **ISO 9001:2015 is no longer current.** ISO published ISO 9001:2026 on
30
+ 2026-09-16, so every quality plan, supplier spec and audit checklist in the world
31
+ that cites the 2015 edition now cites a superseded one. That is the ordinary case
32
+ this checker exists for, and nothing in a repository tells you it happened.
33
+
34
+ **The edition year is the whole difficulty.** Every one of the 24,640
35
+ identifiers in ISO's register is of the form `ISO 9001:2008`. ISO 9001:2008 is
36
+ withdrawn; its replacement is not. So a document that writes just **"ISO 9001"**
37
+ cannot be judged withdrawn or current by anybody, this program included. Those
38
+ citations are printed as *unjudged*, with the editions of that designation the
39
+ register does mark withdrawn, and **they never fail your build**. Call it with
40
+ `--strict` if you would rather be made to write the edition down.
41
+
42
+ ## What the index covers, and what a silence means
43
+
44
+ The shipped data holds deliverables ISO's register marks withdrawn or
45
+ superseded, **and nothing else**.
46
+
47
+ So a hit is a measurement and a miss is not. A designation this program does not
48
+ report may be current, may be withdrawn in a later edition of the register than
49
+ the one we read, or may not exist. It is reported as *not in this extract*, and
50
+ it is never printed as "current". If you need "is this one current?", that is a
51
+ different question and this is not the tool that answers it.
52
+
53
+ Two things ISO's register does not give, so neither does this package:
54
+
55
+ - **The date a standard was withdrawn.** The only date in the file is the
56
+ deliverable's own publication date. It is printed as `published` and never as a
57
+ withdrawal date.
58
+ - **Any guarantee the named replacement exists as a document.** 2,507 of the
59
+ 24,640 records (10.2%) name at least one replacement the register gives no
60
+ publication date; on 416 (1.7%) *every* named replacement is undated — a
61
+ registered revision project you may not be able to obtain. Both cases say so
62
+ in the output rather than handing you a designation that cannot be bought.
63
+
64
+ ## Where the numbers come from
65
+
66
+ Every verdict is read from one file: ISO's own open-data deliverables register,
67
+
68
+ ```
69
+ https://isopublicstorageprod.blob.core.windows.net/opendata/_latest/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv
70
+ ```
71
+
72
+ published by ISO under the Open Data Commons Attribution License v1.0, which
73
+ permits reuse with attribution. Each of the 24,640 records carries the CSV row it
74
+ was read from, verbatim, and is published on its own page with that row visible:
75
+
76
+ **<https://referencesource.org/out/pkg/iso-standard-check/>**
77
+
78
+ Nothing in this package computes, normalises or infers a status. It reads what
79
+ the register states, and where the register states nothing it says so in words.
80
+ The records are re-checked against the live file on a schedule; the index here
81
+ carries the date it was built.
82
+
83
+ ## As a pre-commit hook
84
+
85
+ The Python twin on PyPI ships a hook:
86
+
87
+ ```yaml
88
+ repos:
89
+ - repo: https://github.com/referencesource/iso-standard-check
90
+ rev: v0.1.0
91
+ hooks:
92
+ - id: iso-standard-check
93
+ ```
94
+
95
+ Or `pipx run iso-standard-check .`
96
+
97
+ ## In a GitHub Actions workflow
98
+
99
+ ```yaml
100
+ - name: ISO citations
101
+ run: npx --yes iso-standard-check docs/ specs/
102
+ ```
103
+
104
+ ## Options
105
+
106
+ | | |
107
+ |---|---|
108
+ | `<paths>` | files or directories; defaults to `.` |
109
+ | `--standard "<id>"` | ask about one designation and exit |
110
+ | `--json` | machine-readable, including every line number |
111
+ | `--quiet` | withdrawn citations only, no unjudged ones |
112
+ | `--strict` | also exit 1 when a citation names no edition year |
113
+
114
+ Caches, virtualenvs, `node_modules` and build output are not walked: a citation
115
+ inside something you installed is a fact about somebody else's repository.
116
+
117
+ ## API
118
+
119
+ ```js
120
+ const iso = require('iso-standard-check');
121
+ iso.lookup('ISO 13485:2003'); // one designation
122
+ iso.scanText(readmeText); // findings with line numbers
123
+ iso.scanPaths(['docs']); // grouped by verdict
124
+ ```
125
+
126
+ CC0-1.0. Built by an automated pipeline at
127
+ [referencesource.org](https://referencesource.org/iso-standard-supersessions/);
128
+ the facts are ISO's and are attributed to ISO on every line.
@@ -0,0 +1,37 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "iso-standard-check"
7
+ version = "0.1.0"
8
+ description = "Find ISO and IEC standard citations in a repository and fail CI on the editions ISO's own register marks withdrawn — every verdict with the register row it was read from"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = { text = "CC0-1.0" }
12
+ authors = [{ name = "referencesource.org" }]
13
+ keywords = ["iso", "iec", "standards", "withdrawn", "superseded", "iso-9001",
14
+ "iso-13485", "iso-14001", "iso-27001", "compliance", "ci",
15
+ "pre-commit", "lint", "quality-management", "references"]
16
+ classifiers = [
17
+ "Development Status :: 4 - Beta",
18
+ "Intended Audience :: Developers",
19
+ "Intended Audience :: Manufacturing",
20
+ "Programming Language :: Python :: 3",
21
+ "Topic :: Software Development :: Quality Assurance",
22
+ "Topic :: Text Processing :: Linguistic",
23
+ ]
24
+ dependencies = []
25
+
26
+ [project.urls]
27
+ Homepage = "https://referencesource.org/iso-standard-supersessions/"
28
+ Source = "https://referencesource.org/iso-standard-supersessions/"
29
+
30
+ [project.scripts]
31
+ iso-standard-check = "iso_standard_check.__main__:main"
32
+
33
+ [tool.setuptools.packages.find]
34
+ where = ["src"]
35
+
36
+ [tool.setuptools.package-data]
37
+ iso_standard_check = ["data.json"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,283 @@
1
+ """iso-standard-check — find ISO and IEC standard citations in a repository and
2
+ say which editions ISO's own register marks withdrawn or superseded.
3
+
4
+ The Python twin of the npm package of the same name. Both read the same
5
+ `data.json`, built from our records by `ops/packages/iso-standard-check/build_data.py`,
6
+ and `ops/iso-standard-check-selftest.py` checks that the two implementations of
7
+ the key rule agree -- a second copy of a rule is a second place for it to drift.
8
+
9
+ WHAT THIS INDEX COVERS, because it decides what the program may say. The shipped
10
+ data holds deliverables the register marks withdrawn or superseded, and nothing
11
+ else. So a hit is a measurement and a miss is not: a designation absent from the
12
+ index may be current, may be withdrawn in a later edition of the register than
13
+ the one we read, or may not exist. A miss is reported as `not-in-extract` and is
14
+ never printed as "current".
15
+
16
+ THE EDITION YEAR IS THE WHOLE DIFFICULTY. Every one of the 24,640 identifiers in
17
+ the register is of the form `ISO 9001:2008`. ISO 9001:2008 is withdrawn and
18
+ ISO 9001:2026 superseded ISO 9001:2015 on 2026-09-16, so a document citing
19
+ "ISO 9001" with no edition cannot be judged by anybody, us included. Those are
20
+ reported as `no-edition-cited`, with the editions of that family the register
21
+ does mark withdrawn, and they never fail a build.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import os
27
+ import pathlib
28
+ import re
29
+
30
+ __version__ = "0.1.0"
31
+
32
+ DATA_PATH = pathlib.Path(__file__).resolve().parent / "data.json"
33
+ with DATA_PATH.open(encoding="utf-8") as _fh:
34
+ DATA = json.load(_fh)
35
+
36
+ SKIP_DIRS = frozenset({
37
+ # GENERATED from ops/packages/skip-dirs.json by ops/packages/sync_skip_dirs.py.
38
+ # Do not edit here: add the directory to that file and run the program,
39
+ # or ops/package-skip-dirs-selftest.py fails on the copy you missed.
40
+ # not your code
41
+ "node_modules", "vendor", "bower_components", "site-packages", ".git",
42
+ ".svn", ".hg",
43
+ # build and test output
44
+ "dist", "build", "out", "target", "coverage", ".next", ".nuxt",
45
+ ".svelte-kit", ".angular", ".dart_tool", ".serverless", ".vercel",
46
+ ".netlify",
47
+ # virtualenvs and installed toolchains
48
+ ".venv", "venv", "env", ".direnv", ".conda", ".pyenv", ".nvm",
49
+ ".rustup", ".stack-work",
50
+ # package manager and tool caches
51
+ "__pycache__", ".cache", ".tox", ".nox", ".eggs", ".mypy_cache",
52
+ ".pytest_cache", ".ruff_cache", ".ipynb_checkpoints", ".yarn",
53
+ ".pnpm-store", ".pnpm", ".npm", ".bundle", ".gem", ".cpanm", ".m2",
54
+ ".nuget", ".gradle", ".cargo", ".pub-cache", ".turbo",
55
+ ".parcel-cache", ".sass-cache", ".terraform", ".terragrunt-cache",
56
+ ".dvc",
57
+ # editor and shell state
58
+ ".idea", ".history", ".local",
59
+ })
60
+
61
+ TEXT_EXT = frozenset({
62
+ '.md', '.markdown', '.rst', '.adoc', '.asciidoc', '.txt', '.text',
63
+ '.yaml', '.yml', '.json', '.toml', '.ini', '.cfg', '.csv', '.tsv',
64
+ '.html', '.htm', '.xml', '.tex', '.org',
65
+ '.py', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.java', '.kt', '.cs',
66
+ '.c', '.h', '.cc', '.cpp', '.hpp', '.go', '.rs', '.rb', '.php', '.sql',
67
+ '.sh', '.ps1',
68
+ })
69
+
70
+ MAX_BYTES = 4 * 1024 * 1024
71
+
72
+
73
+ # ------------------------------------------------------------------ keys
74
+
75
+ def norm(text: str) -> str:
76
+ """A designation reduced to what a citation in a document can be matched on.
77
+
78
+ The same rule as the builder's `norm` and the npm package's: case and the
79
+ punctuation that separates words go, the digits, the hyphens inside a part
80
+ number and the edition year stay exactly.
81
+ """
82
+ s = (text or "").lower()
83
+ s = s.replace("–", "-").replace("—", "-")
84
+ s = re.sub(r'\s*:\s*', ':', s)
85
+ s = re.sub(r'[/ ]+', ' ', s)
86
+ s = re.sub(r'\s+', ' ', s).strip()
87
+ s = re.sub(r'(?<=[a-z])-(?=[a-z])', ' ', s)
88
+ return s
89
+
90
+
91
+ def split_edition(key: str):
92
+ """(designation, year) -- year is "" when the citation names no edition."""
93
+ m = re.match(r'^(.+?):(\d{4})$', key)
94
+ return (m.group(1), m.group(2)) if m else (key, "")
95
+
96
+
97
+ def slug_id(identifier: str) -> str:
98
+ """`IEC 31010:2009` -> `iec-31010-2009`, our own record id rule."""
99
+ return re.sub(r'[^0-9a-z]+', '-', (identifier or "").lower()).strip('-')
100
+
101
+
102
+ def page_for(rec: dict, data: dict | None = None) -> str:
103
+ d = data or DATA
104
+ return f"{d['site']}/{d['asset']}/{rec.get('id') or slug_id(rec['identifier'])}/"
105
+
106
+
107
+ # ------------------------------------------------------------- the scanner
108
+
109
+ def prefixes(data: dict | None = None):
110
+ """The body-and-type prefixes the index can speak about, longest first.
111
+
112
+ Read off the shipped data's own `families` keys rather than typed here, so
113
+ the program never looks for a shape of designation it holds no records of.
114
+ """
115
+ d = data or DATA
116
+ seen = set()
117
+ for family in d.get("families", {}):
118
+ words = []
119
+ for tok in family.split(" "):
120
+ if any(c.isdigit() for c in tok):
121
+ break
122
+ words.append(tok)
123
+ if words:
124
+ seen.add(" ".join(words))
125
+ return sorted(seen, key=lambda p: (-len(p), p))
126
+
127
+
128
+ def citation_regex(data: dict | None = None):
129
+ alts = ["[\\s/._-]*".join(re.escape(w) for w in p.split(" "))
130
+ for p in prefixes(data)]
131
+ return re.compile(
132
+ r'\b(?:' + "|".join(alts) + r')[\s/._-]+'
133
+ r'\d+(?:[-–—]\d+)*\s*(?::\s*\d{4})?',
134
+ re.I)
135
+
136
+
137
+ _RX = None
138
+
139
+
140
+ def citations(text: str, data: dict | None = None):
141
+ """Every ISO or IEC citation in a block of text, in the order it appears."""
142
+ global _RX
143
+ d = data or DATA
144
+ if d is DATA:
145
+ if _RX is None:
146
+ _RX = citation_regex(d)
147
+ rx = _RX
148
+ else:
149
+ rx = citation_regex(d)
150
+ out = []
151
+ for m in rx.finditer(text):
152
+ raw = m.group(0).strip().rstrip('.,;:)]')
153
+ key = norm(raw)
154
+ designation, year = split_edition(key)
155
+ out.append({
156
+ "raw": raw, "key": key, "designation": designation, "year": year,
157
+ "line": text.count("\n", 0, m.start()) + 1,
158
+ })
159
+ return out
160
+
161
+
162
+ def judge(citation: dict, data: dict | None = None) -> dict:
163
+ """What the register says about one citation. Never a guess."""
164
+ d = data or DATA
165
+ rec = d.get("withdrawn", {}).get(citation["key"])
166
+ if rec:
167
+ out = dict(rec)
168
+ out["page"] = page_for(rec, d)
169
+ return {"verdict": "withdrawn", "record": out}
170
+ editions = d.get("families", {}).get(citation["designation"], [])
171
+ if not citation["year"]:
172
+ return {"verdict": "no-edition-cited", "withdrawn_editions": editions}
173
+ return {"verdict": "not-in-extract", "withdrawn_editions": editions}
174
+
175
+
176
+ def scan_text(text: str, data: dict | None = None):
177
+ """One finding per citation, deduplicated by key, every line it appears on."""
178
+ d = data or DATA
179
+ by_key: dict = {}
180
+ for c in citations(text, d):
181
+ prev = by_key.get(c["key"])
182
+ if prev:
183
+ prev["lines"].append(c["line"])
184
+ continue
185
+ row = {k: c[k] for k in ("raw", "key", "designation", "year")}
186
+ row["lines"] = [c["line"]]
187
+ row.update(judge(c, d))
188
+ by_key[c["key"]] = row
189
+ return list(by_key.values())
190
+
191
+
192
+ # ------------------------------------------------------------------ files
193
+
194
+ def _walk(target: str, acc: dict, depth: int = 0) -> None:
195
+ try:
196
+ st = os.stat(target)
197
+ except OSError as e:
198
+ acc["unreadable"].append({"path": target, "why": e.strerror or str(e)})
199
+ return
200
+ if os.path.isdir(target):
201
+ if depth > 24:
202
+ return
203
+ try:
204
+ names = sorted(os.listdir(target))
205
+ except OSError as e:
206
+ acc["unreadable"].append({"path": target, "why": e.strerror or str(e)})
207
+ return
208
+ for name in names:
209
+ p = os.path.join(target, name)
210
+ if os.path.isdir(p):
211
+ if name not in SKIP_DIRS:
212
+ _walk(p, acc, depth + 1)
213
+ elif os.path.isfile(p):
214
+ if os.path.splitext(name)[1].lower() in TEXT_EXT:
215
+ _walk(p, acc, depth + 1)
216
+ return
217
+ if not os.path.isfile(target):
218
+ return
219
+ if st.st_size > MAX_BYTES:
220
+ acc["skipped_too_large"].append({"path": target, "bytes": st.st_size})
221
+ return
222
+ acc["files"].append(target)
223
+
224
+
225
+ def scan_paths(paths, data: dict | None = None) -> dict:
226
+ """Scan files and directories.
227
+
228
+ A file that cannot be read is reported in `unreadable`, never counted as a
229
+ file with no citations in it: those two are different facts.
230
+ """
231
+ d = data or DATA
232
+ acc = {"files": [], "unreadable": [], "skipped_too_large": []}
233
+ for p in paths:
234
+ _walk(p, acc)
235
+
236
+ withdrawn, unjudged, missing = [], [], []
237
+ files_with, count = 0, 0
238
+ for f in acc["files"]:
239
+ try:
240
+ text = pathlib.Path(f).read_text(encoding="utf-8", errors="replace")
241
+ except OSError as e:
242
+ acc["unreadable"].append({"path": f, "why": e.strerror or str(e)})
243
+ continue
244
+ findings = scan_text(text, d)
245
+ if not findings:
246
+ continue
247
+ files_with += 1
248
+ for fd in findings:
249
+ count += len(fd["lines"])
250
+ row = dict(fd, path=f)
251
+ if fd["verdict"] == "withdrawn":
252
+ withdrawn.append(row)
253
+ elif fd["verdict"] == "no-edition-cited":
254
+ unjudged.append(row)
255
+ else:
256
+ missing.append(row)
257
+
258
+ return {
259
+ "files_read": len(acc["files"]),
260
+ "files_with_a_citation": files_with,
261
+ "citations": count,
262
+ "withdrawn": withdrawn,
263
+ "no_edition_cited": unjudged,
264
+ "not_in_extract": missing,
265
+ "unreadable": acc["unreadable"],
266
+ "skipped_too_large": acc["skipped_too_large"],
267
+ "index_covers": d.get("index_covers", ""),
268
+ "index_built": d.get("built", ""),
269
+ "index_records": d.get("counts", {}).get("shipped"),
270
+ "register": d.get("register", ""),
271
+ "asset_page": d.get("asset_page", ""),
272
+ }
273
+
274
+
275
+ def lookup(designation: str, data: dict | None = None) -> dict:
276
+ """Ask about one designation, typed on the command line."""
277
+ d = data or DATA
278
+ key = norm(designation)
279
+ des, year = split_edition(key)
280
+ c = {"raw": str(designation).strip(), "key": key,
281
+ "designation": des, "year": year, "lines": []}
282
+ c.update(judge(c, d))
283
+ return c