iso-standard-check 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- iso_standard_check-0.1.0/LICENSE +27 -0
- iso_standard_check-0.1.0/PKG-INFO +148 -0
- iso_standard_check-0.1.0/README.md +128 -0
- iso_standard_check-0.1.0/pyproject.toml +37 -0
- iso_standard_check-0.1.0/setup.cfg +4 -0
- iso_standard_check-0.1.0/src/iso_standard_check/__init__.py +283 -0
- iso_standard_check-0.1.0/src/iso_standard_check/__main__.py +155 -0
- iso_standard_check-0.1.0/src/iso_standard_check/data.json +1 -0
- iso_standard_check-0.1.0/src/iso_standard_check.egg-info/PKG-INFO +148 -0
- iso_standard_check-0.1.0/src/iso_standard_check.egg-info/SOURCES.txt +11 -0
- iso_standard_check-0.1.0/src/iso_standard_check.egg-info/dependency_links.txt +1 -0
- iso_standard_check-0.1.0/src/iso_standard_check.egg-info/entry_points.txt +2 -0
- iso_standard_check-0.1.0/src/iso_standard_check.egg-info/top_level.txt +1 -0
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
CC0 1.0 Universal — Public Domain Dedication
|
|
2
|
+
|
|
3
|
+
The person who associated a work with this deed has dedicated the work to the
|
|
4
|
+
public domain by waiving all of his or her rights to the work worldwide under
|
|
5
|
+
copyright law, including all related and neighboring rights, to the extent
|
|
6
|
+
allowed by law.
|
|
7
|
+
|
|
8
|
+
You can copy, modify, distribute and perform the work, even for commercial
|
|
9
|
+
purposes, all without asking permission.
|
|
10
|
+
|
|
11
|
+
Full text: https://creativecommons.org/publicdomain/zero/1.0/legalcode
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
This dedication covers the compilation and the code in this package.
|
|
16
|
+
|
|
17
|
+
The underlying facts are taken from third-party sources, each of which is cited
|
|
18
|
+
by URL in the shipped data alongside the verbatim quote the value was read from:
|
|
19
|
+
|
|
20
|
+
* PyTorch's RELEASE.md and .ci/manywheel/build_cuda.sh — BSD-3-Clause
|
|
21
|
+
(pytorch/pytorch LICENSE)
|
|
22
|
+
* NVIDIA's CUDA Toolkit release notes and CUDA GPUs page — vendor technical
|
|
23
|
+
documentation
|
|
24
|
+
|
|
25
|
+
Facts are not copyrightable. What is reproduced from those sources is a fact
|
|
26
|
+
plus a short attributed quote linking back to it; neither document is
|
|
27
|
+
reproduced.
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: iso-standard-check
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Find ISO and IEC standard citations in a repository and fail CI on the editions ISO's own register marks withdrawn — every verdict with the register row it was read from
|
|
5
|
+
Author: referencesource.org
|
|
6
|
+
License: CC0-1.0
|
|
7
|
+
Project-URL: Homepage, https://referencesource.org/iso-standard-supersessions/
|
|
8
|
+
Project-URL: Source, https://referencesource.org/iso-standard-supersessions/
|
|
9
|
+
Keywords: iso,iec,standards,withdrawn,superseded,iso-9001,iso-13485,iso-14001,iso-27001,compliance,ci,pre-commit,lint,quality-management,references
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: Intended Audience :: Manufacturing
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
15
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
16
|
+
Requires-Python: >=3.8
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
License-File: LICENSE
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# iso-standard-check
|
|
22
|
+
|
|
23
|
+
Find the ISO and IEC standards your repository cites, and fail CI on the ones
|
|
24
|
+
ISO's own register marks **withdrawn or superseded** — each with the register row
|
|
25
|
+
it was read from.
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
npx iso-standard-check . # scan a repository
|
|
29
|
+
npx iso-standard-check docs/ supplier-spec.md # or named paths
|
|
30
|
+
npx iso-standard-check --standard "ISO 9001:2008"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
FAIL ISO 9001:2015 is withdrawn or superseded — docs/quality-plan.md:12, 48
|
|
35
|
+
Quality management systems — Requirements
|
|
36
|
+
the register names ISO 9001:2026 as its replacement
|
|
37
|
+
register row: 62085,IS,,ISO 9001:2015,Quality management systems — Requirements,…,2015-09-22,5,…
|
|
38
|
+
https://referencesource.org/iso-standard-supersessions/iso-9001-2015/
|
|
39
|
+
|
|
40
|
+
? ISO 14001 names no edition year — docs/quality-plan.md:19
|
|
41
|
+
not judged: the register marks 1996, 2004, 2015 withdrawn. Cite the edition
|
|
42
|
+
and this becomes a check.
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Exit **1** if a cited edition is withdrawn. Exit **0** otherwise.
|
|
46
|
+
|
|
47
|
+
## Why this is hard to do by hand
|
|
48
|
+
|
|
49
|
+
**ISO 9001:2015 is no longer current.** ISO published ISO 9001:2026 on
|
|
50
|
+
2026-09-16, so every quality plan, supplier spec and audit checklist in the world
|
|
51
|
+
that cites the 2015 edition now cites a superseded one. That is the ordinary case
|
|
52
|
+
this checker exists for, and nothing in a repository tells you it happened.
|
|
53
|
+
|
|
54
|
+
**The edition year is the whole difficulty.** Every one of the 24,640
|
|
55
|
+
identifiers in ISO's register is of the form `ISO 9001:2008`. ISO 9001:2008 is
|
|
56
|
+
withdrawn; its replacement is not. So a document that writes just **"ISO 9001"**
|
|
57
|
+
cannot be judged withdrawn or current by anybody, this program included. Those
|
|
58
|
+
citations are printed as *unjudged*, with the editions of that designation the
|
|
59
|
+
register does mark withdrawn, and **they never fail your build**. Call it with
|
|
60
|
+
`--strict` if you would rather be made to write the edition down.
|
|
61
|
+
|
|
62
|
+
## What the index covers, and what a silence means
|
|
63
|
+
|
|
64
|
+
The shipped data holds deliverables ISO's register marks withdrawn or
|
|
65
|
+
superseded, **and nothing else**.
|
|
66
|
+
|
|
67
|
+
So a hit is a measurement and a miss is not. A designation this program does not
|
|
68
|
+
report may be current, may be withdrawn in a later edition of the register than
|
|
69
|
+
the one we read, or may not exist. It is reported as *not in this extract*, and
|
|
70
|
+
it is never printed as "current". If you need "is this one current?", that is a
|
|
71
|
+
different question and this is not the tool that answers it.
|
|
72
|
+
|
|
73
|
+
Two things ISO's register does not give, so neither does this package:
|
|
74
|
+
|
|
75
|
+
- **The date a standard was withdrawn.** The only date in the file is the
|
|
76
|
+
deliverable's own publication date. It is printed as `published` and never as a
|
|
77
|
+
withdrawal date.
|
|
78
|
+
- **Any guarantee the named replacement exists as a document.** 2,507 of the
|
|
79
|
+
24,640 records (10.2%) name at least one replacement the register gives no
|
|
80
|
+
publication date; on 416 (1.7%) *every* named replacement is undated — a
|
|
81
|
+
registered revision project you may not be able to obtain. Both cases say so
|
|
82
|
+
in the output rather than handing you a designation that cannot be bought.
|
|
83
|
+
|
|
84
|
+
## Where the numbers come from
|
|
85
|
+
|
|
86
|
+
Every verdict is read from one file: ISO's own open-data deliverables register,
|
|
87
|
+
|
|
88
|
+
```
|
|
89
|
+
https://isopublicstorageprod.blob.core.windows.net/opendata/_latest/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
published by ISO under the Open Data Commons Attribution License v1.0, which
|
|
93
|
+
permits reuse with attribution. Each of the 24,640 records carries the CSV row it
|
|
94
|
+
was read from, verbatim, and is published on its own page with that row visible:
|
|
95
|
+
|
|
96
|
+
**<https://referencesource.org/out/pkg/iso-standard-check/>**
|
|
97
|
+
|
|
98
|
+
Nothing in this package computes, normalises or infers a status. It reads what
|
|
99
|
+
the register states, and where the register states nothing it says so in words.
|
|
100
|
+
The records are re-checked against the live file on a schedule; the index here
|
|
101
|
+
carries the date it was built.
|
|
102
|
+
|
|
103
|
+
## As a pre-commit hook
|
|
104
|
+
|
|
105
|
+
The Python twin on PyPI ships a hook:
|
|
106
|
+
|
|
107
|
+
```yaml
|
|
108
|
+
repos:
|
|
109
|
+
- repo: https://github.com/referencesource/iso-standard-check
|
|
110
|
+
rev: v0.1.0
|
|
111
|
+
hooks:
|
|
112
|
+
- id: iso-standard-check
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Or `pipx run iso-standard-check .`
|
|
116
|
+
|
|
117
|
+
## In a GitHub Actions workflow
|
|
118
|
+
|
|
119
|
+
```yaml
|
|
120
|
+
- name: ISO citations
|
|
121
|
+
run: npx --yes iso-standard-check docs/ specs/
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Options
|
|
125
|
+
|
|
126
|
+
| | |
|
|
127
|
+
|---|---|
|
|
128
|
+
| `<paths>` | files or directories; defaults to `.` |
|
|
129
|
+
| `--standard "<id>"` | ask about one designation and exit |
|
|
130
|
+
| `--json` | machine-readable, including every line number |
|
|
131
|
+
| `--quiet` | withdrawn citations only, no unjudged ones |
|
|
132
|
+
| `--strict` | also exit 1 when a citation names no edition year |
|
|
133
|
+
|
|
134
|
+
Caches, virtualenvs, `node_modules` and build output are not walked: a citation
|
|
135
|
+
inside something you installed is a fact about somebody else's repository.
|
|
136
|
+
|
|
137
|
+
## API
|
|
138
|
+
|
|
139
|
+
```js
|
|
140
|
+
const iso = require('iso-standard-check');
|
|
141
|
+
iso.lookup('ISO 13485:2003'); // one designation
|
|
142
|
+
iso.scanText(readmeText); // findings with line numbers
|
|
143
|
+
iso.scanPaths(['docs']); // grouped by verdict
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
CC0-1.0. Built by an automated pipeline at
|
|
147
|
+
[referencesource.org](https://referencesource.org/iso-standard-supersessions/);
|
|
148
|
+
the facts are ISO's and are attributed to ISO on every line.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# iso-standard-check
|
|
2
|
+
|
|
3
|
+
Find the ISO and IEC standards your repository cites, and fail CI on the ones
|
|
4
|
+
ISO's own register marks **withdrawn or superseded** — each with the register row
|
|
5
|
+
it was read from.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npx iso-standard-check . # scan a repository
|
|
9
|
+
npx iso-standard-check docs/ supplier-spec.md # or named paths
|
|
10
|
+
npx iso-standard-check --standard "ISO 9001:2008"
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
```
|
|
14
|
+
FAIL ISO 9001:2015 is withdrawn or superseded — docs/quality-plan.md:12, 48
|
|
15
|
+
Quality management systems — Requirements
|
|
16
|
+
the register names ISO 9001:2026 as its replacement
|
|
17
|
+
register row: 62085,IS,,ISO 9001:2015,Quality management systems — Requirements,…,2015-09-22,5,…
|
|
18
|
+
https://referencesource.org/iso-standard-supersessions/iso-9001-2015/
|
|
19
|
+
|
|
20
|
+
? ISO 14001 names no edition year — docs/quality-plan.md:19
|
|
21
|
+
not judged: the register marks 1996, 2004, 2015 withdrawn. Cite the edition
|
|
22
|
+
and this becomes a check.
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Exit **1** if a cited edition is withdrawn. Exit **0** otherwise.
|
|
26
|
+
|
|
27
|
+
## Why this is hard to do by hand
|
|
28
|
+
|
|
29
|
+
**ISO 9001:2015 is no longer current.** ISO published ISO 9001:2026 on
|
|
30
|
+
2026-09-16, so every quality plan, supplier spec and audit checklist in the world
|
|
31
|
+
that cites the 2015 edition now cites a superseded one. That is the ordinary case
|
|
32
|
+
this checker exists for, and nothing in a repository tells you it happened.
|
|
33
|
+
|
|
34
|
+
**The edition year is the whole difficulty.** Every one of the 24,640
|
|
35
|
+
identifiers in ISO's register is of the form `ISO 9001:2008`. ISO 9001:2008 is
|
|
36
|
+
withdrawn; its replacement is not. So a document that writes just **"ISO 9001"**
|
|
37
|
+
cannot be judged withdrawn or current by anybody, this program included. Those
|
|
38
|
+
citations are printed as *unjudged*, with the editions of that designation the
|
|
39
|
+
register does mark withdrawn, and **they never fail your build**. Call it with
|
|
40
|
+
`--strict` if you would rather be made to write the edition down.
|
|
41
|
+
|
|
42
|
+
## What the index covers, and what a silence means
|
|
43
|
+
|
|
44
|
+
The shipped data holds deliverables ISO's register marks withdrawn or
|
|
45
|
+
superseded, **and nothing else**.
|
|
46
|
+
|
|
47
|
+
So a hit is a measurement and a miss is not. A designation this program does not
|
|
48
|
+
report may be current, may be withdrawn in a later edition of the register than
|
|
49
|
+
the one we read, or may not exist. It is reported as *not in this extract*, and
|
|
50
|
+
it is never printed as "current". If you need "is this one current?", that is a
|
|
51
|
+
different question and this is not the tool that answers it.
|
|
52
|
+
|
|
53
|
+
Two things ISO's register does not give, so neither does this package:
|
|
54
|
+
|
|
55
|
+
- **The date a standard was withdrawn.** The only date in the file is the
|
|
56
|
+
deliverable's own publication date. It is printed as `published` and never as a
|
|
57
|
+
withdrawal date.
|
|
58
|
+
- **Any guarantee the named replacement exists as a document.** 2,507 of the
|
|
59
|
+
24,640 records (10.2%) name at least one replacement the register gives no
|
|
60
|
+
publication date; on 416 (1.7%) *every* named replacement is undated — a
|
|
61
|
+
registered revision project you may not be able to obtain. Both cases say so
|
|
62
|
+
in the output rather than handing you a designation that cannot be bought.
|
|
63
|
+
|
|
64
|
+
## Where the numbers come from
|
|
65
|
+
|
|
66
|
+
Every verdict is read from one file: ISO's own open-data deliverables register,
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
https://isopublicstorageprod.blob.core.windows.net/opendata/_latest/iso_deliverables_metadata/csv/iso_deliverables_metadata.csv
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
published by ISO under the Open Data Commons Attribution License v1.0, which
|
|
73
|
+
permits reuse with attribution. Each of the 24,640 records carries the CSV row it
|
|
74
|
+
was read from, verbatim, and is published on its own page with that row visible:
|
|
75
|
+
|
|
76
|
+
**<https://referencesource.org/out/pkg/iso-standard-check/>**
|
|
77
|
+
|
|
78
|
+
Nothing in this package computes, normalises or infers a status. It reads what
|
|
79
|
+
the register states, and where the register states nothing it says so in words.
|
|
80
|
+
The records are re-checked against the live file on a schedule; the index here
|
|
81
|
+
carries the date it was built.
|
|
82
|
+
|
|
83
|
+
## As a pre-commit hook
|
|
84
|
+
|
|
85
|
+
The Python twin on PyPI ships a hook:
|
|
86
|
+
|
|
87
|
+
```yaml
|
|
88
|
+
repos:
|
|
89
|
+
- repo: https://github.com/referencesource/iso-standard-check
|
|
90
|
+
rev: v0.1.0
|
|
91
|
+
hooks:
|
|
92
|
+
- id: iso-standard-check
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
Or `pipx run iso-standard-check .`
|
|
96
|
+
|
|
97
|
+
## In a GitHub Actions workflow
|
|
98
|
+
|
|
99
|
+
```yaml
|
|
100
|
+
- name: ISO citations
|
|
101
|
+
run: npx --yes iso-standard-check docs/ specs/
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## Options
|
|
105
|
+
|
|
106
|
+
| | |
|
|
107
|
+
|---|---|
|
|
108
|
+
| `<paths>` | files or directories; defaults to `.` |
|
|
109
|
+
| `--standard "<id>"` | ask about one designation and exit |
|
|
110
|
+
| `--json` | machine-readable, including every line number |
|
|
111
|
+
| `--quiet` | withdrawn citations only, no unjudged ones |
|
|
112
|
+
| `--strict` | also exit 1 when a citation names no edition year |
|
|
113
|
+
|
|
114
|
+
Caches, virtualenvs, `node_modules` and build output are not walked: a citation
|
|
115
|
+
inside something you installed is a fact about somebody else's repository.
|
|
116
|
+
|
|
117
|
+
## API
|
|
118
|
+
|
|
119
|
+
```js
|
|
120
|
+
const iso = require('iso-standard-check');
|
|
121
|
+
iso.lookup('ISO 13485:2003'); // one designation
|
|
122
|
+
iso.scanText(readmeText); // findings with line numbers
|
|
123
|
+
iso.scanPaths(['docs']); // grouped by verdict
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
CC0-1.0. Built by an automated pipeline at
|
|
127
|
+
[referencesource.org](https://referencesource.org/iso-standard-supersessions/);
|
|
128
|
+
the facts are ISO's and are attributed to ISO on every line.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "iso-standard-check"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Find ISO and IEC standard citations in a repository and fail CI on the editions ISO's own register marks withdrawn — every verdict with the register row it was read from"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "CC0-1.0" }
|
|
12
|
+
authors = [{ name = "referencesource.org" }]
|
|
13
|
+
keywords = ["iso", "iec", "standards", "withdrawn", "superseded", "iso-9001",
|
|
14
|
+
"iso-13485", "iso-14001", "iso-27001", "compliance", "ci",
|
|
15
|
+
"pre-commit", "lint", "quality-management", "references"]
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Intended Audience :: Manufacturing",
|
|
20
|
+
"Programming Language :: Python :: 3",
|
|
21
|
+
"Topic :: Software Development :: Quality Assurance",
|
|
22
|
+
"Topic :: Text Processing :: Linguistic",
|
|
23
|
+
]
|
|
24
|
+
dependencies = []
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
Homepage = "https://referencesource.org/iso-standard-supersessions/"
|
|
28
|
+
Source = "https://referencesource.org/iso-standard-supersessions/"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
iso-standard-check = "iso_standard_check.__main__:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
iso_standard_check = ["data.json"]
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""iso-standard-check — find ISO and IEC standard citations in a repository and
|
|
2
|
+
say which editions ISO's own register marks withdrawn or superseded.
|
|
3
|
+
|
|
4
|
+
The Python twin of the npm package of the same name. Both read the same
|
|
5
|
+
`data.json`, built from our records by `ops/packages/iso-standard-check/build_data.py`,
|
|
6
|
+
and `ops/iso-standard-check-selftest.py` checks that the two implementations of
|
|
7
|
+
the key rule agree -- a second copy of a rule is a second place for it to drift.
|
|
8
|
+
|
|
9
|
+
WHAT THIS INDEX COVERS, because it decides what the program may say. The shipped
|
|
10
|
+
data holds deliverables the register marks withdrawn or superseded, and nothing
|
|
11
|
+
else. So a hit is a measurement and a miss is not: a designation absent from the
|
|
12
|
+
index may be current, may be withdrawn in a later edition of the register than
|
|
13
|
+
the one we read, or may not exist. A miss is reported as `not-in-extract` and is
|
|
14
|
+
never printed as "current".
|
|
15
|
+
|
|
16
|
+
THE EDITION YEAR IS THE WHOLE DIFFICULTY. Every one of the 24,640 identifiers in
|
|
17
|
+
the register is of the form `ISO 9001:2008`. ISO 9001:2008 is withdrawn and
|
|
18
|
+
ISO 9001:2026 superseded ISO 9001:2015 on 2026-09-16, so a document citing
|
|
19
|
+
"ISO 9001" with no edition cannot be judged by anybody, us included. Those are
|
|
20
|
+
reported as `no-edition-cited`, with the editions of that family the register
|
|
21
|
+
does mark withdrawn, and they never fail a build.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import os
|
|
27
|
+
import pathlib
|
|
28
|
+
import re
|
|
29
|
+
|
|
30
|
+
__version__ = "0.1.0"
|
|
31
|
+
|
|
32
|
+
DATA_PATH = pathlib.Path(__file__).resolve().parent / "data.json"
|
|
33
|
+
with DATA_PATH.open(encoding="utf-8") as _fh:
|
|
34
|
+
DATA = json.load(_fh)
|
|
35
|
+
|
|
36
|
+
SKIP_DIRS = frozenset({
|
|
37
|
+
# GENERATED from ops/packages/skip-dirs.json by ops/packages/sync_skip_dirs.py.
|
|
38
|
+
# Do not edit here: add the directory to that file and run the program,
|
|
39
|
+
# or ops/package-skip-dirs-selftest.py fails on the copy you missed.
|
|
40
|
+
# not your code
|
|
41
|
+
"node_modules", "vendor", "bower_components", "site-packages", ".git",
|
|
42
|
+
".svn", ".hg",
|
|
43
|
+
# build and test output
|
|
44
|
+
"dist", "build", "out", "target", "coverage", ".next", ".nuxt",
|
|
45
|
+
".svelte-kit", ".angular", ".dart_tool", ".serverless", ".vercel",
|
|
46
|
+
".netlify",
|
|
47
|
+
# virtualenvs and installed toolchains
|
|
48
|
+
".venv", "venv", "env", ".direnv", ".conda", ".pyenv", ".nvm",
|
|
49
|
+
".rustup", ".stack-work",
|
|
50
|
+
# package manager and tool caches
|
|
51
|
+
"__pycache__", ".cache", ".tox", ".nox", ".eggs", ".mypy_cache",
|
|
52
|
+
".pytest_cache", ".ruff_cache", ".ipynb_checkpoints", ".yarn",
|
|
53
|
+
".pnpm-store", ".pnpm", ".npm", ".bundle", ".gem", ".cpanm", ".m2",
|
|
54
|
+
".nuget", ".gradle", ".cargo", ".pub-cache", ".turbo",
|
|
55
|
+
".parcel-cache", ".sass-cache", ".terraform", ".terragrunt-cache",
|
|
56
|
+
".dvc",
|
|
57
|
+
# editor and shell state
|
|
58
|
+
".idea", ".history", ".local",
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
TEXT_EXT = frozenset({
|
|
62
|
+
'.md', '.markdown', '.rst', '.adoc', '.asciidoc', '.txt', '.text',
|
|
63
|
+
'.yaml', '.yml', '.json', '.toml', '.ini', '.cfg', '.csv', '.tsv',
|
|
64
|
+
'.html', '.htm', '.xml', '.tex', '.org',
|
|
65
|
+
'.py', '.js', '.mjs', '.cjs', '.ts', '.tsx', '.java', '.kt', '.cs',
|
|
66
|
+
'.c', '.h', '.cc', '.cpp', '.hpp', '.go', '.rs', '.rb', '.php', '.sql',
|
|
67
|
+
'.sh', '.ps1',
|
|
68
|
+
})
|
|
69
|
+
|
|
70
|
+
MAX_BYTES = 4 * 1024 * 1024
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
# ------------------------------------------------------------------ keys
|
|
74
|
+
|
|
75
|
+
def norm(text: str) -> str:
|
|
76
|
+
"""A designation reduced to what a citation in a document can be matched on.
|
|
77
|
+
|
|
78
|
+
The same rule as the builder's `norm` and the npm package's: case and the
|
|
79
|
+
punctuation that separates words go, the digits, the hyphens inside a part
|
|
80
|
+
number and the edition year stay exactly.
|
|
81
|
+
"""
|
|
82
|
+
s = (text or "").lower()
|
|
83
|
+
s = s.replace("–", "-").replace("—", "-")
|
|
84
|
+
s = re.sub(r'\s*:\s*', ':', s)
|
|
85
|
+
s = re.sub(r'[/ ]+', ' ', s)
|
|
86
|
+
s = re.sub(r'\s+', ' ', s).strip()
|
|
87
|
+
s = re.sub(r'(?<=[a-z])-(?=[a-z])', ' ', s)
|
|
88
|
+
return s
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def split_edition(key: str):
|
|
92
|
+
"""(designation, year) -- year is "" when the citation names no edition."""
|
|
93
|
+
m = re.match(r'^(.+?):(\d{4})$', key)
|
|
94
|
+
return (m.group(1), m.group(2)) if m else (key, "")
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def slug_id(identifier: str) -> str:
|
|
98
|
+
"""`IEC 31010:2009` -> `iec-31010-2009`, our own record id rule."""
|
|
99
|
+
return re.sub(r'[^0-9a-z]+', '-', (identifier or "").lower()).strip('-')
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def page_for(rec: dict, data: dict | None = None) -> str:
|
|
103
|
+
d = data or DATA
|
|
104
|
+
return f"{d['site']}/{d['asset']}/{rec.get('id') or slug_id(rec['identifier'])}/"
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ------------------------------------------------------------- the scanner
|
|
108
|
+
|
|
109
|
+
def prefixes(data: dict | None = None):
|
|
110
|
+
"""The body-and-type prefixes the index can speak about, longest first.
|
|
111
|
+
|
|
112
|
+
Read off the shipped data's own `families` keys rather than typed here, so
|
|
113
|
+
the program never looks for a shape of designation it holds no records of.
|
|
114
|
+
"""
|
|
115
|
+
d = data or DATA
|
|
116
|
+
seen = set()
|
|
117
|
+
for family in d.get("families", {}):
|
|
118
|
+
words = []
|
|
119
|
+
for tok in family.split(" "):
|
|
120
|
+
if any(c.isdigit() for c in tok):
|
|
121
|
+
break
|
|
122
|
+
words.append(tok)
|
|
123
|
+
if words:
|
|
124
|
+
seen.add(" ".join(words))
|
|
125
|
+
return sorted(seen, key=lambda p: (-len(p), p))
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def citation_regex(data: dict | None = None):
|
|
129
|
+
alts = ["[\\s/._-]*".join(re.escape(w) for w in p.split(" "))
|
|
130
|
+
for p in prefixes(data)]
|
|
131
|
+
return re.compile(
|
|
132
|
+
r'\b(?:' + "|".join(alts) + r')[\s/._-]+'
|
|
133
|
+
r'\d+(?:[-–—]\d+)*\s*(?::\s*\d{4})?',
|
|
134
|
+
re.I)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
_RX = None
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def citations(text: str, data: dict | None = None):
|
|
141
|
+
"""Every ISO or IEC citation in a block of text, in the order it appears."""
|
|
142
|
+
global _RX
|
|
143
|
+
d = data or DATA
|
|
144
|
+
if d is DATA:
|
|
145
|
+
if _RX is None:
|
|
146
|
+
_RX = citation_regex(d)
|
|
147
|
+
rx = _RX
|
|
148
|
+
else:
|
|
149
|
+
rx = citation_regex(d)
|
|
150
|
+
out = []
|
|
151
|
+
for m in rx.finditer(text):
|
|
152
|
+
raw = m.group(0).strip().rstrip('.,;:)]')
|
|
153
|
+
key = norm(raw)
|
|
154
|
+
designation, year = split_edition(key)
|
|
155
|
+
out.append({
|
|
156
|
+
"raw": raw, "key": key, "designation": designation, "year": year,
|
|
157
|
+
"line": text.count("\n", 0, m.start()) + 1,
|
|
158
|
+
})
|
|
159
|
+
return out
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def judge(citation: dict, data: dict | None = None) -> dict:
|
|
163
|
+
"""What the register says about one citation. Never a guess."""
|
|
164
|
+
d = data or DATA
|
|
165
|
+
rec = d.get("withdrawn", {}).get(citation["key"])
|
|
166
|
+
if rec:
|
|
167
|
+
out = dict(rec)
|
|
168
|
+
out["page"] = page_for(rec, d)
|
|
169
|
+
return {"verdict": "withdrawn", "record": out}
|
|
170
|
+
editions = d.get("families", {}).get(citation["designation"], [])
|
|
171
|
+
if not citation["year"]:
|
|
172
|
+
return {"verdict": "no-edition-cited", "withdrawn_editions": editions}
|
|
173
|
+
return {"verdict": "not-in-extract", "withdrawn_editions": editions}
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def scan_text(text: str, data: dict | None = None):
|
|
177
|
+
"""One finding per citation, deduplicated by key, every line it appears on."""
|
|
178
|
+
d = data or DATA
|
|
179
|
+
by_key: dict = {}
|
|
180
|
+
for c in citations(text, d):
|
|
181
|
+
prev = by_key.get(c["key"])
|
|
182
|
+
if prev:
|
|
183
|
+
prev["lines"].append(c["line"])
|
|
184
|
+
continue
|
|
185
|
+
row = {k: c[k] for k in ("raw", "key", "designation", "year")}
|
|
186
|
+
row["lines"] = [c["line"]]
|
|
187
|
+
row.update(judge(c, d))
|
|
188
|
+
by_key[c["key"]] = row
|
|
189
|
+
return list(by_key.values())
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
# ------------------------------------------------------------------ files
|
|
193
|
+
|
|
194
|
+
def _walk(target: str, acc: dict, depth: int = 0) -> None:
|
|
195
|
+
try:
|
|
196
|
+
st = os.stat(target)
|
|
197
|
+
except OSError as e:
|
|
198
|
+
acc["unreadable"].append({"path": target, "why": e.strerror or str(e)})
|
|
199
|
+
return
|
|
200
|
+
if os.path.isdir(target):
|
|
201
|
+
if depth > 24:
|
|
202
|
+
return
|
|
203
|
+
try:
|
|
204
|
+
names = sorted(os.listdir(target))
|
|
205
|
+
except OSError as e:
|
|
206
|
+
acc["unreadable"].append({"path": target, "why": e.strerror or str(e)})
|
|
207
|
+
return
|
|
208
|
+
for name in names:
|
|
209
|
+
p = os.path.join(target, name)
|
|
210
|
+
if os.path.isdir(p):
|
|
211
|
+
if name not in SKIP_DIRS:
|
|
212
|
+
_walk(p, acc, depth + 1)
|
|
213
|
+
elif os.path.isfile(p):
|
|
214
|
+
if os.path.splitext(name)[1].lower() in TEXT_EXT:
|
|
215
|
+
_walk(p, acc, depth + 1)
|
|
216
|
+
return
|
|
217
|
+
if not os.path.isfile(target):
|
|
218
|
+
return
|
|
219
|
+
if st.st_size > MAX_BYTES:
|
|
220
|
+
acc["skipped_too_large"].append({"path": target, "bytes": st.st_size})
|
|
221
|
+
return
|
|
222
|
+
acc["files"].append(target)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def scan_paths(paths, data: dict | None = None) -> dict:
|
|
226
|
+
"""Scan files and directories.
|
|
227
|
+
|
|
228
|
+
A file that cannot be read is reported in `unreadable`, never counted as a
|
|
229
|
+
file with no citations in it: those two are different facts.
|
|
230
|
+
"""
|
|
231
|
+
d = data or DATA
|
|
232
|
+
acc = {"files": [], "unreadable": [], "skipped_too_large": []}
|
|
233
|
+
for p in paths:
|
|
234
|
+
_walk(p, acc)
|
|
235
|
+
|
|
236
|
+
withdrawn, unjudged, missing = [], [], []
|
|
237
|
+
files_with, count = 0, 0
|
|
238
|
+
for f in acc["files"]:
|
|
239
|
+
try:
|
|
240
|
+
text = pathlib.Path(f).read_text(encoding="utf-8", errors="replace")
|
|
241
|
+
except OSError as e:
|
|
242
|
+
acc["unreadable"].append({"path": f, "why": e.strerror or str(e)})
|
|
243
|
+
continue
|
|
244
|
+
findings = scan_text(text, d)
|
|
245
|
+
if not findings:
|
|
246
|
+
continue
|
|
247
|
+
files_with += 1
|
|
248
|
+
for fd in findings:
|
|
249
|
+
count += len(fd["lines"])
|
|
250
|
+
row = dict(fd, path=f)
|
|
251
|
+
if fd["verdict"] == "withdrawn":
|
|
252
|
+
withdrawn.append(row)
|
|
253
|
+
elif fd["verdict"] == "no-edition-cited":
|
|
254
|
+
unjudged.append(row)
|
|
255
|
+
else:
|
|
256
|
+
missing.append(row)
|
|
257
|
+
|
|
258
|
+
return {
|
|
259
|
+
"files_read": len(acc["files"]),
|
|
260
|
+
"files_with_a_citation": files_with,
|
|
261
|
+
"citations": count,
|
|
262
|
+
"withdrawn": withdrawn,
|
|
263
|
+
"no_edition_cited": unjudged,
|
|
264
|
+
"not_in_extract": missing,
|
|
265
|
+
"unreadable": acc["unreadable"],
|
|
266
|
+
"skipped_too_large": acc["skipped_too_large"],
|
|
267
|
+
"index_covers": d.get("index_covers", ""),
|
|
268
|
+
"index_built": d.get("built", ""),
|
|
269
|
+
"index_records": d.get("counts", {}).get("shipped"),
|
|
270
|
+
"register": d.get("register", ""),
|
|
271
|
+
"asset_page": d.get("asset_page", ""),
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def lookup(designation: str, data: dict | None = None) -> dict:
|
|
276
|
+
"""Ask about one designation, typed on the command line."""
|
|
277
|
+
d = data or DATA
|
|
278
|
+
key = norm(designation)
|
|
279
|
+
des, year = split_edition(key)
|
|
280
|
+
c = {"raw": str(designation).strip(), "key": key,
|
|
281
|
+
"designation": des, "year": year, "lines": []}
|
|
282
|
+
c.update(judge(c, d))
|
|
283
|
+
return c
|