claims-cli 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claims/__init__.py +132 -0
- claims/__main__.py +5 -0
- claims/adapters/__init__.py +6 -0
- claims/adapters/array.py +212 -0
- claims/adapters/base.py +75 -0
- claims/adapters/cells.py +94 -0
- claims/adapters/columnar.py +188 -0
- claims/adapters/prose.py +261 -0
- claims/adapters/reads.py +47 -0
- claims/adapters/rows.py +116 -0
- claims/adapters/sheet.py +149 -0
- claims/adapters/sqlite.py +73 -0
- claims/adapters/table.py +135 -0
- claims/adapters/tree.py +163 -0
- claims/checks.py +511 -0
- claims/cli.py +1103 -0
- claims/core/__init__.py +9 -0
- claims/core/artifacts.py +121 -0
- claims/core/checks.py +132 -0
- claims/core/claims.py +100 -0
- claims/core/evidence.py +708 -0
- claims/core/manifest.py +349 -0
- claims/core/numbers.py +146 -0
- claims/core/outcomes.py +410 -0
- claims/core/report.py +503 -0
- claims/corpus.py +344 -0
- claims/exceptions.py +106 -0
- claims/export/__init__.py +23 -0
- claims/export/rocrate.py +787 -0
- claims/export/sarif.py +401 -0
- claims/manifest.py +59 -0
- claims/models.py +222 -0
- claims/numbers.py +453 -0
- claims/origins.py +303 -0
- claims/policy.py +369 -0
- claims/provenance.py +42 -0
- claims/record.py +1099 -0
- claims/regression.py +171 -0
- claims/renderers/__init__.py +10 -0
- claims/renderers/sarif.py +294 -0
- claims/reruns.py +111 -0
- claims/resolve.py +116 -0
- claims/stale.py +333 -0
- claims/starter.py +432 -0
- claims/toolchain.py +120 -0
- claims/verify.py +1371 -0
- claims/why.py +398 -0
- claims_cli-0.6.0.dist-info/METADATA +139 -0
- claims_cli-0.6.0.dist-info/RECORD +51 -0
- claims_cli-0.6.0.dist-info/WHEEL +4 -0
- claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/record.py
ADDED
|
@@ -0,0 +1,1099 @@
|
|
|
1
|
+
"""One checked assertion, as a record another program can read.
|
|
2
|
+
|
|
3
|
+
`repro verify` prints a report and `repro why` an account, and each is shaped for its reader.
|
|
4
|
+
This is the same evaluation stated once per assertion and for no reader in particular: the
|
|
5
|
+
checker and its version read this file, with this digest, at this address, obtained this
|
|
6
|
+
value, compared it with the value the manuscript reports under this rule, and returned this
|
|
7
|
+
outcome. An exporter to another format reads these records and nothing else, so a fact an
|
|
8
|
+
exporter needs is added here and reaches every format.
|
|
9
|
+
|
|
10
|
+
Nothing is evaluated here that `repro verify` does not evaluate. Each assertion goes through
|
|
11
|
+
`claims.verify.check_evidence`, each pin through `claims.verify.artifact_states`, and what is
|
|
12
|
+
known of the commands upstream comes from `claims.stale.stale` and the record of re-runs.
|
|
13
|
+
Executes nothing and writes nothing.
|
|
14
|
+
|
|
15
|
+
**Experimental.** The version is in every record, and a field may change before `1.0`.
|
|
16
|
+
|
|
17
|
+
A conversion, where the file stores a fraction and the manuscript prints a percentage, is a
|
|
18
|
+
second value beside `stored` on the file read: `converted`, which is then the one compared.
|
|
19
|
+
An aggregate, a statistic over several runs, is one entry in `reads` per member, each with
|
|
20
|
+
its own `stored`, and the statistic and what it computed in `comparison`.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import collections
|
|
26
|
+
import datetime
|
|
27
|
+
import enum
|
|
28
|
+
import json
|
|
29
|
+
import pathlib
|
|
30
|
+
import re
|
|
31
|
+
from collections.abc import Iterable, Mapping
|
|
32
|
+
from typing import Any
|
|
33
|
+
|
|
34
|
+
import yaml
|
|
35
|
+
from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, ValidationError
|
|
36
|
+
|
|
37
|
+
from claims.adapters.reads import reading_once
|
|
38
|
+
from claims.exceptions import ReproError
|
|
39
|
+
from claims.models import (
|
|
40
|
+
AggregateEvidence,
|
|
41
|
+
ArtifactState,
|
|
42
|
+
Claim,
|
|
43
|
+
ComparisonMode,
|
|
44
|
+
ComparisonStatus,
|
|
45
|
+
CorrespondenceEvidence,
|
|
46
|
+
Decision,
|
|
47
|
+
Digest,
|
|
48
|
+
Evidence,
|
|
49
|
+
ExecutionStatus,
|
|
50
|
+
ExtractionStatus,
|
|
51
|
+
Freshness,
|
|
52
|
+
Manifest,
|
|
53
|
+
MetricEvidence,
|
|
54
|
+
Origin,
|
|
55
|
+
Outcome,
|
|
56
|
+
QuoteEvidence,
|
|
57
|
+
Reason,
|
|
58
|
+
Regeneration,
|
|
59
|
+
RegenerationState,
|
|
60
|
+
StalenessReport,
|
|
61
|
+
TableCellEvidence,
|
|
62
|
+
Validity,
|
|
63
|
+
ValueEvidence,
|
|
64
|
+
)
|
|
65
|
+
from claims.reruns import reruns, upstream
|
|
66
|
+
from claims.resolve import stored
|
|
67
|
+
from claims.stale import stale
|
|
68
|
+
from claims.toolchain import adapter_version
|
|
69
|
+
from claims.verify import (
|
|
70
|
+
DEFAULT_BACKENDS,
|
|
71
|
+
Backend,
|
|
72
|
+
artifact_states,
|
|
73
|
+
check_evidence,
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
RECORD_VERSION = "reproducible.science/claim-check/0.1-experimental"
|
|
77
|
+
"""The version every record written here carries. A common format for claim checks, named
|
|
78
|
+
under the domain that publishes its schema and its RO-Crate profile."""
|
|
79
|
+
|
|
80
|
+
LEGACY_RECORD_VERSION = "repro.check/0.1-experimental"
|
|
81
|
+
"""The version 0.5.6 wrote, for the same format under its earlier name. Read throughout the
|
|
82
|
+
0.6 series and never written. The next minor release after 0.6 drops it, and a record that
|
|
83
|
+
carries it is then a record of a version this reader does not know."""
|
|
84
|
+
|
|
85
|
+
#: Every version a record may carry and be read here.
|
|
86
|
+
READ_RECORD_VERSIONS = (RECORD_VERSION, LEGACY_RECORD_VERSION)
|
|
87
|
+
|
|
88
|
+
#: Where the JSON Schema of the record is published. `docs/schemas/` holds the file and the
|
|
89
|
+
#: site serves a copy of it under `/schemas/`.
|
|
90
|
+
SCHEMA_URL = "https://reproducible.science/schemas/claim-check-0.1-experimental.schema.json"
|
|
91
|
+
|
|
92
|
+
CHECKER = "repro"
|
|
93
|
+
|
|
94
|
+
#: Where the checker is described. A record names it so a reader can find out what the tool
|
|
95
|
+
#: is without knowing the name already.
|
|
96
|
+
CHECKER_URL = "https://github.com/elliottower/reproducible-science"
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class CheckType(enum.StrEnum):
|
|
100
|
+
"""Which proposition a record is about. Closed, and defined once, in SPEC 7.10.
|
|
101
|
+
|
|
102
|
+
Two tools can both say `verified` and have checked different things, so a record names
|
|
103
|
+
the proposition as well as the outcome. The first three are the ones this tool checks. The
|
|
104
|
+
rest are reserved for a record written from another tool's result, and this tool emits
|
|
105
|
+
none of them.
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
VALUE_COMPARISON = "value-comparison"
|
|
109
|
+
"""A value the manuscript prints, against the value at an address in a file."""
|
|
110
|
+
PASSAGE_PRESENCE = "passage-presence"
|
|
111
|
+
"""A passage occurs in a file."""
|
|
112
|
+
CORRESPONDENCE = "correspondence"
|
|
113
|
+
"""The values at two addresses, in two files, agree."""
|
|
114
|
+
AGGREGATE_COMPARISON = "aggregate-comparison"
|
|
115
|
+
"""A value the manuscript prints, against a statistic computed over the values at
|
|
116
|
+
several addresses. No one stored value is the value compared."""
|
|
117
|
+
ARTIFACT_IDENTITY = "artifact-identity"
|
|
118
|
+
"""A file's bytes are the bytes that were recorded. No value in it is compared."""
|
|
119
|
+
LINEAGE_CHECK = "lineage-check"
|
|
120
|
+
"""A file was produced by a recorded process from recorded inputs."""
|
|
121
|
+
RERUN_COMPARISON = "rerun-comparison"
|
|
122
|
+
"""What a command wrote when run again, against what it wrote before."""
|
|
123
|
+
COVERAGE_CHECK = "coverage-check"
|
|
124
|
+
"""Whether something the manuscript states has a claim declared for it."""
|
|
125
|
+
OTHER = "other"
|
|
126
|
+
"""A proposition none of the above names. Never read as a comparison of values."""
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class NormalizedOutcome(enum.StrEnum):
|
|
130
|
+
"""What a check found, in five words defined by their preconditions. SPEC 7.10.
|
|
131
|
+
|
|
132
|
+
The emitting tool's own word travels beside it in `native_outcome`.
|
|
133
|
+
"""
|
|
134
|
+
|
|
135
|
+
MATCH = "match"
|
|
136
|
+
"""The file was read, the address resolved, a comparison ran, and the two agreed."""
|
|
137
|
+
MISMATCH = "mismatch"
|
|
138
|
+
"""The file was read, the address resolved, a comparison ran, and the two differed."""
|
|
139
|
+
NOT_FOUND = "not-found"
|
|
140
|
+
"""The file was readable and the address did not resolve to exactly one value of the kind
|
|
141
|
+
the assertion needs. Nothing was compared."""
|
|
142
|
+
NOT_CHECKED = "not-checked"
|
|
143
|
+
"""No comparison occurred, because something it needed was absent or was skipped."""
|
|
144
|
+
ERROR = "error"
|
|
145
|
+
"""The tool failed."""
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
#: The proposition each evidence kind asserts.
|
|
149
|
+
CHECK_TYPES: dict[str, CheckType] = {
|
|
150
|
+
"metric": CheckType.VALUE_COMPARISON,
|
|
151
|
+
"aggregate": CheckType.AGGREGATE_COMPARISON,
|
|
152
|
+
"table": CheckType.VALUE_COMPARISON,
|
|
153
|
+
"value": CheckType.VALUE_COMPARISON,
|
|
154
|
+
"quote": CheckType.PASSAGE_PRESENCE,
|
|
155
|
+
"correspondence": CheckType.CORRESPONDENCE,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
#: The check types whose `match` this version can read as a comparison that held.
|
|
159
|
+
UNDERSTOOD_CHECK_TYPES = frozenset(CHECK_TYPES.values())
|
|
160
|
+
|
|
161
|
+
#: The address kinds this version knows: every locator of SPEC 3.6, and a quotation's.
|
|
162
|
+
KNOWN_ADDRESS_KINDS = frozenset(
|
|
163
|
+
{"tree", "table", "table_position", "array", "sheet", "sqlite", "prose", "passage"}
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
#: The outcome words each checker is known to emit, by checker name. A word from a checker
|
|
167
|
+
#: that is not listed here is not understood, whatever it looks like.
|
|
168
|
+
NATIVE_OUTCOMES: dict[str, frozenset[str]] = {
|
|
169
|
+
CHECKER: frozenset(outcome.value for outcome in Outcome),
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class UnmappedError(ReproError):
|
|
174
|
+
"""An outcome or an evidence kind has no place in the record's vocabulary yet."""
|
|
175
|
+
|
|
176
|
+
def __init__(self, what: str, value: str) -> None:
|
|
177
|
+
self.what = what
|
|
178
|
+
self.value = value
|
|
179
|
+
super().__init__(f"{what} {value!r} has no mapping into a check record")
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def normalized(outcome: Outcome) -> NormalizedOutcome:
|
|
183
|
+
"""The record's outcome for one of this tool's own. The one place the mapping is written.
|
|
184
|
+
|
|
185
|
+
Total by a test over every member and never by a default: an outcome added to the engine
|
|
186
|
+
and not placed here raises, since any fallback would be a guess about what a check found.
|
|
187
|
+
`not_offered` is here for that reason, though no record is written for a claim that offers
|
|
188
|
+
no evidence.
|
|
189
|
+
"""
|
|
190
|
+
mapping = {
|
|
191
|
+
Outcome.VERIFIED: NormalizedOutcome.MATCH,
|
|
192
|
+
Outcome.MISMATCH: NormalizedOutcome.MISMATCH,
|
|
193
|
+
Outcome.NOT_FOUND: NormalizedOutcome.NOT_FOUND,
|
|
194
|
+
Outcome.UNCHECKED: NormalizedOutcome.NOT_CHECKED,
|
|
195
|
+
Outcome.NOT_OFFERED: NormalizedOutcome.NOT_CHECKED,
|
|
196
|
+
Outcome.ERROR: NormalizedOutcome.ERROR,
|
|
197
|
+
}
|
|
198
|
+
if outcome not in mapping:
|
|
199
|
+
raise UnmappedError("outcome", str(outcome))
|
|
200
|
+
return mapping[outcome]
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def check_type(kind: str) -> CheckType:
|
|
204
|
+
"""The proposition an evidence kind asserts, or `UnmappedError` for a kind not placed."""
|
|
205
|
+
if kind not in CHECK_TYPES:
|
|
206
|
+
raise UnmappedError("evidence kind", kind)
|
|
207
|
+
return CHECK_TYPES[kind]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
#: The address kind of a quotation, which has no locator: the passage is the address.
|
|
211
|
+
PASSAGE = "passage"
|
|
212
|
+
|
|
213
|
+
#: Suffixes read as text when a quoted passage is looked for by line. Anything else is left
|
|
214
|
+
#: without a line, since a line in an extraction is not a line in the file.
|
|
215
|
+
TEXT_SUFFIXES = (".txt", ".md", ".markdown", ".tex", ".rst")
|
|
216
|
+
|
|
217
|
+
_SHA256_HEX = r"^[a-f0-9]{64}$"
|
|
218
|
+
|
|
219
|
+
_CONFIG = ConfigDict(frozen=True, extra="allow")
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class Checker(BaseModel):
|
|
223
|
+
model_config = _CONFIG
|
|
224
|
+
|
|
225
|
+
name: str
|
|
226
|
+
version: str
|
|
227
|
+
url: str = ""
|
|
228
|
+
"""Where the checker is described."""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class ManifestRef(BaseModel):
|
|
232
|
+
"""The manifest the assertion was read from."""
|
|
233
|
+
|
|
234
|
+
model_config = _CONFIG
|
|
235
|
+
|
|
236
|
+
path: str = ""
|
|
237
|
+
"""As named from the directory the checker ran in where the manifest lies under it, and
|
|
238
|
+
its file name otherwise. Never absolute: records are committed and published."""
|
|
239
|
+
digest: str | None = Field(default=None, pattern=_SHA256_HEX)
|
|
240
|
+
"""sha256 of the manifest's bytes. None for a manifest that was not read from a file."""
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class Passage(BaseModel):
|
|
244
|
+
"""A passage one of the claim's quote assertions names, and where it sits."""
|
|
245
|
+
|
|
246
|
+
model_config = _CONFIG
|
|
247
|
+
|
|
248
|
+
assertion: int
|
|
249
|
+
"""Index of the quote assertion within the claim."""
|
|
250
|
+
artifact: str
|
|
251
|
+
path: str = ""
|
|
252
|
+
text: str
|
|
253
|
+
line: int | None = Field(default=None, ge=1)
|
|
254
|
+
"""The line the passage starts on, where the artifact is a text file and holds the
|
|
255
|
+
passage exactly once up to whitespace. None otherwise: a line is never guessed."""
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
class ClaimRef(BaseModel):
|
|
259
|
+
model_config = _CONFIG
|
|
260
|
+
|
|
261
|
+
id: str
|
|
262
|
+
text: str = ""
|
|
263
|
+
where: str | None = None
|
|
264
|
+
"""Where the manuscript makes the claim, in the author's words."""
|
|
265
|
+
digest: str
|
|
266
|
+
line: int | None = Field(default=None, ge=1)
|
|
267
|
+
"""The line of the manifest the claim is declared on."""
|
|
268
|
+
passages: tuple[Passage, ...] = ()
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
class AssertionRef(BaseModel):
|
|
272
|
+
model_config = _CONFIG
|
|
273
|
+
|
|
274
|
+
kind: str
|
|
275
|
+
name: str = ""
|
|
276
|
+
position: int = Field(default=0, ge=0)
|
|
277
|
+
"""Which of the claim's assertions of this kind and name it is, from 0. With the claim's
|
|
278
|
+
id, the kind and the name, this is the identity an assertion keeps across edits, and the
|
|
279
|
+
one `.repro/checks.jsonl` records a history under."""
|
|
280
|
+
index: int = Field(ge=0)
|
|
281
|
+
"""Where it is among all of the claim's assertions, from 0, as the manifest orders them."""
|
|
282
|
+
line: int | None = Field(default=None, ge=1)
|
|
283
|
+
"""The line of the manifest the assertion is declared on."""
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class Address(BaseModel):
|
|
287
|
+
"""Where in a file an assertion looks, tagged by `kind`.
|
|
288
|
+
|
|
289
|
+
The fields beside `kind` are those of the locator: `{"kind": "tree", "pointer": ...}`,
|
|
290
|
+
`{"kind": "table", "column": ..., "where": {...}}`, and so on for each kind SPEC 3.6
|
|
291
|
+
defines. A quotation is `{"kind": "passage"}` with the page or the neighbouring text it
|
|
292
|
+
names. A kind this version does not know is kept as it was written.
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
model_config = _CONFIG
|
|
296
|
+
|
|
297
|
+
kind: str
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
class FileState(BaseModel):
|
|
301
|
+
"""One declared file: its pin, and what is on disk."""
|
|
302
|
+
|
|
303
|
+
model_config = _CONFIG
|
|
304
|
+
|
|
305
|
+
artifact: str
|
|
306
|
+
path: str = ""
|
|
307
|
+
media_type: str = ""
|
|
308
|
+
validity: Validity | None = None
|
|
309
|
+
"""None where the manifest declares no such artifact."""
|
|
310
|
+
pinned: str | None = None
|
|
311
|
+
on_disk: str | None = None
|
|
312
|
+
origin: Origin | None = None
|
|
313
|
+
"""The repository, commit and path the manifest says the bytes were copied from. Recorded
|
|
314
|
+
as declared; whether the bytes there are the pinned ones is `claims origins`'s question."""
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
class FileRead(FileState):
|
|
318
|
+
"""One file an assertion reads, the address it reads, and the value there."""
|
|
319
|
+
|
|
320
|
+
side: str = ""
|
|
321
|
+
"""Which side of a `correspondence` this is, in the author's word for it."""
|
|
322
|
+
address: Address | None = None
|
|
323
|
+
"""None where the assertion names no one address."""
|
|
324
|
+
stored: str | None = None
|
|
325
|
+
"""The value at the address, as the file stores it. None where none was read."""
|
|
326
|
+
converted: str | None = None
|
|
327
|
+
"""`stored` under the conversion `comparison` names, which is then the value compared.
|
|
328
|
+
None where no conversion is declared or no number was read."""
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
class Tie(BaseModel):
|
|
332
|
+
"""A compared value exactly halfway between two values at the printed precision, either
|
|
333
|
+
of which a manuscript may print."""
|
|
334
|
+
|
|
335
|
+
model_config = _CONFIG
|
|
336
|
+
|
|
337
|
+
lower: str
|
|
338
|
+
upper: str
|
|
339
|
+
printed: str
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
class Comparison(BaseModel):
|
|
343
|
+
"""The rule a printed value was compared under, and what it was compared with where that
|
|
344
|
+
is not a stored value as it stands."""
|
|
345
|
+
|
|
346
|
+
model_config = _CONFIG
|
|
347
|
+
|
|
348
|
+
mode: ComparisonMode
|
|
349
|
+
tolerance: str = "0"
|
|
350
|
+
places: int | None = None
|
|
351
|
+
"""The decimal places the assertion declares the manuscript reports the value to."""
|
|
352
|
+
conversion: str | None = None
|
|
353
|
+
"""The conversion declared between what a file stores and what is printed: `percent` or
|
|
354
|
+
`fraction`. A side of a `correspondence` that declares one says so in its address."""
|
|
355
|
+
statistic: str | None = None
|
|
356
|
+
"""What an `aggregate` computes over the values in `reads`."""
|
|
357
|
+
count: int | None = None
|
|
358
|
+
"""How many members an `aggregate` declares, which is how many entries `reads` has."""
|
|
359
|
+
computed: str | None = None
|
|
360
|
+
"""That statistic of the values read, which is what was compared with `printed`."""
|
|
361
|
+
tie: Tie | None = None
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
class Stages(BaseModel):
|
|
365
|
+
"""The three stages SPEC 4 reports separately, from which the outcome is derived."""
|
|
366
|
+
|
|
367
|
+
model_config = _CONFIG
|
|
368
|
+
|
|
369
|
+
execution: ExecutionStatus
|
|
370
|
+
extraction: ExtractionStatus
|
|
371
|
+
comparison: ComparisonStatus
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
class Reader(BaseModel):
|
|
375
|
+
"""What read the bytes: the backend, and the program that extracted from the file."""
|
|
376
|
+
|
|
377
|
+
model_config = _CONFIG
|
|
378
|
+
|
|
379
|
+
backend: str = ""
|
|
380
|
+
backend_version: str = ""
|
|
381
|
+
tool: str = ""
|
|
382
|
+
tool_version: str = ""
|
|
383
|
+
extraction_digest: str = ""
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class Rerun(BaseModel):
|
|
387
|
+
"""One witnessed re-run of a command, as `repro reproduce` recorded it."""
|
|
388
|
+
|
|
389
|
+
model_config = _CONFIG
|
|
390
|
+
|
|
391
|
+
started_at: str = ""
|
|
392
|
+
ended_at: str = ""
|
|
393
|
+
"""When the invocation that ran the command started and ended. The command ran between
|
|
394
|
+
the two; `duration_seconds` is its own."""
|
|
395
|
+
state: Regeneration
|
|
396
|
+
reason: str = ""
|
|
397
|
+
checker_version: str = ""
|
|
398
|
+
exit_code: int | None = None
|
|
399
|
+
duration_seconds: float | None = None
|
|
400
|
+
output_digest: str | None = None
|
|
401
|
+
"""The digest of the output the re-run wrote."""
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
class Command(BaseModel):
|
|
405
|
+
"""A declared command upstream of what the assertion reads."""
|
|
406
|
+
|
|
407
|
+
model_config = _CONFIG
|
|
408
|
+
|
|
409
|
+
id: str
|
|
410
|
+
command: tuple[str, ...]
|
|
411
|
+
steps: int = Field(ge=0)
|
|
412
|
+
"""How many commands lie between the assertion and this one. 0 for a command that writes
|
|
413
|
+
a file the assertion reads."""
|
|
414
|
+
output: FileState
|
|
415
|
+
inputs: tuple[FileState, ...] = ()
|
|
416
|
+
state: Freshness
|
|
417
|
+
causes: tuple[str, ...] = ()
|
|
418
|
+
last_rerun: Rerun | None = None
|
|
419
|
+
"""The latest re-run of the command that the record of re-runs holds, whatever it found.
|
|
420
|
+
None where none is recorded."""
|
|
421
|
+
|
|
422
|
+
|
|
423
|
+
class Execution(BaseModel):
|
|
424
|
+
"""What is known of the commands that produce what the assertion reads."""
|
|
425
|
+
|
|
426
|
+
model_config = _CONFIG
|
|
427
|
+
|
|
428
|
+
state: Freshness
|
|
429
|
+
"""The assertion's state against the record of re-runs, as `repro stale` gives it."""
|
|
430
|
+
commands: tuple[Command, ...]
|
|
431
|
+
"""Nearest first."""
|
|
432
|
+
|
|
433
|
+
|
|
434
|
+
class History(BaseModel):
|
|
435
|
+
"""A pointer into the record of checks, for the assertion a record is about."""
|
|
436
|
+
|
|
437
|
+
model_config = _CONFIG
|
|
438
|
+
|
|
439
|
+
states: int = Field(ge=0)
|
|
440
|
+
"""How many states the record of checks holds for the assertion."""
|
|
441
|
+
last_changed_at: str | None = None
|
|
442
|
+
"""When a check last found it different from the state recorded before."""
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
class ExportRecord(BaseModel):
|
|
446
|
+
"""One assertion, as it was checked. Frozen, and carries no verdict."""
|
|
447
|
+
|
|
448
|
+
model_config = ConfigDict(
|
|
449
|
+
frozen=True,
|
|
450
|
+
extra="allow",
|
|
451
|
+
json_schema_extra={
|
|
452
|
+
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
453
|
+
"$id": SCHEMA_URL,
|
|
454
|
+
},
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
record: str = Field(
|
|
458
|
+
default=RECORD_VERSION, json_schema_extra={"enum": list(READ_RECORD_VERSIONS)}
|
|
459
|
+
)
|
|
460
|
+
"""Any string is held as written, and `problems` names one this reader does not know."""
|
|
461
|
+
checker: Checker
|
|
462
|
+
checked_at: AwareDatetime
|
|
463
|
+
manifest: ManifestRef = ManifestRef()
|
|
464
|
+
claim: ClaimRef
|
|
465
|
+
assertion: AssertionRef
|
|
466
|
+
check_type: CheckType
|
|
467
|
+
reads: tuple[FileRead, ...] = ()
|
|
468
|
+
printed: str | None = None
|
|
469
|
+
"""The value as the manuscript prints it, the manifest's `reported`, or the passage a
|
|
470
|
+
quotation asserts. None for a `correspondence`, which compares two files and prints no
|
|
471
|
+
value of its own."""
|
|
472
|
+
comparison: Comparison | None = None
|
|
473
|
+
"""None for a quotation, which is present or absent."""
|
|
474
|
+
outcome: NormalizedOutcome
|
|
475
|
+
native_outcome: str
|
|
476
|
+
"""The checker's own word for what it found, as it prints it."""
|
|
477
|
+
native_reason: str
|
|
478
|
+
"""The checker's own reason for the outcome."""
|
|
479
|
+
detail: str = ""
|
|
480
|
+
stages: Stages | None = None
|
|
481
|
+
"""The three stages this tool derives its outcome from. None in a record written from a
|
|
482
|
+
tool that reports none."""
|
|
483
|
+
reader: Reader = Reader()
|
|
484
|
+
warnings: tuple[str, ...] = ()
|
|
485
|
+
"""The checker's own notes on the evidence, as it words them."""
|
|
486
|
+
execution: Execution | None = None
|
|
487
|
+
"""Present only where a declared command writes a file the assertion reads."""
|
|
488
|
+
history: History | None = None
|
|
489
|
+
"""What the project's record of checks holds for this assertion. Reserved: nothing writes
|
|
490
|
+
it yet, and a record is derived when it is exported, from no stored state."""
|
|
491
|
+
|
|
492
|
+
def to_json(self) -> str:
|
|
493
|
+
"""One line. A field with nothing known is left out."""
|
|
494
|
+
return json.dumps(self.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
# ---------------------------------------------------------------------------------- building
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _state(artifact_id: str, manifest: Manifest, states: dict[str, ArtifactState]) -> dict:
|
|
501
|
+
declared, state = manifest.artifact(artifact_id), states.get(artifact_id)
|
|
502
|
+
return {
|
|
503
|
+
"artifact": artifact_id,
|
|
504
|
+
"path": declared.path.as_posix() if declared else "",
|
|
505
|
+
"media_type": declared.media_type if declared else "",
|
|
506
|
+
"validity": state.validity if state else None,
|
|
507
|
+
"pinned": state.expected if state else None,
|
|
508
|
+
"on_disk": state.actual if state else None,
|
|
509
|
+
"origin": declared.origin if declared else None,
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _address(
|
|
514
|
+
evidence: QuoteEvidence | MetricEvidence | TableCellEvidence | ValueEvidence,
|
|
515
|
+
) -> Address | None:
|
|
516
|
+
if isinstance(evidence, QuoteEvidence):
|
|
517
|
+
named = evidence.model_dump(
|
|
518
|
+
mode="json", exclude={"kind", "artifact", "text"}, exclude_defaults=True
|
|
519
|
+
)
|
|
520
|
+
return Address(kind=PASSAGE, **named)
|
|
521
|
+
if isinstance(evidence, TableCellEvidence) and not evidence.addresses_one_row:
|
|
522
|
+
# Neither `row` nor `where`, or both: there is no one address to state.
|
|
523
|
+
return None
|
|
524
|
+
return Address.model_validate(evidence.locator.model_dump(mode="json"))
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _reads(
|
|
528
|
+
evidence: Evidence,
|
|
529
|
+
decision: Decision,
|
|
530
|
+
manifest: Manifest,
|
|
531
|
+
states: dict[str, ArtifactState],
|
|
532
|
+
paths: dict[str, pathlib.Path],
|
|
533
|
+
) -> tuple[FileRead, ...]:
|
|
534
|
+
# A value is stated only where the check ran. An address may still resolve where a
|
|
535
|
+
# backend failed or a format on the other side has no reader, and a record that held the
|
|
536
|
+
# value would report a reading no check was made with.
|
|
537
|
+
ran = decision.execution is ExecutionStatus.COMPLETED
|
|
538
|
+
if isinstance(evidence, CorrespondenceEvidence):
|
|
539
|
+
read = {side.name: side for side in decision.sides}
|
|
540
|
+
return tuple(
|
|
541
|
+
FileRead(
|
|
542
|
+
**_state(side.artifact, manifest, states),
|
|
543
|
+
side=side.name,
|
|
544
|
+
address=Address.model_validate(
|
|
545
|
+
side.locator.model_dump(mode="json")
|
|
546
|
+
| ({"conversion": side.conversion.value} if side.conversion else {})
|
|
547
|
+
),
|
|
548
|
+
stored=read[side.name].extracted if ran and side.name in read else None,
|
|
549
|
+
converted=read[side.name].converted if ran and side.name in read else None,
|
|
550
|
+
)
|
|
551
|
+
for side in evidence.sides
|
|
552
|
+
)
|
|
553
|
+
if isinstance(evidence, AggregateEvidence):
|
|
554
|
+
# No member was read where a file is absent or undeclared: the engine stops before
|
|
555
|
+
# the backend, and the decision lists none.
|
|
556
|
+
found = [(member.extracted, member.converted) for member in decision.members]
|
|
557
|
+
return tuple(
|
|
558
|
+
FileRead(
|
|
559
|
+
**_state(member.artifact, manifest, states),
|
|
560
|
+
address=Address.model_validate(member.locator.model_dump(mode="json")),
|
|
561
|
+
stored=value if ran else None,
|
|
562
|
+
converted=converted if ran else None,
|
|
563
|
+
)
|
|
564
|
+
for member, (value, converted) in zip(
|
|
565
|
+
evidence.members, found or [(None, None)] * evidence.count, strict=True
|
|
566
|
+
)
|
|
567
|
+
)
|
|
568
|
+
path = paths.get(evidence.artifact)
|
|
569
|
+
return (
|
|
570
|
+
FileRead(
|
|
571
|
+
**_state(evidence.artifact, manifest, states),
|
|
572
|
+
address=_address(evidence),
|
|
573
|
+
stored=stored(evidence, path) if ran and path is not None else None,
|
|
574
|
+
converted=decision.converted if ran else None,
|
|
575
|
+
),
|
|
576
|
+
)
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def _comparison(evidence: Evidence, decision: Decision) -> Comparison | None:
|
|
580
|
+
if isinstance(evidence, QuoteEvidence):
|
|
581
|
+
return None
|
|
582
|
+
conversion = getattr(evidence, "conversion", None)
|
|
583
|
+
return Comparison(
|
|
584
|
+
mode=evidence.mode,
|
|
585
|
+
tolerance=evidence.tolerance,
|
|
586
|
+
places=getattr(evidence, "places", None),
|
|
587
|
+
conversion=conversion.value if conversion else None,
|
|
588
|
+
statistic=decision.statistic.value if decision.statistic else None,
|
|
589
|
+
count=evidence.count if isinstance(evidence, AggregateEvidence) else None,
|
|
590
|
+
computed=decision.computed,
|
|
591
|
+
tie=Tie(**decision.tie.model_dump()) if decision.tie else None,
|
|
592
|
+
)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def _declared_lines(path: pathlib.Path) -> dict[int, tuple[int, tuple[int, ...]]]:
|
|
596
|
+
"""For each claim by position in the manifest: its line, and the line of each assertion.
|
|
597
|
+
|
|
598
|
+
Read off the YAML as written, so a line is where an editor shows the claim. Empty where
|
|
599
|
+
the file does not parse into the expected shape; the manifest loader has then already
|
|
600
|
+
refused it.
|
|
601
|
+
"""
|
|
602
|
+
loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
|
|
603
|
+
try:
|
|
604
|
+
root = yaml.compose(path.read_text(encoding="utf-8"), Loader=loader)
|
|
605
|
+
except (OSError, yaml.YAMLError):
|
|
606
|
+
return {}
|
|
607
|
+
if not isinstance(root, yaml.MappingNode):
|
|
608
|
+
return {}
|
|
609
|
+
|
|
610
|
+
def child(node: yaml.MappingNode, key: str) -> yaml.Node | None:
|
|
611
|
+
return next((value for name, value in node.value if name.value == key), None)
|
|
612
|
+
|
|
613
|
+
claims = child(root, "claims")
|
|
614
|
+
if not isinstance(claims, yaml.SequenceNode):
|
|
615
|
+
return {}
|
|
616
|
+
lines = {}
|
|
617
|
+
for index, claim in enumerate(claims.value):
|
|
618
|
+
evidence = child(claim, "evidence") if isinstance(claim, yaml.MappingNode) else None
|
|
619
|
+
lines[index] = (
|
|
620
|
+
claim.start_mark.line + 1,
|
|
621
|
+
tuple(item.start_mark.line + 1 for item in evidence.value)
|
|
622
|
+
if isinstance(evidence, yaml.SequenceNode)
|
|
623
|
+
else (),
|
|
624
|
+
)
|
|
625
|
+
return lines
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _passage_line(
|
|
629
|
+
path: pathlib.Path, passage: str, texts: dict[pathlib.Path, str | None]
|
|
630
|
+
) -> int | None:
|
|
631
|
+
"""The line a passage starts on in a text file that holds it exactly once, else None.
|
|
632
|
+
|
|
633
|
+
Matched up to whitespace and no further. A passage found only after the normalization a
|
|
634
|
+
quotation is checked under gets no line, and neither does one that occurs twice.
|
|
635
|
+
"""
|
|
636
|
+
if path.suffix.lower() not in TEXT_SUFFIXES or not passage.split():
|
|
637
|
+
return None
|
|
638
|
+
if path not in texts:
|
|
639
|
+
try:
|
|
640
|
+
texts[path] = path.read_text(encoding="utf-8")
|
|
641
|
+
except (OSError, UnicodeDecodeError):
|
|
642
|
+
texts[path] = None
|
|
643
|
+
text = texts[path]
|
|
644
|
+
if text is None:
|
|
645
|
+
return None
|
|
646
|
+
found = list(re.finditer(r"\s+".join(re.escape(word) for word in passage.split()), text))
|
|
647
|
+
return text.count("\n", 0, found[0].start()) + 1 if len(found) == 1 else None
|
|
648
|
+
|
|
649
|
+
|
|
650
|
+
def _rerun(when: tuple[str, str, str], entry: RegenerationState) -> Rerun:
|
|
651
|
+
started_at, ended_at, version = when
|
|
652
|
+
return Rerun(
|
|
653
|
+
started_at=started_at,
|
|
654
|
+
ended_at=ended_at,
|
|
655
|
+
state=entry.state,
|
|
656
|
+
reason=entry.reason.value,
|
|
657
|
+
checker_version=version,
|
|
658
|
+
exit_code=entry.exit_code,
|
|
659
|
+
duration_seconds=entry.duration_seconds,
|
|
660
|
+
output_digest=entry.actual,
|
|
661
|
+
)
|
|
662
|
+
|
|
663
|
+
|
|
664
|
+
def _execution(
|
|
665
|
+
evidence: Evidence,
|
|
666
|
+
state: Freshness,
|
|
667
|
+
manifest: Manifest,
|
|
668
|
+
states: dict[str, ArtifactState],
|
|
669
|
+
report: StalenessReport,
|
|
670
|
+
latest: dict[str, Rerun],
|
|
671
|
+
) -> Execution | None:
|
|
672
|
+
"""The commands upstream of what one assertion reads, nearest first, or None where no
|
|
673
|
+
declared command writes a file it reads."""
|
|
674
|
+
records = {record.id: record for record in manifest.regenerations}
|
|
675
|
+
producers = {record.output.artifact: record for record in manifest.regenerations}
|
|
676
|
+
steps: dict[str, int] = {}
|
|
677
|
+
for name in evidence.artifacts:
|
|
678
|
+
producer = producers.get(name)
|
|
679
|
+
if producer is None:
|
|
680
|
+
continue
|
|
681
|
+
for record_id, distance in {**upstream(producer, producers), producer.id: 0}.items():
|
|
682
|
+
steps[record_id] = min(distance, steps.get(record_id, distance))
|
|
683
|
+
if not steps:
|
|
684
|
+
return None
|
|
685
|
+
nodes = {node.id: node for node in report.regenerations}
|
|
686
|
+
return Execution(
|
|
687
|
+
state=state,
|
|
688
|
+
commands=tuple(
|
|
689
|
+
Command(
|
|
690
|
+
id=record_id,
|
|
691
|
+
command=records[record_id].command,
|
|
692
|
+
steps=steps[record_id],
|
|
693
|
+
output=FileState(**_state(records[record_id].output.artifact, manifest, states)),
|
|
694
|
+
inputs=tuple(
|
|
695
|
+
FileState(**_state(wanted.artifact, manifest, states))
|
|
696
|
+
for wanted in records[record_id].inputs
|
|
697
|
+
),
|
|
698
|
+
state=nodes[record_id].state,
|
|
699
|
+
causes=tuple(cause.value for cause in nodes[record_id].causes),
|
|
700
|
+
last_rerun=latest.get(record_id),
|
|
701
|
+
)
|
|
702
|
+
for record_id in sorted(steps, key=steps.__getitem__)
|
|
703
|
+
),
|
|
704
|
+
)
|
|
705
|
+
|
|
706
|
+
|
|
707
|
+
def _manifest_ref(manifest: Manifest) -> ManifestRef:
|
|
708
|
+
if manifest.path is None:
|
|
709
|
+
return ManifestRef()
|
|
710
|
+
here = pathlib.Path.cwd().resolve()
|
|
711
|
+
resolved = manifest.path.resolve()
|
|
712
|
+
return ManifestRef(
|
|
713
|
+
path=resolved.relative_to(here).as_posix()
|
|
714
|
+
if resolved.is_relative_to(here)
|
|
715
|
+
else resolved.name,
|
|
716
|
+
digest=Digest.of_file(manifest.path).value,
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def check_records(
|
|
721
|
+
manifest: Manifest,
|
|
722
|
+
checked_at: datetime.datetime | None = None,
|
|
723
|
+
backends: tuple[Backend, ...] = DEFAULT_BACKENDS,
|
|
724
|
+
) -> tuple[ExportRecord, ...]:
|
|
725
|
+
"""Check every assertion in a manifest and return one record for each, in the order the
|
|
726
|
+
manifest declares them.
|
|
727
|
+
|
|
728
|
+
`checked_at` is the time every record carries; now, where none is given. Executes nothing
|
|
729
|
+
and writes nothing.
|
|
730
|
+
"""
|
|
731
|
+
when = checked_at or datetime.datetime.now(datetime.UTC).replace(microsecond=0)
|
|
732
|
+
checker = Checker(name=CHECKER, version=adapter_version(), url=CHECKER_URL)
|
|
733
|
+
reference = _manifest_ref(manifest)
|
|
734
|
+
lines = _declared_lines(manifest.path) if manifest.path else {}
|
|
735
|
+
|
|
736
|
+
report = stale(manifest)
|
|
737
|
+
root = manifest.path.parent if manifest.path else pathlib.Path.cwd()
|
|
738
|
+
latest: dict[str, Rerun] = {}
|
|
739
|
+
for invocation in reruns(root):
|
|
740
|
+
stamp = (invocation.started_at, invocation.ended_at, invocation.tool_version)
|
|
741
|
+
for entry in invocation.regenerations:
|
|
742
|
+
if entry.executed:
|
|
743
|
+
latest[entry.regeneration_id] = _rerun(stamp, entry)
|
|
744
|
+
|
|
745
|
+
states, paths = artifact_states(manifest)
|
|
746
|
+
registry: dict[str, Backend] = {b.kind: b for b in backends}
|
|
747
|
+
texts: dict[pathlib.Path, str | None] = {}
|
|
748
|
+
records = []
|
|
749
|
+
with reading_once():
|
|
750
|
+
for position, (claim, node) in enumerate(zip(manifest.claims, report.claims, strict=True)):
|
|
751
|
+
claim_line, evidence_lines = lines.get(position, (None, ()))
|
|
752
|
+
reference_to_claim = ClaimRef(
|
|
753
|
+
id=claim.id,
|
|
754
|
+
text=claim.text,
|
|
755
|
+
where=claim.where,
|
|
756
|
+
digest=claim.digest.value,
|
|
757
|
+
line=claim_line,
|
|
758
|
+
passages=_passages(claim, manifest, paths, texts),
|
|
759
|
+
)
|
|
760
|
+
seen: collections.Counter[tuple[str, str]] = collections.Counter()
|
|
761
|
+
for index, (evidence, freshness) in enumerate(
|
|
762
|
+
zip(claim.evidence, node.assertions, strict=True)
|
|
763
|
+
):
|
|
764
|
+
decision = check_evidence(claim, evidence, manifest, paths, states, registry)
|
|
765
|
+
records.append(
|
|
766
|
+
ExportRecord(
|
|
767
|
+
checker=checker,
|
|
768
|
+
checked_at=when,
|
|
769
|
+
manifest=reference,
|
|
770
|
+
claim=reference_to_claim,
|
|
771
|
+
assertion=AssertionRef(
|
|
772
|
+
kind=evidence.kind,
|
|
773
|
+
name=getattr(evidence, "name", ""),
|
|
774
|
+
position=seen[evidence.kind, getattr(evidence, "name", "")],
|
|
775
|
+
index=index,
|
|
776
|
+
line=evidence_lines[index] if index < len(evidence_lines) else None,
|
|
777
|
+
),
|
|
778
|
+
check_type=check_type(evidence.kind),
|
|
779
|
+
reads=_reads(evidence, decision, manifest, states, paths),
|
|
780
|
+
printed=evidence.text
|
|
781
|
+
if isinstance(evidence, QuoteEvidence)
|
|
782
|
+
else getattr(evidence, "reported", None),
|
|
783
|
+
comparison=_comparison(evidence, decision),
|
|
784
|
+
outcome=normalized(decision.outcome),
|
|
785
|
+
native_outcome=decision.outcome.value,
|
|
786
|
+
native_reason=decision.reason.value,
|
|
787
|
+
detail=decision.detail,
|
|
788
|
+
stages=Stages(
|
|
789
|
+
execution=decision.execution,
|
|
790
|
+
extraction=decision.extraction,
|
|
791
|
+
comparison=decision.comparison,
|
|
792
|
+
),
|
|
793
|
+
reader=Reader(
|
|
794
|
+
backend=decision.backend,
|
|
795
|
+
backend_version=decision.backend_version,
|
|
796
|
+
tool=decision.tool,
|
|
797
|
+
tool_version=decision.tool_version,
|
|
798
|
+
extraction_digest=decision.extraction_digest,
|
|
799
|
+
),
|
|
800
|
+
warnings=tuple(warning.value for warning in decision.warnings),
|
|
801
|
+
execution=_execution(
|
|
802
|
+
evidence, freshness.state, manifest, states, report, latest
|
|
803
|
+
),
|
|
804
|
+
)
|
|
805
|
+
)
|
|
806
|
+
seen[evidence.kind, getattr(evidence, "name", "")] += 1
|
|
807
|
+
return tuple(records)
|
|
808
|
+
|
|
809
|
+
|
|
810
|
+
def _passages(
|
|
811
|
+
claim: Claim,
|
|
812
|
+
manifest: Manifest,
|
|
813
|
+
paths: dict[str, pathlib.Path],
|
|
814
|
+
texts: dict[pathlib.Path, str | None],
|
|
815
|
+
) -> tuple[Passage, ...]:
|
|
816
|
+
found = []
|
|
817
|
+
for index, evidence in enumerate(claim.evidence):
|
|
818
|
+
if not isinstance(evidence, QuoteEvidence):
|
|
819
|
+
continue
|
|
820
|
+
declared, path = manifest.artifact(evidence.artifact), paths.get(evidence.artifact)
|
|
821
|
+
found.append(
|
|
822
|
+
Passage(
|
|
823
|
+
assertion=index,
|
|
824
|
+
artifact=evidence.artifact,
|
|
825
|
+
path=declared.path.as_posix() if declared else "",
|
|
826
|
+
text=evidence.text,
|
|
827
|
+
line=_passage_line(path, evidence.text, texts) if path is not None else None,
|
|
828
|
+
)
|
|
829
|
+
)
|
|
830
|
+
return tuple(found)
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def to_jsonl(records: Iterable[ExportRecord]) -> str:
|
|
834
|
+
"""JSON Lines: one record per line, in the order given."""
|
|
835
|
+
return "".join(f"{record.to_json()}\n" for record in records)
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
# -------------------------------------------------------------------------------- validating
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
class InvalidRecordError(ReproError):
|
|
842
|
+
"""A record does not say something a check could have found."""
|
|
843
|
+
|
|
844
|
+
def __init__(self, problems: tuple[str, ...]) -> None:
|
|
845
|
+
self.problems = problems
|
|
846
|
+
super().__init__("; ".join(problems))
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _flattened(stages: Stages) -> Outcome:
|
|
850
|
+
"""The outcome SPEC 4 derives from three stages, by the rule `Decision` derives it."""
|
|
851
|
+
return Decision(
|
|
852
|
+
claim_id="",
|
|
853
|
+
claim_digest="",
|
|
854
|
+
kind="",
|
|
855
|
+
execution=stages.execution,
|
|
856
|
+
extraction=stages.extraction,
|
|
857
|
+
comparison=stages.comparison,
|
|
858
|
+
reason=Reason.NOT_OFFERED,
|
|
859
|
+
).outcome
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _pin_problems(file: FileState) -> list[str]:
|
|
863
|
+
"""Validity is a fact about a file and its pin, and the two digests have to bear it out."""
|
|
864
|
+
name = f"file {file.artifact!r}"
|
|
865
|
+
if file.validity is Validity.AUTHORITATIVE and (not file.pinned or file.pinned != file.on_disk):
|
|
866
|
+
return [f"{name} is authoritative, so its pinned digest must be the one on disk"]
|
|
867
|
+
if file.validity is Validity.BROKEN_PIN and (
|
|
868
|
+
not file.pinned or not file.on_disk or file.pinned == file.on_disk
|
|
869
|
+
):
|
|
870
|
+
return [f"{name} is a broken pin, so it must carry two digests that differ"]
|
|
871
|
+
if file.validity is Validity.UNPINNED_ARTIFACT and file.pinned:
|
|
872
|
+
return [f"{name} is unpinned, so it must carry no pinned digest"]
|
|
873
|
+
if file.validity is Validity.ARTIFACT_ABSENT and file.on_disk:
|
|
874
|
+
return [f"{name} is absent, so it must carry no digest on disk"]
|
|
875
|
+
return []
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def understood(record: ExportRecord) -> bool:
|
|
879
|
+
"""Whether this version knows what the record's outcome is an outcome of.
|
|
880
|
+
|
|
881
|
+
It does where the checker's own word is one that checker is known to emit, the check type
|
|
882
|
+
is one whose comparison this version can read, and every address is of a kind it knows.
|
|
883
|
+
"""
|
|
884
|
+
return (
|
|
885
|
+
record.native_outcome in NATIVE_OUTCOMES.get(record.checker.name, frozenset())
|
|
886
|
+
and record.check_type in UNDERSTOOD_CHECK_TYPES
|
|
887
|
+
and all(
|
|
888
|
+
read.address is None or read.address.kind in KNOWN_ADDRESS_KINDS
|
|
889
|
+
for read in record.reads
|
|
890
|
+
)
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def read_outcome(record: ExportRecord) -> NormalizedOutcome:
|
|
895
|
+
"""The outcome a consumer may act on: the record's, except that a `match` it does not
|
|
896
|
+
understand is `not-checked`.
|
|
897
|
+
|
|
898
|
+
Unknown stays unknown. A word, a check type or an address kind this version has never
|
|
899
|
+
seen may be a success in the tool that wrote it, and nothing here can tell, so it is never
|
|
900
|
+
passed on as one. Every exporter reads the outcome through this.
|
|
901
|
+
"""
|
|
902
|
+
if record.outcome is NormalizedOutcome.MATCH and not understood(record):
|
|
903
|
+
return NormalizedOutcome.NOT_CHECKED
|
|
904
|
+
return record.outcome
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def _a(word: str) -> str:
|
|
908
|
+
return "an" if word[:1] in "aeiou" else "a"
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
def _value_problems(record: ExportRecord) -> list[str]:
|
|
912
|
+
"""What each check type has to carry for its outcome to be one a check could reach."""
|
|
913
|
+
values = [read.stored for read in record.reads if read.stored is not None]
|
|
914
|
+
converted = [read.converted for read in record.reads if read.converted is not None]
|
|
915
|
+
compared = record.outcome in (NormalizedOutcome.MATCH, NormalizedOutcome.MISMATCH)
|
|
916
|
+
comparison = record.comparison
|
|
917
|
+
word = record.outcome.value
|
|
918
|
+
found = []
|
|
919
|
+
if comparison is not None and comparison.tie is not None:
|
|
920
|
+
tie = comparison.tie
|
|
921
|
+
if record.outcome is NormalizedOutcome.MATCH and tie.printed not in (tie.lower, tie.upper):
|
|
922
|
+
found.append("a match on a tie prints one of the two values the tie lies between")
|
|
923
|
+
if record.check_type is CheckType.VALUE_COMPARISON:
|
|
924
|
+
if compared and (record.printed is None or not values):
|
|
925
|
+
found.append(f"a value-comparison that is a {word} needs a stored and a printed value")
|
|
926
|
+
elif compared and comparison is not None and comparison.conversion and not converted:
|
|
927
|
+
found.append(
|
|
928
|
+
f"a {word} under the conversion {comparison.conversion!r} needs the converted "
|
|
929
|
+
f"value that was compared"
|
|
930
|
+
)
|
|
931
|
+
elif record.check_type is CheckType.AGGREGATE_COMPARISON:
|
|
932
|
+
if comparison is None or not comparison.statistic or comparison.count is None:
|
|
933
|
+
found.append("an aggregate-comparison names its statistic and its count")
|
|
934
|
+
elif comparison.count != len(record.reads):
|
|
935
|
+
found.append(
|
|
936
|
+
f"an aggregate-comparison of {comparison.count} members reads "
|
|
937
|
+
f"{len(record.reads)} files"
|
|
938
|
+
)
|
|
939
|
+
elif compared and (
|
|
940
|
+
record.printed is None or comparison.computed is None or len(values) != comparison.count
|
|
941
|
+
):
|
|
942
|
+
found.append(
|
|
943
|
+
f"an aggregate-comparison that is a {word} needs the printed value, every "
|
|
944
|
+
f"member's stored value and what the statistic computed"
|
|
945
|
+
)
|
|
946
|
+
elif record.check_type is CheckType.PASSAGE_PRESENCE:
|
|
947
|
+
if compared and not record.printed:
|
|
948
|
+
found.append(f"a passage-presence that is a {word} needs the passage looked for")
|
|
949
|
+
if values:
|
|
950
|
+
found.append("a passage-presence compares no value, so none can be stored")
|
|
951
|
+
elif record.check_type is CheckType.CORRESPONDENCE:
|
|
952
|
+
if len(record.reads) != 2 or record.printed is not None:
|
|
953
|
+
found.append("a correspondence reads two files and prints no value of its own")
|
|
954
|
+
elif compared and len(values) != 2:
|
|
955
|
+
found.append(f"a correspondence that is a {word} needs the value stored on each side")
|
|
956
|
+
elif record.check_type is not CheckType.RERUN_COMPARISON and (
|
|
957
|
+
values or record.printed is not None or record.comparison is not None
|
|
958
|
+
):
|
|
959
|
+
# Identity, lineage and coverage are facts about a file or a manuscript as a whole.
|
|
960
|
+
# A pair of values on one of them would say a value was compared, and none was.
|
|
961
|
+
found.append(f"a {record.check_type.value} compares no values, so it carries none")
|
|
962
|
+
return found
|
|
963
|
+
|
|
964
|
+
|
|
965
|
+
def problems(record: ExportRecord) -> tuple[str, ...]:
|
|
966
|
+
"""What a record says that no check could have found. Empty where it is consistent.
|
|
967
|
+
|
|
968
|
+
A schema says which fields a record may hold. This says which records are possible: an
|
|
969
|
+
outcome whose preconditions the rest of the record contradicts, fields its check type
|
|
970
|
+
cannot have, a `match` under a word this version does not know.
|
|
971
|
+
"""
|
|
972
|
+
found = []
|
|
973
|
+
if record.record not in READ_RECORD_VERSIONS:
|
|
974
|
+
versions = " and ".join(repr(version) for version in READ_RECORD_VERSIONS)
|
|
975
|
+
found.append(f"record is {record.record!r}; this reader knows {versions}")
|
|
976
|
+
|
|
977
|
+
known = record.native_outcome in NATIVE_OUTCOMES.get(record.checker.name, frozenset())
|
|
978
|
+
if not known and record.outcome is NormalizedOutcome.MATCH:
|
|
979
|
+
found.append(
|
|
980
|
+
f"{record.checker.name!r} is not known to emit {record.native_outcome!r}, and a "
|
|
981
|
+
f"word that is not understood is not-checked at best, never a match"
|
|
982
|
+
)
|
|
983
|
+
if known and record.checker.name == CHECKER:
|
|
984
|
+
expected = normalized(Outcome(record.native_outcome))
|
|
985
|
+
if expected is not record.outcome:
|
|
986
|
+
found.append(
|
|
987
|
+
f"outcome {record.outcome.value!r} is not what {record.native_outcome!r} "
|
|
988
|
+
f"normalizes to, which is {expected.value!r}"
|
|
989
|
+
)
|
|
990
|
+
if record.stages is not None and _flattened(record.stages).value != record.native_outcome:
|
|
991
|
+
found.append(f"{record.native_outcome!r} does not follow from the stages recorded")
|
|
992
|
+
|
|
993
|
+
placed = CHECK_TYPES.get(record.assertion.kind)
|
|
994
|
+
if record.checker.name == CHECKER and placed is not None and placed is not record.check_type:
|
|
995
|
+
found.append(
|
|
996
|
+
f"{_a(record.assertion.kind)} {record.assertion.kind!r} assertion is {_a(placed.value)} "
|
|
997
|
+
f"{placed.value}, and the record "
|
|
998
|
+
f"says {record.check_type.value}"
|
|
999
|
+
)
|
|
1000
|
+
|
|
1001
|
+
values = [read.stored for read in record.reads if read.stored is not None]
|
|
1002
|
+
stages = record.stages
|
|
1003
|
+
if record.outcome in (NormalizedOutcome.MATCH, NormalizedOutcome.MISMATCH):
|
|
1004
|
+
# The file was read, the address resolved and a comparison ran.
|
|
1005
|
+
if any(read.validity is Validity.ARTIFACT_ABSENT for read in record.reads):
|
|
1006
|
+
found.append(f"a {record.outcome.value} needs every file to have been read")
|
|
1007
|
+
if stages is not None and (
|
|
1008
|
+
stages.execution is not ExecutionStatus.COMPLETED
|
|
1009
|
+
or stages.extraction is not ExtractionStatus.EXTRACTED
|
|
1010
|
+
or stages.comparison.value != record.outcome.value
|
|
1011
|
+
):
|
|
1012
|
+
found.append(f"a {record.outcome.value} needs a comparison that ran and found one")
|
|
1013
|
+
else:
|
|
1014
|
+
if stages is not None and stages.comparison is not ComparisonStatus.NOT_APPLICABLE:
|
|
1015
|
+
found.append(f"{record.outcome.value} means nothing was compared")
|
|
1016
|
+
if record.outcome is NormalizedOutcome.NOT_FOUND:
|
|
1017
|
+
if any(read.validity is Validity.ARTIFACT_ABSENT for read in record.reads):
|
|
1018
|
+
found.append("not-found needs a readable file; an absent one is not-checked")
|
|
1019
|
+
if stages is not None and stages.execution is not ExecutionStatus.COMPLETED:
|
|
1020
|
+
found.append("not-found needs a check that ran")
|
|
1021
|
+
if record.outcome in (NormalizedOutcome.NOT_CHECKED, NormalizedOutcome.ERROR) and values:
|
|
1022
|
+
found.append(f"{record.outcome.value} means no check ran, so no stored value was read")
|
|
1023
|
+
if (
|
|
1024
|
+
record.outcome is NormalizedOutcome.ERROR
|
|
1025
|
+
and stages is not None
|
|
1026
|
+
and stages.execution is not ExecutionStatus.FAILED
|
|
1027
|
+
):
|
|
1028
|
+
found.append("error means the tool failed, and the stages say it did not")
|
|
1029
|
+
|
|
1030
|
+
found += _value_problems(record)
|
|
1031
|
+
for read in record.reads:
|
|
1032
|
+
found += _pin_problems(read)
|
|
1033
|
+
if record.execution is not None:
|
|
1034
|
+
if not record.execution.commands:
|
|
1035
|
+
found.append("an execution block names one or more commands, or is left out")
|
|
1036
|
+
for command in record.execution.commands:
|
|
1037
|
+
for file in (command.output, *command.inputs):
|
|
1038
|
+
found += _pin_problems(file)
|
|
1039
|
+
return tuple(found)
|
|
1040
|
+
|
|
1041
|
+
|
|
1042
|
+
def validate(data: Mapping[str, Any]) -> ExportRecord:
|
|
1043
|
+
"""Read one record, or raise `InvalidRecordError` saying everything wrong with it."""
|
|
1044
|
+
try:
|
|
1045
|
+
record = ExportRecord.model_validate(data)
|
|
1046
|
+
except ValidationError as e:
|
|
1047
|
+
raise InvalidRecordError(
|
|
1048
|
+
tuple(
|
|
1049
|
+
f"{'.'.join(str(x) for x in error['loc']) or '(top level)'}: {error['msg']}"
|
|
1050
|
+
for error in e.errors()
|
|
1051
|
+
)
|
|
1052
|
+
) from e
|
|
1053
|
+
if found := problems(record):
|
|
1054
|
+
raise InvalidRecordError(found)
|
|
1055
|
+
return record
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
__all__ = [
|
|
1059
|
+
"CHECKER",
|
|
1060
|
+
"CHECKER_URL",
|
|
1061
|
+
"CHECK_TYPES",
|
|
1062
|
+
"KNOWN_ADDRESS_KINDS",
|
|
1063
|
+
"LEGACY_RECORD_VERSION",
|
|
1064
|
+
"NATIVE_OUTCOMES",
|
|
1065
|
+
"PASSAGE",
|
|
1066
|
+
"READ_RECORD_VERSIONS",
|
|
1067
|
+
"RECORD_VERSION",
|
|
1068
|
+
"SCHEMA_URL",
|
|
1069
|
+
"UNDERSTOOD_CHECK_TYPES",
|
|
1070
|
+
"Address",
|
|
1071
|
+
"AssertionRef",
|
|
1072
|
+
"CheckType",
|
|
1073
|
+
"Checker",
|
|
1074
|
+
"ClaimRef",
|
|
1075
|
+
"Command",
|
|
1076
|
+
"Comparison",
|
|
1077
|
+
"Execution",
|
|
1078
|
+
"ExportRecord",
|
|
1079
|
+
"FileRead",
|
|
1080
|
+
"FileState",
|
|
1081
|
+
"History",
|
|
1082
|
+
"InvalidRecordError",
|
|
1083
|
+
"ManifestRef",
|
|
1084
|
+
"NormalizedOutcome",
|
|
1085
|
+
"Passage",
|
|
1086
|
+
"Reader",
|
|
1087
|
+
"Rerun",
|
|
1088
|
+
"Stages",
|
|
1089
|
+
"Tie",
|
|
1090
|
+
"UnmappedError",
|
|
1091
|
+
"check_records",
|
|
1092
|
+
"check_type",
|
|
1093
|
+
"normalized",
|
|
1094
|
+
"problems",
|
|
1095
|
+
"read_outcome",
|
|
1096
|
+
"to_jsonl",
|
|
1097
|
+
"understood",
|
|
1098
|
+
"validate",
|
|
1099
|
+
]
|