claims-cli 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claims/__init__.py +132 -0
- claims/__main__.py +5 -0
- claims/adapters/__init__.py +6 -0
- claims/adapters/array.py +212 -0
- claims/adapters/base.py +75 -0
- claims/adapters/cells.py +94 -0
- claims/adapters/columnar.py +188 -0
- claims/adapters/prose.py +261 -0
- claims/adapters/reads.py +47 -0
- claims/adapters/rows.py +116 -0
- claims/adapters/sheet.py +149 -0
- claims/adapters/sqlite.py +73 -0
- claims/adapters/table.py +135 -0
- claims/adapters/tree.py +163 -0
- claims/checks.py +511 -0
- claims/cli.py +1103 -0
- claims/core/__init__.py +9 -0
- claims/core/artifacts.py +121 -0
- claims/core/checks.py +132 -0
- claims/core/claims.py +100 -0
- claims/core/evidence.py +708 -0
- claims/core/manifest.py +349 -0
- claims/core/numbers.py +146 -0
- claims/core/outcomes.py +410 -0
- claims/core/report.py +503 -0
- claims/corpus.py +344 -0
- claims/exceptions.py +106 -0
- claims/export/__init__.py +23 -0
- claims/export/rocrate.py +787 -0
- claims/export/sarif.py +401 -0
- claims/manifest.py +59 -0
- claims/models.py +222 -0
- claims/numbers.py +453 -0
- claims/origins.py +303 -0
- claims/policy.py +369 -0
- claims/provenance.py +42 -0
- claims/record.py +1099 -0
- claims/regression.py +171 -0
- claims/renderers/__init__.py +10 -0
- claims/renderers/sarif.py +294 -0
- claims/reruns.py +111 -0
- claims/resolve.py +116 -0
- claims/stale.py +333 -0
- claims/starter.py +432 -0
- claims/toolchain.py +120 -0
- claims/verify.py +1371 -0
- claims/why.py +398 -0
- claims_cli-0.6.0.dist-info/METADATA +139 -0
- claims_cli-0.6.0.dist-info/RECORD +51 -0
- claims_cli-0.6.0.dist-info/WHEEL +4 -0
- claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/verify.py
ADDED
|
@@ -0,0 +1,1371 @@
|
|
|
1
|
+
"""The verification engine.
|
|
2
|
+
|
|
3
|
+
The engine reads no artifacts itself. It resolves each artifact's pin once, dispatches each
|
|
4
|
+
evidence assertion to the backend registered for its kind, and records what came back. What
|
|
5
|
+
it adds on top of the backends is the part that has to be uniform:
|
|
6
|
+
|
|
7
|
+
* **Pins are checked before evidence.** A source that moved is known before any decision is
|
|
8
|
+
computed against it, and those decisions are marked non-authoritative rather than being
|
|
9
|
+
silently reported as if the declared file had been read.
|
|
10
|
+
* **A backend that cannot run is not a claim that failed.** `BackendUnavailableError`
|
|
11
|
+
becomes `execution=unavailable` with the backend's own reason attached.
|
|
12
|
+
* **A defect is not a scientific outcome.** Any other exception becomes
|
|
13
|
+
`execution=failed`, which flattens to `error` and never to `unchecked`. Letting a
|
|
14
|
+
`TypeError` become an abstention is the same failure this package exists to prevent, one
|
|
15
|
+
level up.
|
|
16
|
+
* **Every decision names the toolchain that read the bytes**, and the digest of what that
|
|
17
|
+
toolchain produced. See `claims.toolchain` for why the backend's protocol version is not
|
|
18
|
+
that.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import decimal
|
|
24
|
+
import fractions
|
|
25
|
+
import math
|
|
26
|
+
import pathlib
|
|
27
|
+
import statistics
|
|
28
|
+
from collections.abc import Mapping
|
|
29
|
+
from typing import Protocol
|
|
30
|
+
|
|
31
|
+
from claims.adapters.reads import reading_once
|
|
32
|
+
from claims.exceptions import (
|
|
33
|
+
ArtifactUnreadableError,
|
|
34
|
+
BackendUnavailableError,
|
|
35
|
+
UnknownEvidenceKindError,
|
|
36
|
+
)
|
|
37
|
+
from claims.models import (
|
|
38
|
+
AggregateEvidence,
|
|
39
|
+
ArtifactRef,
|
|
40
|
+
ArtifactState,
|
|
41
|
+
Claim,
|
|
42
|
+
ClaimAssessment,
|
|
43
|
+
ComparisonMode,
|
|
44
|
+
ComparisonStatus,
|
|
45
|
+
Conversion,
|
|
46
|
+
CorrespondenceEvidence,
|
|
47
|
+
Decision,
|
|
48
|
+
DecisionMember,
|
|
49
|
+
DecisionSide,
|
|
50
|
+
Digest,
|
|
51
|
+
Evidence,
|
|
52
|
+
ExecutionStatus,
|
|
53
|
+
ExtractionStatus,
|
|
54
|
+
Manifest,
|
|
55
|
+
MetricEvidence,
|
|
56
|
+
Ordering,
|
|
57
|
+
OrderingReason,
|
|
58
|
+
QuoteEvidence,
|
|
59
|
+
Reason,
|
|
60
|
+
RegistrationAuthority,
|
|
61
|
+
Statistic,
|
|
62
|
+
TableCellEvidence,
|
|
63
|
+
Tie,
|
|
64
|
+
Validity,
|
|
65
|
+
ValueEvidence,
|
|
66
|
+
VerificationReport,
|
|
67
|
+
Warning_,
|
|
68
|
+
)
|
|
69
|
+
from claims.resolve import Resolution, resolve
|
|
70
|
+
from claims.toolchain import DISTRIBUTION, UNKNOWN, adapter_version, binary_version
|
|
71
|
+
|
|
72
|
+
#: The distribution that ships `claims.adapters`, which is what reads a JSON pointer, a
|
|
73
|
+
#: delimited cell, a SQLite row or an array element. Its name on PyPI, because that is what
|
|
74
|
+
#: `importlib.metadata` answers to.
|
|
75
|
+
ADAPTER_DISTRIBUTION = DISTRIBUTION
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class Backend(Protocol):
|
|
79
|
+
"""Evaluates one kind of evidence assertion against the artifacts it names.
|
|
80
|
+
|
|
81
|
+
`paths` holds one entry per artifact the assertion reads, keyed by id. Every kind but
|
|
82
|
+
`correspondence` reads one, and takes it with `paths[evidence.artifact]`. A mapping rather
|
|
83
|
+
than a path because an assertion whose two sides are both artifacts has no single file to
|
|
84
|
+
be handed, and giving it one would make the engine pick a side.
|
|
85
|
+
"""
|
|
86
|
+
|
|
87
|
+
kind: str
|
|
88
|
+
version: str
|
|
89
|
+
"""The protocol version of this interface, written by hand."""
|
|
90
|
+
tool: str
|
|
91
|
+
"""What performs the extraction: a binary's name, or an installed distribution's."""
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def tool_version(self) -> str:
|
|
95
|
+
"""The version `tool` reports, or `toolchain.UNKNOWN`."""
|
|
96
|
+
...
|
|
97
|
+
|
|
98
|
+
def check(
|
|
99
|
+
self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
|
|
100
|
+
) -> Decision: ...
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
#: Weakest first. A decision over two artifacts is no more authoritative than the worse of
|
|
104
|
+
#: them, and taking the better one would report a comparison against a file that moved as
|
|
105
|
+
#: though the declared bytes had been read.
|
|
106
|
+
_VALIDITY_RANK = {
|
|
107
|
+
Validity.ARTIFACT_ABSENT: 0,
|
|
108
|
+
Validity.BROKEN_PIN: 1,
|
|
109
|
+
Validity.UNPINNED_ARTIFACT: 2,
|
|
110
|
+
Validity.AUTHORITATIVE: 3,
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _base(
|
|
115
|
+
claim: Claim, evidence: Evidence, backend: Backend, extraction_digest: str = UNKNOWN
|
|
116
|
+
) -> dict:
|
|
117
|
+
"""The fields every decision carries, whichever stage it stopped at.
|
|
118
|
+
|
|
119
|
+
`extraction_digest` defaults to `unknown` rather than to an empty string: a decision the
|
|
120
|
+
engine produced was one an extraction was sought for, and the empty value is reserved for
|
|
121
|
+
a decision no backend produced.
|
|
122
|
+
"""
|
|
123
|
+
named = evidence.artifacts
|
|
124
|
+
return {
|
|
125
|
+
"claim_id": claim.id,
|
|
126
|
+
"claim_digest": claim.digest.value,
|
|
127
|
+
"kind": evidence.kind,
|
|
128
|
+
# Left empty where an assertion reads two files or more: `artifact_id` is singular,
|
|
129
|
+
# and filling it with one of them would name a side the engine does not rank. They are
|
|
130
|
+
# recorded in `sides`, or in `members`.
|
|
131
|
+
"artifact_id": named[0] if len(named) == 1 else "",
|
|
132
|
+
"backend": backend.kind,
|
|
133
|
+
"backend_version": backend.version,
|
|
134
|
+
"tool": backend.tool,
|
|
135
|
+
"tool_version": backend.tool_version,
|
|
136
|
+
"extraction_digest": extraction_digest,
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
# -------------------------------------------------------------------------------- quotations
|
|
141
|
+
|
|
142
|
+
#: Every state `citations.verify` can return. It answers with five, and only two were mapped, so
|
|
143
|
+
#: a passage occurring twice in its source raised `KeyError: 'ambiguous'` from inside the backend
|
|
144
|
+
#: and was reported as a defect in the tool rather than as an undecided quotation.
|
|
145
|
+
_QUOTE_STATE = {
|
|
146
|
+
"found": (ExtractionStatus.EXTRACTED, ComparisonStatus.MATCH, Reason.PASSAGE_PRESENT),
|
|
147
|
+
"not found": (ExtractionStatus.EXTRACTED, ComparisonStatus.MISMATCH, Reason.PASSAGE_ABSENT),
|
|
148
|
+
# The address matched more than once, which is where `row_ambiguous` sits for a table: the
|
|
149
|
+
# extraction did not single out one thing to compare, so there is no comparison, and no first
|
|
150
|
+
# match is taken. Not `passage_ambiguous`, which says the document states two different
|
|
151
|
+
# numbers: a passage written twice contradicts nothing, and an author told it does goes
|
|
152
|
+
# looking for a defect in the manuscript when the fix is a `prefix` or `suffix` on the quote
|
|
153
|
+
# assertion. One that singles out no occurrence is this state too, and `citations` says so
|
|
154
|
+
# in the detail: the record names text on either side and still has not said which it means.
|
|
155
|
+
"ambiguous": (
|
|
156
|
+
ExtractionStatus.INVALID,
|
|
157
|
+
ComparisonStatus.NOT_APPLICABLE,
|
|
158
|
+
Reason.QUOTATION_AMBIGUOUS,
|
|
159
|
+
),
|
|
160
|
+
# Not `passage_ambiguous` either: that says the document states two numbers where one was
|
|
161
|
+
# asserted. This says the extractors on this machine do not settle what text the document
|
|
162
|
+
# holds, which accuses nothing and asks for a better reader.
|
|
163
|
+
"indeterminate": (
|
|
164
|
+
ExtractionStatus.INVALID,
|
|
165
|
+
ComparisonStatus.NOT_APPLICABLE,
|
|
166
|
+
Reason.EXTRACTORS_DISAGREE,
|
|
167
|
+
),
|
|
168
|
+
# Not `passage_absent`. The stages were right and the reason was not, so an author whose
|
|
169
|
+
# extractor never ran was told their quotation is missing from the source -- the "nothing ran"
|
|
170
|
+
# against "not there" collapse this package exists to keep apart.
|
|
171
|
+
"unchecked": (
|
|
172
|
+
ExtractionStatus.NOT_ATTEMPTED,
|
|
173
|
+
ComparisonStatus.NOT_APPLICABLE,
|
|
174
|
+
Reason.EXTRACTOR_MISSING,
|
|
175
|
+
),
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
#: Every warning `citations.verify` can put on a result. `page unchecked` was not here, and a
|
|
179
|
+
#: warning with no entry is dropped, so the one warning that says a check was skipped was the
|
|
180
|
+
#: one `repro verify` never showed.
|
|
181
|
+
_QUOTE_WARNINGS = {
|
|
182
|
+
"short": Warning_.SHORT,
|
|
183
|
+
"truncated": Warning_.TRUNCATED,
|
|
184
|
+
"normalized": Warning_.NORMALIZED,
|
|
185
|
+
"page": Warning_.WRONG_PAGE,
|
|
186
|
+
"page unchecked": Warning_.PAGE_UNCHECKED,
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _page_unchecked(page: int | None, state: str, warnings: list[str], paginated: bool) -> str:
|
|
191
|
+
"""Why a declared page was not checked on a passage that was found, or empty where it was.
|
|
192
|
+
|
|
193
|
+
`citations` gives the warning `page unchecked` for each of these. Before 0.5.6 it gave it
|
|
194
|
+
only where a source declares its own extractor, which a manifest cannot do, and the two
|
|
195
|
+
cases a manifest reaches were silent: a source that is not read page by page had its page
|
|
196
|
+
dropped before the passage was looked for, and a passage that matches only with spacing
|
|
197
|
+
ignored was returned before the page was looked at. Both are still worked out here, so
|
|
198
|
+
the answer is the same against an earlier `citations` of the series this package accepts.
|
|
199
|
+
"""
|
|
200
|
+
if page is None or state != "found" or "page" in warnings:
|
|
201
|
+
return ""
|
|
202
|
+
if not paginated:
|
|
203
|
+
return f"page {page} was not checked: the artifact is not read page by page"
|
|
204
|
+
if "normalized" in warnings:
|
|
205
|
+
return f"page {page} was not checked: the passage matched only with spacing ignored"
|
|
206
|
+
if "page unchecked" in warnings:
|
|
207
|
+
return f"page {page} was not checked: the extractor cannot be asked for one page"
|
|
208
|
+
return ""
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
class QuoteBackend:
|
|
212
|
+
"""Assertion: this passage occurs in this artifact.
|
|
213
|
+
|
|
214
|
+
A passage that is absent from a source that was read successfully is a `mismatch`: the
|
|
215
|
+
artifact was extracted and does not contain what the assertion says it contains. That is
|
|
216
|
+
a different fact from a source that could not be read, which is `unavailable`.
|
|
217
|
+
"""
|
|
218
|
+
|
|
219
|
+
kind = "quote"
|
|
220
|
+
version = "1"
|
|
221
|
+
tool = "pdftotext"
|
|
222
|
+
|
|
223
|
+
@property
|
|
224
|
+
def tool_version(self) -> str:
|
|
225
|
+
return binary_version(self.tool)
|
|
226
|
+
|
|
227
|
+
def check(
|
|
228
|
+
self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
|
|
229
|
+
) -> Decision:
|
|
230
|
+
if not isinstance(evidence, QuoteEvidence):
|
|
231
|
+
raise TypeError(f"QuoteBackend received {type(evidence).__name__}")
|
|
232
|
+
try:
|
|
233
|
+
# Imported here rather than at module scope so an absent `citations` reports as a
|
|
234
|
+
# backend that could not run, which is what the caller can act on.
|
|
235
|
+
from citations.exceptions import SourceUnreadableError
|
|
236
|
+
from citations.verify import check_one, extract, is_paginated
|
|
237
|
+
except ImportError as e:
|
|
238
|
+
raise BackendUnavailableError("quote", f"citations is not installed: {e}") from e
|
|
239
|
+
|
|
240
|
+
path = paths[evidence.artifact]
|
|
241
|
+
result = check_one(
|
|
242
|
+
evidence.text, path, evidence.page, prefix=evidence.prefix, suffix=evidence.suffix
|
|
243
|
+
)
|
|
244
|
+
# The page of a passage that was not found is not a second finding beside it.
|
|
245
|
+
named = [w for w in result.warnings if w != "page unchecked" or result.state == "found"]
|
|
246
|
+
# Asked only of a source that was read: `is_paginated` opens the file to tell text from
|
|
247
|
+
# a format with pages.
|
|
248
|
+
paginated = result.state == "found" and is_paginated(path)
|
|
249
|
+
unchecked = _page_unchecked(evidence.page, result.state, named, paginated)
|
|
250
|
+
if unchecked and "page unchecked" not in named:
|
|
251
|
+
named.append("page unchecked")
|
|
252
|
+
warnings = tuple(_QUOTE_WARNINGS[w] for w in named if w in _QUOTE_WARNINGS)
|
|
253
|
+
|
|
254
|
+
if result.state == "unchecked":
|
|
255
|
+
# The extractor could not produce text. Distinguishing this from an absent
|
|
256
|
+
# passage is the reason the three stages are separate.
|
|
257
|
+
raise BackendUnavailableError("quote", result.detail or "no text extracted")
|
|
258
|
+
|
|
259
|
+
try:
|
|
260
|
+
# `extract` memoizes per file, so this is the text `check_one` has just read
|
|
261
|
+
# rather than a second run of `pdftotext`. It is the whole document: a digest over
|
|
262
|
+
# the matched passage alone would be silent on an extractor that changed how it
|
|
263
|
+
# reads every other page.
|
|
264
|
+
extraction_digest = Digest.of_text(extract(path)).value
|
|
265
|
+
except SourceUnreadableError:
|
|
266
|
+
extraction_digest = UNKNOWN
|
|
267
|
+
|
|
268
|
+
base = _base(claim, evidence, self, extraction_digest)
|
|
269
|
+
extraction, comparison, reason = _QUOTE_STATE[result.state]
|
|
270
|
+
if Warning_.WRONG_PAGE in warnings:
|
|
271
|
+
# `page` is documented as verified when present, and the passage is not on the
|
|
272
|
+
# page the manifest asserts. Reporting `match` with a warning made the assertion
|
|
273
|
+
# unenforceable: no policy reads decision warnings, so it graded as verified.
|
|
274
|
+
comparison, reason = ComparisonStatus.MISMATCH, Reason.WRONG_PAGE
|
|
275
|
+
return Decision(
|
|
276
|
+
**base,
|
|
277
|
+
execution=ExecutionStatus.COMPLETED,
|
|
278
|
+
extraction=extraction,
|
|
279
|
+
comparison=comparison,
|
|
280
|
+
reason=reason,
|
|
281
|
+
detail=result.detail or unchecked,
|
|
282
|
+
warnings=warnings,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
# ------------------------------------------------------------------------------- values
|
|
287
|
+
|
|
288
|
+
#: How a resolution maps onto the extraction stage and a reason. `format_unsupported` is not
|
|
289
|
+
#: here: a format with no adapter means the check never ran, which is an execution fact.
|
|
290
|
+
_RESOLUTION_STATE = {
|
|
291
|
+
Resolution.ABSENT: (ExtractionStatus.ABSENT, None),
|
|
292
|
+
Resolution.COLUMN_ABSENT: (ExtractionStatus.ABSENT, Reason.COLUMN_ABSENT),
|
|
293
|
+
Resolution.AMBIGUOUS: (ExtractionStatus.INVALID, Reason.ROW_AMBIGUOUS),
|
|
294
|
+
Resolution.NOT_SCALAR: (ExtractionStatus.INVALID, Reason.SELECTOR_NOT_SCALAR),
|
|
295
|
+
Resolution.SELECTOR_INVALID: (ExtractionStatus.INVALID, Reason.ROW_SELECTOR_INVALID),
|
|
296
|
+
Resolution.NUMBER_AS_WORD: (ExtractionStatus.INVALID, Reason.NUMBER_AS_WORD),
|
|
297
|
+
Resolution.PASSAGE_AMBIGUOUS: (ExtractionStatus.INVALID, Reason.PASSAGE_AMBIGUOUS),
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
#: Which "not there" reason fits each addressing scheme.
|
|
301
|
+
_ABSENT_REASON = {
|
|
302
|
+
"tree": Reason.POINTER_ABSENT,
|
|
303
|
+
"table": Reason.ROW_ABSENT,
|
|
304
|
+
"table_position": Reason.ROW_ABSENT,
|
|
305
|
+
"sqlite": Reason.ROW_ABSENT,
|
|
306
|
+
"array": Reason.POINTER_ABSENT,
|
|
307
|
+
"sheet": Reason.ROW_ABSENT,
|
|
308
|
+
"prose": Reason.PASSAGE_ABSENT,
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def compare_decimal(
|
|
313
|
+
stored: decimal.Decimal, evidence: MetricEvidence | TableCellEvidence | ValueEvidence
|
|
314
|
+
) -> bool:
|
|
315
|
+
"""Does the stored value agree with the reported one, under the declared conversion and
|
|
316
|
+
mode?"""
|
|
317
|
+
return _agree(
|
|
318
|
+
_compared(stored, evidence.conversion),
|
|
319
|
+
evidence.value,
|
|
320
|
+
evidence.mode,
|
|
321
|
+
evidence.tolerance_value,
|
|
322
|
+
)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def _compared(stored: decimal.Decimal, conversion: Conversion | None) -> decimal.Decimal:
|
|
326
|
+
"""The stored value in the unit it is compared in.
|
|
327
|
+
|
|
328
|
+
Converted first and rounded after: a file holding 0.0426 printed as a percentage is 4.26,
|
|
329
|
+
which is 4.3 at one decimal, where rounding the fraction to 0.04 first would print 4. A
|
|
330
|
+
non-finite value has no unit and is returned as it is, for `_agree` to refuse.
|
|
331
|
+
"""
|
|
332
|
+
return conversion.apply(stored) if conversion is not None and stored.is_finite() else stored
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _held(stored: decimal.Decimal, compared: decimal.Decimal, conversion: Conversion | None) -> str:
|
|
336
|
+
"""A stored value as a detail shows it: with what it is under a declared conversion."""
|
|
337
|
+
if conversion is None:
|
|
338
|
+
return f"{stored}"
|
|
339
|
+
return f"{stored} ({_plain(compared)} {conversion.said})"
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _places(count: int) -> str:
|
|
343
|
+
"""A number of decimal places, in words a detail can use. A value printed to the tens
|
|
344
|
+
has fewer than none, which is said as it is."""
|
|
345
|
+
return f"{count} decimal place{'' if count == 1 else 's'}"
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _misplaced(
|
|
349
|
+
evidence: MetricEvidence | TableCellEvidence | ValueEvidence | AggregateEvidence,
|
|
350
|
+
) -> str | None:
|
|
351
|
+
"""What to say where `places` is declared and `reported` is written to another number of
|
|
352
|
+
them, or None. Two statements about the manuscript disagree, and no rounding of the
|
|
353
|
+
file's value settles which is meant: `4.20` is not a number reported to one place."""
|
|
354
|
+
printed_places = -_exponent(evidence.value)
|
|
355
|
+
if evidence.places is None or printed_places == evidence.places:
|
|
356
|
+
return None
|
|
357
|
+
return (
|
|
358
|
+
f"{evidence.name}: manuscript prints {evidence.reported}, which is "
|
|
359
|
+
f"{_places(printed_places)}, and {_places(evidence.places)} is declared"
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _plain(converted: decimal.Decimal) -> str:
|
|
364
|
+
"""A converted value with no exponent: 0.5 as a percentage is written 50, where `str`
|
|
365
|
+
writes the same decimal as 5E+1."""
|
|
366
|
+
return format(converted, "f")
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _exponent(value: decimal.Decimal) -> int:
|
|
370
|
+
"""The power of ten a finite decimal's last digit sits at.
|
|
371
|
+
|
|
372
|
+
`Decimal.as_tuple().exponent` is `int | Literal["n", "N", "F"]`: the strings tag NaN and
|
|
373
|
+
the infinities, which have no precision for a comparison to be coarser or finer than.
|
|
374
|
+
Every backend refuses a non-finite value with `value_not_numeric` before comparing, so
|
|
375
|
+
reaching this with one is a defect in a backend rather than a fact about an artifact, and
|
|
376
|
+
§5 says a defect is an `error` and never a scientific outcome. Raising is how it becomes
|
|
377
|
+
one. Returning a number instead would compare two values at a precision nothing chose.
|
|
378
|
+
"""
|
|
379
|
+
exponent = value.as_tuple().exponent
|
|
380
|
+
if not isinstance(exponent, int):
|
|
381
|
+
raise ValueError(
|
|
382
|
+
f"{value} has no exponent: a non-finite value is refused before any comparison, "
|
|
383
|
+
f"so reaching the comparison with one is a defect"
|
|
384
|
+
)
|
|
385
|
+
return exponent
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _tie(
|
|
389
|
+
compared: decimal.Decimal | fractions.Fraction, reported: decimal.Decimal
|
|
390
|
+
) -> tuple[decimal.Decimal, decimal.Decimal] | None:
|
|
391
|
+
"""The two values at the precision `reported` prints that `compared` lies exactly halfway
|
|
392
|
+
between, lower first, or None where it is nearer one of them.
|
|
393
|
+
|
|
394
|
+
Decided in exact rationals, for a stored decimal and for a statistic that may be no
|
|
395
|
+
decimal at all: the value in units of the printed precision is a whole number and a half,
|
|
396
|
+
or it is not. No float is formed and nothing is within a tolerance of a tie, so 4.25 is
|
|
397
|
+
one at one place and 4.2500001 is not.
|
|
398
|
+
"""
|
|
399
|
+
if isinstance(compared, decimal.Decimal) and not compared.is_finite():
|
|
400
|
+
return None
|
|
401
|
+
quantum = _exponent(reported)
|
|
402
|
+
units = fractions.Fraction(compared) / fractions.Fraction(10) ** quantum
|
|
403
|
+
if units.denominator != 2:
|
|
404
|
+
return None
|
|
405
|
+
below = math.floor(units)
|
|
406
|
+
|
|
407
|
+
def at_precision(count: int) -> decimal.Decimal:
|
|
408
|
+
# Zero carries no sign: the neighbor above -0.5 is 0.
|
|
409
|
+
return decimal.Decimal((int(count < 0), tuple(map(int, str(abs(count)))), quantum))
|
|
410
|
+
|
|
411
|
+
return at_precision(below), at_precision(below + 1)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def _tied(
|
|
415
|
+
compared: decimal.Decimal | fractions.Fraction,
|
|
416
|
+
printed: decimal.Decimal,
|
|
417
|
+
mode: ComparisonMode,
|
|
418
|
+
) -> Tie | None:
|
|
419
|
+
"""The record of a tie, where the comparison is at a printed precision and met one."""
|
|
420
|
+
if mode is not ComparisonMode.PRINTED_PRECISION:
|
|
421
|
+
return None
|
|
422
|
+
neighbors = _tie(compared, printed)
|
|
423
|
+
if neighbors is None:
|
|
424
|
+
return None
|
|
425
|
+
lower, upper = neighbors
|
|
426
|
+
return Tie(lower=_plain(lower), upper=_plain(upper), printed=_plain(printed))
|
|
427
|
+
|
|
428
|
+
|
|
429
|
+
def _agree(
|
|
430
|
+
stored: decimal.Decimal,
|
|
431
|
+
reported: decimal.Decimal,
|
|
432
|
+
mode: ComparisonMode,
|
|
433
|
+
tolerance: decimal.Decimal,
|
|
434
|
+
) -> bool:
|
|
435
|
+
"""Do two decimals agree under one mode? The arithmetic, with no evidence attached.
|
|
436
|
+
|
|
437
|
+
Split out because a correspondence has two extracted values and no `reported` field to
|
|
438
|
+
read a mode off, and duplicating the `printed_precision` rounding rule is how two
|
|
439
|
+
spellings of the same comparison come to disagree.
|
|
440
|
+
"""
|
|
441
|
+
if not stored.is_finite():
|
|
442
|
+
# NaN and the infinities parse out of JSON but cannot equal a printed decimal. The
|
|
443
|
+
# backend flags them before reaching here; this keeps the function total.
|
|
444
|
+
return False
|
|
445
|
+
if mode is ComparisonMode.PRINTED_PRECISION:
|
|
446
|
+
# Round the stored value to the precision the manuscript printed. A paper reporting
|
|
447
|
+
# 3.2 is not contradicted by a file holding 3.20001; a paper reporting 3.20000 is.
|
|
448
|
+
# `quantize` raises whenever the *result* needs more digits than the context carries,
|
|
449
|
+
# which is not only the case where the two values differ wildly: two identical
|
|
450
|
+
# thirty-digit integers also exceed the default precision of 28, and were reported as
|
|
451
|
+
# a mismatch. That is a defect in the arithmetic emitted as a contradicted manuscript,
|
|
452
|
+
# which is the one thing this package must never do. Size the context to the operands.
|
|
453
|
+
#
|
|
454
|
+
# A value exactly halfway between two printed values is rounded up by half the
|
|
455
|
+
# software a manuscript's numbers come out of and to the even neighbor by the other
|
|
456
|
+
# half, so either neighbor agrees. Everywhere else the rounding is half-even, and
|
|
457
|
+
# which rule is used makes no difference away from a tie.
|
|
458
|
+
if (neighbors := _tie(stored, reported)) is not None:
|
|
459
|
+
return reported in neighbors
|
|
460
|
+
with decimal.localcontext() as context:
|
|
461
|
+
context.prec = max(
|
|
462
|
+
context.prec,
|
|
463
|
+
len(stored.as_tuple().digits) + abs(int(reported.as_tuple().exponent)) + 2,
|
|
464
|
+
)
|
|
465
|
+
try:
|
|
466
|
+
return stored.quantize(reported, rounding=decimal.ROUND_HALF_EVEN) == reported
|
|
467
|
+
except decimal.InvalidOperation:
|
|
468
|
+
# Genuinely beyond reach: the two differ by more orders of magnitude than any
|
|
469
|
+
# precision reconciles. They disagree, and saying so is the answer.
|
|
470
|
+
return False
|
|
471
|
+
delta = abs(stored - reported)
|
|
472
|
+
if mode is ComparisonMode.ABSOLUTE:
|
|
473
|
+
return delta <= tolerance
|
|
474
|
+
return delta <= tolerance * abs(reported)
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def _agree_exact(
|
|
478
|
+
computed: fractions.Fraction,
|
|
479
|
+
reported: decimal.Decimal,
|
|
480
|
+
mode: ComparisonMode,
|
|
481
|
+
tolerance: decimal.Decimal,
|
|
482
|
+
) -> bool:
|
|
483
|
+
"""`_agree` for a value that may have no finite decimal expansion.
|
|
484
|
+
|
|
485
|
+
A mean of three values is a third of their sum, and dividing in decimals rounds it once
|
|
486
|
+
at whatever precision the context carries before `printed_precision` rounds it again. A
|
|
487
|
+
rational is exact, so the one rounding is the rule's own: half-even to the precision
|
|
488
|
+
`reported` prints, with either neighbor agreeing at an exact tie. On a value that is a
|
|
489
|
+
decimal the two functions agree.
|
|
490
|
+
"""
|
|
491
|
+
printed = fractions.Fraction(reported)
|
|
492
|
+
if mode is ComparisonMode.PRINTED_PRECISION:
|
|
493
|
+
# The tie rule a single value is held to, by the function that holds it.
|
|
494
|
+
if (neighbors := _tie(computed, reported)) is not None:
|
|
495
|
+
return reported in neighbors
|
|
496
|
+
quantum = fractions.Fraction(10) ** _exponent(reported)
|
|
497
|
+
# `round` of a rational is exact and takes a tie to the even neighbor.
|
|
498
|
+
return round(computed / quantum) * quantum == printed
|
|
499
|
+
delta = abs(computed - printed)
|
|
500
|
+
if mode is ComparisonMode.ABSOLUTE:
|
|
501
|
+
return delta <= fractions.Fraction(tolerance)
|
|
502
|
+
return delta <= fractions.Fraction(tolerance) * abs(printed)
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _aggregate(statistic: Statistic, values: list[fractions.Fraction]) -> fractions.Fraction:
|
|
506
|
+
"""One statistic of at least one value, exactly."""
|
|
507
|
+
if statistic is Statistic.MEAN:
|
|
508
|
+
return sum(values, fractions.Fraction(0)) / len(values)
|
|
509
|
+
if statistic is Statistic.MEDIAN:
|
|
510
|
+
# The middle value, or half the sum of the two middle ones, which a rational holds.
|
|
511
|
+
return statistics.median(values)
|
|
512
|
+
if statistic is Statistic.MIN:
|
|
513
|
+
return min(values)
|
|
514
|
+
if statistic is Statistic.MAX:
|
|
515
|
+
return max(values)
|
|
516
|
+
if statistic is Statistic.SUM:
|
|
517
|
+
return sum(values, fractions.Fraction(0))
|
|
518
|
+
return fractions.Fraction(len(values))
|
|
519
|
+
|
|
520
|
+
|
|
521
|
+
#: How many places past the printed precision a statistic with no finite expansion is shown to.
|
|
522
|
+
SHOWN_PLACES = 3
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def _shown(computed: fractions.Fraction, reported: decimal.Decimal) -> str:
|
|
526
|
+
"""A statistic as a report shows it: exactly where it is a terminating decimal, and
|
|
527
|
+
otherwise rounded half-even to `SHOWN_PLACES` past the precision the manuscript prints."""
|
|
528
|
+
places = 0
|
|
529
|
+
scaled = computed
|
|
530
|
+
while scaled.denominator % 2 == 0 or scaled.denominator % 5 == 0:
|
|
531
|
+
scaled *= 10
|
|
532
|
+
places += 1
|
|
533
|
+
if scaled.denominator != 1:
|
|
534
|
+
places = max(0, -_exponent(reported)) + SHOWN_PLACES
|
|
535
|
+
scaled = fractions.Fraction(round(computed * 10**places))
|
|
536
|
+
sign, digits, _ = decimal.Decimal(scaled.numerator).as_tuple()
|
|
537
|
+
return format(decimal.Decimal((sign, digits, -places)), "f")
|
|
538
|
+
|
|
539
|
+
|
|
540
|
+
class ValueBackend:
|
|
541
|
+
"""Assertion: this artifact holds this value at this locator.
|
|
542
|
+
|
|
543
|
+
One implementation for every numeric kind. `metric` and `table` are shorthands that
|
|
544
|
+
expose a locator, so the addressing rules and the invariant -- exactly one scalar -- do
|
|
545
|
+
not depend on which spelling a manifest used.
|
|
546
|
+
"""
|
|
547
|
+
|
|
548
|
+
kind = "value"
|
|
549
|
+
version = "2"
|
|
550
|
+
tool = ADAPTER_DISTRIBUTION
|
|
551
|
+
|
|
552
|
+
@property
|
|
553
|
+
def tool_version(self) -> str:
|
|
554
|
+
return adapter_version()
|
|
555
|
+
|
|
556
|
+
def check(
|
|
557
|
+
self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
|
|
558
|
+
) -> Decision:
|
|
559
|
+
if not isinstance(evidence, (MetricEvidence, TableCellEvidence, ValueEvidence)):
|
|
560
|
+
raise TypeError(f"{type(self).__name__} received {type(evidence).__name__}")
|
|
561
|
+
path = paths[evidence.artifact]
|
|
562
|
+
base = _base(claim, evidence, self)
|
|
563
|
+
base["conversion"] = evidence.conversion
|
|
564
|
+
completed = {"execution": ExecutionStatus.COMPLETED}
|
|
565
|
+
no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
|
|
566
|
+
|
|
567
|
+
if isinstance(evidence, TableCellEvidence) and not evidence.addresses_one_row:
|
|
568
|
+
# Both or neither given. A manifest that does not say which row it means is
|
|
569
|
+
# malformed rather than a table that disagrees.
|
|
570
|
+
return Decision(
|
|
571
|
+
**base,
|
|
572
|
+
**completed,
|
|
573
|
+
**no_compare,
|
|
574
|
+
extraction=ExtractionStatus.INVALID,
|
|
575
|
+
reason=Reason.ROW_SELECTOR_INVALID,
|
|
576
|
+
detail="give exactly one of `row` or `where`",
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
locator = evidence.locator
|
|
580
|
+
base["locator_digest"] = locator.digest.value
|
|
581
|
+
warnings = (Warning_.POSITIONAL_ADDRESS,) if locator.kind == "table_position" else ()
|
|
582
|
+
|
|
583
|
+
resolution, extracted, detail = resolve(locator, path)
|
|
584
|
+
|
|
585
|
+
if resolution is Resolution.FORMAT_UNSUPPORTED:
|
|
586
|
+
# The check did not run. Reporting it as a missing value would say something
|
|
587
|
+
# about the artifact that was never looked into.
|
|
588
|
+
return Decision(
|
|
589
|
+
**base,
|
|
590
|
+
execution=ExecutionStatus.UNAVAILABLE,
|
|
591
|
+
extraction=ExtractionStatus.NOT_ATTEMPTED,
|
|
592
|
+
**no_compare,
|
|
593
|
+
reason=Reason.FORMAT_UNSUPPORTED,
|
|
594
|
+
detail=detail,
|
|
595
|
+
warnings=warnings,
|
|
596
|
+
)
|
|
597
|
+
|
|
598
|
+
if resolution is not Resolution.RESOLVED or extracted is None:
|
|
599
|
+
extraction, reason = _RESOLUTION_STATE[resolution]
|
|
600
|
+
return Decision(
|
|
601
|
+
**base,
|
|
602
|
+
**completed,
|
|
603
|
+
**no_compare,
|
|
604
|
+
extraction=extraction,
|
|
605
|
+
reason=reason or _ABSENT_REASON[locator.kind],
|
|
606
|
+
detail=detail,
|
|
607
|
+
warnings=warnings,
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
# The adapter produced a value, so there is an extraction to hash. Recorded before the
|
|
611
|
+
# comparison, and for a value that turns out not to be numeric: what the extractor
|
|
612
|
+
# read is a fact about the extractor whatever the comparison then makes of it.
|
|
613
|
+
base["extraction_digest"] = Digest.of_text(extracted.raw).value
|
|
614
|
+
base["extracted"] = extracted.raw
|
|
615
|
+
|
|
616
|
+
try:
|
|
617
|
+
stored = decimal.Decimal(extracted.raw)
|
|
618
|
+
except (decimal.InvalidOperation, ValueError):
|
|
619
|
+
return Decision(
|
|
620
|
+
**base,
|
|
621
|
+
**completed,
|
|
622
|
+
**no_compare,
|
|
623
|
+
extraction=ExtractionStatus.INVALID,
|
|
624
|
+
reason=Reason.VALUE_NOT_NUMERIC,
|
|
625
|
+
detail=f"{' '.join(extracted.trace)} holds {extracted.raw!r}",
|
|
626
|
+
warnings=warnings,
|
|
627
|
+
)
|
|
628
|
+
if not stored.is_finite():
|
|
629
|
+
# `json.loads` accepts NaN, Infinity and -Infinity by default, so a result file
|
|
630
|
+
# can carry one. It is not a quantity a printed value can agree with.
|
|
631
|
+
return Decision(
|
|
632
|
+
**base,
|
|
633
|
+
**completed,
|
|
634
|
+
**no_compare,
|
|
635
|
+
extraction=ExtractionStatus.INVALID,
|
|
636
|
+
reason=Reason.VALUE_NOT_NUMERIC,
|
|
637
|
+
detail=f"{' '.join(extracted.trace)} holds {stored}",
|
|
638
|
+
warnings=warnings,
|
|
639
|
+
)
|
|
640
|
+
|
|
641
|
+
compared = _compared(stored, evidence.conversion)
|
|
642
|
+
if evidence.conversion is not None:
|
|
643
|
+
base["converted"] = _plain(compared)
|
|
644
|
+
held = _held(stored, compared, evidence.conversion)
|
|
645
|
+
where = " ".join(extracted.trace) or locator.kind
|
|
646
|
+
|
|
647
|
+
if (misplaced := _misplaced(evidence)) is not None:
|
|
648
|
+
return Decision(
|
|
649
|
+
**base,
|
|
650
|
+
**completed,
|
|
651
|
+
extraction=ExtractionStatus.EXTRACTED,
|
|
652
|
+
comparison=ComparisonStatus.MISMATCH,
|
|
653
|
+
reason=Reason.PLACES_MISMATCH,
|
|
654
|
+
detail=misplaced,
|
|
655
|
+
warnings=warnings,
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
agrees = compare_decimal(stored, evidence)
|
|
659
|
+
tie = _tied(compared, evidence.value, evidence.mode)
|
|
660
|
+
base["tie"] = tie
|
|
661
|
+
between = f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""
|
|
662
|
+
return Decision(
|
|
663
|
+
**base,
|
|
664
|
+
**completed,
|
|
665
|
+
extraction=ExtractionStatus.EXTRACTED,
|
|
666
|
+
comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
|
|
667
|
+
reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
|
|
668
|
+
detail=(
|
|
669
|
+
f"{where} = {held}{between}"
|
|
670
|
+
+ (f"; the manuscript prints {evidence.reported}" if tie else "")
|
|
671
|
+
if agrees
|
|
672
|
+
else f"{evidence.name}: manuscript prints {evidence.reported}, "
|
|
673
|
+
f"{path.name} holds {held} at {where}{between}"
|
|
674
|
+
),
|
|
675
|
+
warnings=warnings,
|
|
676
|
+
)
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
class MetricBackend(ValueBackend):
|
|
680
|
+
"""`kind: metric` -- a JSON Pointer into a structured file."""
|
|
681
|
+
|
|
682
|
+
kind = "metric"
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
class TableBackend(ValueBackend):
|
|
686
|
+
"""`kind: table` -- a column plus a row selector in a delimited file."""
|
|
687
|
+
|
|
688
|
+
kind = "table"
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
# --------------------------------------------------------------------------- correspondences
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
class CorrespondenceBackend:
|
|
695
|
+
"""Assertion: two artifacts hold the same value.
|
|
696
|
+
|
|
697
|
+
Two extractions precede one comparison, and the stages report that faithfully. A side that
|
|
698
|
+
does not extract makes the comparison impossible rather than false: a document that never
|
|
699
|
+
states the number does not contradict the file holding it, and reporting that as a
|
|
700
|
+
disagreement would manufacture a finding out of a gap. The decision carries the failing
|
|
701
|
+
side's own reason, so `pointer_absent` and `number_as_word` stay distinguishable.
|
|
702
|
+
|
|
703
|
+
When both sides extract and disagree, the decision reports both values and names neither
|
|
704
|
+
as wrong. Which of a specification and a test suite is in error is not something a byte
|
|
705
|
+
comparison establishes.
|
|
706
|
+
"""
|
|
707
|
+
|
|
708
|
+
kind = "correspondence"
|
|
709
|
+
version = "1"
|
|
710
|
+
#: The format adapters are this distribution, as they are for `ValueBackend`. The
|
|
711
|
+
#: declaration is an approximation for one case: a `prose` side over a paginated source
|
|
712
|
+
#: reaches `citations.verify.extract`, which runs the same `pdftotext` that `QuoteBackend`
|
|
713
|
+
#: declares, and `Decision.tool` is one field where such an assertion has two extractors.
|
|
714
|
+
#: `DecisionSide.extraction_digest` is what catches a change in either of them, since it
|
|
715
|
+
#: moves whenever what a side read moves, whatever did the reading. Naming the tool per
|
|
716
|
+
#: side needs the adapters to report which one they used, which this revision does not
|
|
717
|
+
#: define.
|
|
718
|
+
tool = ADAPTER_DISTRIBUTION
|
|
719
|
+
|
|
720
|
+
@property
|
|
721
|
+
def tool_version(self) -> str:
|
|
722
|
+
return adapter_version()
|
|
723
|
+
|
|
724
|
+
def check(
|
|
725
|
+
self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
|
|
726
|
+
) -> Decision:
|
|
727
|
+
if not isinstance(evidence, CorrespondenceEvidence):
|
|
728
|
+
raise TypeError(f"CorrespondenceBackend received {type(evidence).__name__}")
|
|
729
|
+
base = _base(claim, evidence, self)
|
|
730
|
+
no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
|
|
731
|
+
|
|
732
|
+
readings = [(side, *resolve(side.locator, paths[side.artifact])) for side in evidence.sides]
|
|
733
|
+
base["sides"] = tuple(
|
|
734
|
+
DecisionSide(
|
|
735
|
+
name=side.name,
|
|
736
|
+
artifact_id=side.artifact,
|
|
737
|
+
locator_digest=side.locator.digest.value,
|
|
738
|
+
extracted=extracted.raw if extracted is not None else None,
|
|
739
|
+
conversion=side.conversion,
|
|
740
|
+
# Hashed per side before any comparison, and for a value that turns out not
|
|
741
|
+
# to be numeric: what an extractor read is a fact about the extractor whatever
|
|
742
|
+
# the comparison then makes of it.
|
|
743
|
+
extraction_digest=(
|
|
744
|
+
Digest.of_text(extracted.raw).value if extracted is not None else UNKNOWN
|
|
745
|
+
),
|
|
746
|
+
)
|
|
747
|
+
for side, _, extracted, _ in readings
|
|
748
|
+
)
|
|
749
|
+
# The decision-level field covers both extractions, in the order the manifest declares
|
|
750
|
+
# the sides, so it moves whenever either side's extraction moves. Which of the two
|
|
751
|
+
# moved is on the sides; leaving this `unknown` would say the extraction was sought
|
|
752
|
+
# and not obtained, and it was sought twice.
|
|
753
|
+
base["extraction_digest"] = Digest.of_text(
|
|
754
|
+
"\x00".join(s.extraction_digest for s in base["sides"])
|
|
755
|
+
).value
|
|
756
|
+
|
|
757
|
+
unsupported = [(s, d) for s, r, _, d in readings if r is Resolution.FORMAT_UNSUPPORTED]
|
|
758
|
+
if unsupported:
|
|
759
|
+
side, detail = unsupported[0]
|
|
760
|
+
return Decision(
|
|
761
|
+
**base,
|
|
762
|
+
execution=ExecutionStatus.UNAVAILABLE,
|
|
763
|
+
extraction=ExtractionStatus.NOT_ATTEMPTED,
|
|
764
|
+
**no_compare,
|
|
765
|
+
reason=Reason.FORMAT_UNSUPPORTED,
|
|
766
|
+
detail=f"{side.name}: {detail}",
|
|
767
|
+
)
|
|
768
|
+
|
|
769
|
+
completed = {"execution": ExecutionStatus.COMPLETED}
|
|
770
|
+
unresolved = [
|
|
771
|
+
(s, r, d) for s, r, e, d in readings if r is not Resolution.RESOLVED or e is None
|
|
772
|
+
]
|
|
773
|
+
if unresolved:
|
|
774
|
+
# Reported in the order the manifest declares the sides, which is a fact about the
|
|
775
|
+
# manifest and not a ranking. Every failing side appears in the detail.
|
|
776
|
+
side, resolution, _ = unresolved[0]
|
|
777
|
+
extraction, reason = _RESOLUTION_STATE[resolution]
|
|
778
|
+
return Decision(
|
|
779
|
+
**base,
|
|
780
|
+
**completed,
|
|
781
|
+
**no_compare,
|
|
782
|
+
extraction=extraction,
|
|
783
|
+
reason=reason or _ABSENT_REASON[side.locator.kind],
|
|
784
|
+
detail="; ".join(f"{s.name} ({s.artifact}): {d}" for s, _, d in unresolved),
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
numbers: list[decimal.Decimal] = []
|
|
788
|
+
for side, _, extracted, _ in readings:
|
|
789
|
+
raw = extracted.raw if extracted is not None else ""
|
|
790
|
+
try:
|
|
791
|
+
number = decimal.Decimal(raw)
|
|
792
|
+
except (decimal.InvalidOperation, ValueError):
|
|
793
|
+
number = decimal.Decimal("NaN")
|
|
794
|
+
if not number.is_finite():
|
|
795
|
+
return Decision(
|
|
796
|
+
**base,
|
|
797
|
+
**completed,
|
|
798
|
+
**no_compare,
|
|
799
|
+
extraction=ExtractionStatus.INVALID,
|
|
800
|
+
reason=Reason.VALUE_NOT_NUMERIC,
|
|
801
|
+
detail=f"{side.name} ({side.artifact}) holds {raw!r}",
|
|
802
|
+
)
|
|
803
|
+
numbers.append(number)
|
|
804
|
+
|
|
805
|
+
# Each side in the unit it declares, and the comparison is over those: a document
|
|
806
|
+
# printing 4.2 beside a file holding 0.0424 as a percentage compares 4.2 with 4.24.
|
|
807
|
+
stored = numbers
|
|
808
|
+
numbers = [
|
|
809
|
+
_compared(n, side.conversion) for n, side in zip(stored, evidence.sides, strict=True)
|
|
810
|
+
]
|
|
811
|
+
base["sides"] = tuple(
|
|
812
|
+
recorded.model_copy(update={"converted": _plain(number)})
|
|
813
|
+
if recorded.conversion is not None
|
|
814
|
+
else recorded
|
|
815
|
+
for recorded, number in zip(base["sides"], numbers, strict=True)
|
|
816
|
+
)
|
|
817
|
+
|
|
818
|
+
left, right = evidence.sides
|
|
819
|
+
a, b = numbers
|
|
820
|
+
held = [
|
|
821
|
+
_held(was, number, side.conversion)
|
|
822
|
+
for was, number, side in zip(stored, numbers, evidence.sides, strict=True)
|
|
823
|
+
]
|
|
824
|
+
# `printed_precision` compares at the coarser of the two precisions, so a document
|
|
825
|
+
# printing 0.65 agrees with a file holding 0.6478 and the answer does not depend on
|
|
826
|
+
# which side the manifest wrote first. A larger exponent is the coarser value.
|
|
827
|
+
coarse, fine = (a, b) if _exponent(a) >= _exponent(b) else (b, a)
|
|
828
|
+
agrees = _agree(fine, coarse, evidence.mode, evidence.tolerance_value)
|
|
829
|
+
tie = _tied(fine, coarse, evidence.mode)
|
|
830
|
+
base["tie"] = tie
|
|
831
|
+
return Decision(
|
|
832
|
+
**base,
|
|
833
|
+
**completed,
|
|
834
|
+
extraction=ExtractionStatus.EXTRACTED,
|
|
835
|
+
comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
|
|
836
|
+
reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
|
|
837
|
+
detail=f"{evidence.name}: {left.name} {held[0]}, {right.name} {held[1]}"
|
|
838
|
+
+ (f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""),
|
|
839
|
+
warnings=(
|
|
840
|
+
(Warning_.POSITIONAL_ADDRESS,)
|
|
841
|
+
if any(side.locator.kind == "table_position" for side in evidence.sides)
|
|
842
|
+
else ()
|
|
843
|
+
),
|
|
844
|
+
)
|
|
845
|
+
|
|
846
|
+
|
|
847
|
+
# -------------------------------------------------------------------------------- aggregates
|
|
848
|
+
|
|
849
|
+
|
|
850
|
+
class AggregateBackend:
|
|
851
|
+
"""Assertion: the printed value is a statistic of the values its members hold.
|
|
852
|
+
|
|
853
|
+
As many extractions as there are members precede one comparison. A member that does not
|
|
854
|
+
extract makes the comparison impossible rather than false, as one side of a correspondence
|
|
855
|
+
does: a statistic over the members that did resolve is a statistic over a set the manifest
|
|
856
|
+
did not declare. The decision carries the first failing member's own reason and its detail
|
|
857
|
+
names every member that failed, by its position in the list and its artifact.
|
|
858
|
+
|
|
859
|
+
The statistic is computed over rationals, so a mean is the exact quotient and is rounded
|
|
860
|
+
once, by the comparison.
|
|
861
|
+
"""
|
|
862
|
+
|
|
863
|
+
kind = "aggregate"
|
|
864
|
+
version = "1"
|
|
865
|
+
#: As for `CorrespondenceBackend`: one field where the members may be read by more than
|
|
866
|
+
#: one program, and `DecisionMember.extraction_digest` is what moves when any of them does.
|
|
867
|
+
tool = ADAPTER_DISTRIBUTION
|
|
868
|
+
|
|
869
|
+
@property
|
|
870
|
+
def tool_version(self) -> str:
|
|
871
|
+
return adapter_version()
|
|
872
|
+
|
|
873
|
+
def check(
|
|
874
|
+
self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
|
|
875
|
+
) -> Decision:
|
|
876
|
+
if not isinstance(evidence, AggregateEvidence):
|
|
877
|
+
raise TypeError(f"AggregateBackend received {type(evidence).__name__}")
|
|
878
|
+
base = _base(claim, evidence, self)
|
|
879
|
+
base["statistic"] = evidence.statistic
|
|
880
|
+
base["conversion"] = evidence.conversion
|
|
881
|
+
no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
|
|
882
|
+
|
|
883
|
+
def named(position: int, artifact: str) -> str:
|
|
884
|
+
return f"member {position} of {evidence.count} ({artifact})"
|
|
885
|
+
|
|
886
|
+
readings = []
|
|
887
|
+
for position, member in enumerate(evidence.members, start=1):
|
|
888
|
+
try:
|
|
889
|
+
found = resolve(member.locator, paths[member.artifact])
|
|
890
|
+
except ArtifactUnreadableError as e:
|
|
891
|
+
# Still a file that could not be parsed, and the engine reports it as one.
|
|
892
|
+
# Raised again so that it says which of the members the file is.
|
|
893
|
+
raise ArtifactUnreadableError(
|
|
894
|
+
e.path, f"{named(position, member.artifact)}: {e.detail}"
|
|
895
|
+
) from e
|
|
896
|
+
readings.append((position, member, *found))
|
|
897
|
+
base["members"] = tuple(
|
|
898
|
+
DecisionMember(
|
|
899
|
+
artifact_id=member.artifact,
|
|
900
|
+
locator_digest=member.locator.digest.value,
|
|
901
|
+
extracted=extracted.raw if extracted is not None else None,
|
|
902
|
+
extraction_digest=(
|
|
903
|
+
Digest.of_text(extracted.raw).value if extracted is not None else UNKNOWN
|
|
904
|
+
),
|
|
905
|
+
)
|
|
906
|
+
for _, member, _, extracted, _ in readings
|
|
907
|
+
)
|
|
908
|
+
# Over every member's extraction, in the order the manifest lists them, so it moves
|
|
909
|
+
# whenever any member's extraction moves. Which one moved is on the member.
|
|
910
|
+
base["extraction_digest"] = Digest.of_text(
|
|
911
|
+
"\x00".join(m.extraction_digest for m in base["members"])
|
|
912
|
+
).value
|
|
913
|
+
warnings = (
|
|
914
|
+
(Warning_.POSITIONAL_ADDRESS,)
|
|
915
|
+
if any(member.locator.kind == "table_position" for member in evidence.members)
|
|
916
|
+
else ()
|
|
917
|
+
)
|
|
918
|
+
|
|
919
|
+
unsupported = [
|
|
920
|
+
(p, m, d) for p, m, r, _, d in readings if r is Resolution.FORMAT_UNSUPPORTED
|
|
921
|
+
]
|
|
922
|
+
if unsupported:
|
|
923
|
+
return Decision(
|
|
924
|
+
**base,
|
|
925
|
+
execution=ExecutionStatus.UNAVAILABLE,
|
|
926
|
+
extraction=ExtractionStatus.NOT_ATTEMPTED,
|
|
927
|
+
**no_compare,
|
|
928
|
+
reason=Reason.FORMAT_UNSUPPORTED,
|
|
929
|
+
detail="; ".join(f"{named(p, m.artifact)}: {d}" for p, m, d in unsupported),
|
|
930
|
+
warnings=warnings,
|
|
931
|
+
)
|
|
932
|
+
|
|
933
|
+
completed = {"execution": ExecutionStatus.COMPLETED}
|
|
934
|
+
unresolved = [
|
|
935
|
+
(p, m, r, d) for p, m, r, e, d in readings if r is not Resolution.RESOLVED or e is None
|
|
936
|
+
]
|
|
937
|
+
if unresolved:
|
|
938
|
+
_, member, resolution, _ = unresolved[0]
|
|
939
|
+
extraction, reason = _RESOLUTION_STATE[resolution]
|
|
940
|
+
return Decision(
|
|
941
|
+
**base,
|
|
942
|
+
**completed,
|
|
943
|
+
**no_compare,
|
|
944
|
+
extraction=extraction,
|
|
945
|
+
reason=reason or _ABSENT_REASON[member.locator.kind],
|
|
946
|
+
detail="; ".join(f"{named(p, m.artifact)}: {d}" for p, m, _, d in unresolved),
|
|
947
|
+
warnings=warnings,
|
|
948
|
+
)
|
|
949
|
+
|
|
950
|
+
# Each member in the unit the assertion declares, and the statistic is over those.
|
|
951
|
+
converted: list[decimal.Decimal] = []
|
|
952
|
+
not_numeric = []
|
|
953
|
+
for position, member, _, extracted, _ in readings:
|
|
954
|
+
raw = extracted.raw if extracted is not None else ""
|
|
955
|
+
try:
|
|
956
|
+
number = decimal.Decimal(raw)
|
|
957
|
+
except (decimal.InvalidOperation, ValueError):
|
|
958
|
+
number = decimal.Decimal("NaN")
|
|
959
|
+
if number.is_finite():
|
|
960
|
+
converted.append(_compared(number, evidence.conversion))
|
|
961
|
+
else:
|
|
962
|
+
not_numeric.append(f"{named(position, member.artifact)} holds {raw!r}")
|
|
963
|
+
if not_numeric:
|
|
964
|
+
return Decision(
|
|
965
|
+
**base,
|
|
966
|
+
**completed,
|
|
967
|
+
**no_compare,
|
|
968
|
+
extraction=ExtractionStatus.INVALID,
|
|
969
|
+
reason=Reason.VALUE_NOT_NUMERIC,
|
|
970
|
+
detail="; ".join(not_numeric),
|
|
971
|
+
warnings=warnings,
|
|
972
|
+
)
|
|
973
|
+
values = [fractions.Fraction(number) for number in converted]
|
|
974
|
+
|
|
975
|
+
if evidence.conversion is not None:
|
|
976
|
+
base["members"] = tuple(
|
|
977
|
+
recorded.model_copy(update={"converted": _plain(number)})
|
|
978
|
+
for recorded, number in zip(base["members"], converted, strict=True)
|
|
979
|
+
)
|
|
980
|
+
|
|
981
|
+
if (misplaced := _misplaced(evidence)) is not None:
|
|
982
|
+
return Decision(
|
|
983
|
+
**base,
|
|
984
|
+
**completed,
|
|
985
|
+
extraction=ExtractionStatus.EXTRACTED,
|
|
986
|
+
comparison=ComparisonStatus.MISMATCH,
|
|
987
|
+
reason=Reason.PLACES_MISMATCH,
|
|
988
|
+
detail=misplaced,
|
|
989
|
+
warnings=warnings,
|
|
990
|
+
)
|
|
991
|
+
|
|
992
|
+
computed = _aggregate(evidence.statistic, values)
|
|
993
|
+
agrees = _agree_exact(computed, evidence.value, evidence.mode, evidence.tolerance_value)
|
|
994
|
+
shown = _shown(computed, evidence.value)
|
|
995
|
+
base["computed"] = shown
|
|
996
|
+
tie = _tied(computed, evidence.value, evidence.mode)
|
|
997
|
+
base["tie"] = tie
|
|
998
|
+
between = f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""
|
|
999
|
+
unit = f" {evidence.conversion.said}" if evidence.conversion is not None else ""
|
|
1000
|
+
over = f"{evidence.statistic.value} of {evidence.count}{unit}"
|
|
1001
|
+
return Decision(
|
|
1002
|
+
**base,
|
|
1003
|
+
**completed,
|
|
1004
|
+
extraction=ExtractionStatus.EXTRACTED,
|
|
1005
|
+
comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
|
|
1006
|
+
reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
|
|
1007
|
+
detail=(
|
|
1008
|
+
f"{evidence.name}: {over} = {shown}{between}"
|
|
1009
|
+
+ (f"; the manuscript prints {evidence.reported}" if tie else "")
|
|
1010
|
+
if agrees
|
|
1011
|
+
else f"{evidence.name}: manuscript prints {evidence.reported}, "
|
|
1012
|
+
f"{over} is {shown}{between}"
|
|
1013
|
+
),
|
|
1014
|
+
warnings=warnings,
|
|
1015
|
+
)
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
DEFAULT_BACKENDS: tuple[Backend, ...] = (
|
|
1019
|
+
QuoteBackend(),
|
|
1020
|
+
MetricBackend(),
|
|
1021
|
+
TableBackend(),
|
|
1022
|
+
ValueBackend(),
|
|
1023
|
+
CorrespondenceBackend(),
|
|
1024
|
+
AggregateBackend(),
|
|
1025
|
+
)
|
|
1026
|
+
|
|
1027
|
+
|
|
1028
|
+
# ------------------------------------------------------------------------------------ engine
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def _artifact_state(artifact: ArtifactRef, path: pathlib.Path) -> ArtifactState:
|
|
1032
|
+
if not path.exists():
|
|
1033
|
+
# Not authoritative, whether or not it was pinned: there are no bytes to be the
|
|
1034
|
+
# declared ones. Calling it authoritative made every decision against it claim to
|
|
1035
|
+
# describe a file that was never read.
|
|
1036
|
+
return ArtifactState(
|
|
1037
|
+
artifact_id=artifact.id,
|
|
1038
|
+
validity=Validity.ARTIFACT_ABSENT,
|
|
1039
|
+
exists=False,
|
|
1040
|
+
expected=artifact.digest.value if artifact.digest else None,
|
|
1041
|
+
)
|
|
1042
|
+
# A path that exists but cannot be hashed -- a directory, a mode-000 file, a dangling
|
|
1043
|
+
# symlink -- is a fact about that artifact. Raising suppressed the report for every other
|
|
1044
|
+
# claim in the manifest because of one bad path.
|
|
1045
|
+
try:
|
|
1046
|
+
actual = Digest.of_file(path).value
|
|
1047
|
+
except OSError as e:
|
|
1048
|
+
return ArtifactState(
|
|
1049
|
+
artifact_id=artifact.id,
|
|
1050
|
+
validity=Validity.ARTIFACT_ABSENT,
|
|
1051
|
+
expected=artifact.digest.value if artifact.digest else None,
|
|
1052
|
+
exists=False,
|
|
1053
|
+
detail=f"cannot be read: {e.strerror or e}",
|
|
1054
|
+
)
|
|
1055
|
+
if not artifact.is_pinned or artifact.digest is None:
|
|
1056
|
+
return ArtifactState(
|
|
1057
|
+
artifact_id=artifact.id,
|
|
1058
|
+
validity=Validity.UNPINNED_ARTIFACT,
|
|
1059
|
+
actual=actual,
|
|
1060
|
+
)
|
|
1061
|
+
if actual != artifact.digest.value:
|
|
1062
|
+
return ArtifactState(
|
|
1063
|
+
artifact_id=artifact.id,
|
|
1064
|
+
validity=Validity.BROKEN_PIN,
|
|
1065
|
+
expected=artifact.digest.value,
|
|
1066
|
+
actual=actual,
|
|
1067
|
+
)
|
|
1068
|
+
return ArtifactState(
|
|
1069
|
+
artifact_id=artifact.id,
|
|
1070
|
+
validity=Validity.AUTHORITATIVE,
|
|
1071
|
+
expected=artifact.digest.value,
|
|
1072
|
+
actual=actual,
|
|
1073
|
+
)
|
|
1074
|
+
|
|
1075
|
+
|
|
1076
|
+
def _ordering(
|
|
1077
|
+
claim: Claim, manifest: Manifest, states: dict[str, ArtifactState]
|
|
1078
|
+
) -> tuple[Ordering, OrderingReason, str, RegistrationAuthority | None]:
|
|
1079
|
+
"""Did the runs producing this claim's evidence start after its plan was registered?
|
|
1080
|
+
|
|
1081
|
+
Returns an outcome, the reason it was reached, a human-readable detail, and the weakest
|
|
1082
|
+
registration authority among the runs that applied. Every condition that stops the check
|
|
1083
|
+
yields `unchecked` with its own reason rather than a verdict: an absent record is not
|
|
1084
|
+
evidence that a result predates its plan, and reporting it as one would manufacture a
|
|
1085
|
+
finding out of a gap.
|
|
1086
|
+
"""
|
|
1087
|
+
if not claim.confirmatory:
|
|
1088
|
+
return Ordering.NOT_APPLICABLE, OrderingReason.NOT_CONFIRMATORY, "", None
|
|
1089
|
+
|
|
1090
|
+
produced = {a for e in claim.evidence for a in e.artifacts}
|
|
1091
|
+
runs = [r for r in manifest.runs if produced & {o.artifact for o in r.outputs}]
|
|
1092
|
+
covered = {o.artifact for r in runs for o in r.outputs}
|
|
1093
|
+
uncovered = sorted(produced - covered)
|
|
1094
|
+
if uncovered:
|
|
1095
|
+
# A claim can cite several artifacts. Taking the runs that produce *any* of them meant
|
|
1096
|
+
# adding a run record for one incidental artifact turned an unregistered result into
|
|
1097
|
+
# an ordered one, while the artifact carrying the number had no run at all.
|
|
1098
|
+
return (
|
|
1099
|
+
Ordering.UNCHECKED,
|
|
1100
|
+
OrderingReason.NO_RUN_RECORD,
|
|
1101
|
+
f"no run record produces {', '.join(uncovered)}",
|
|
1102
|
+
None,
|
|
1103
|
+
)
|
|
1104
|
+
|
|
1105
|
+
if not runs:
|
|
1106
|
+
# A confirmatory claim offering no evidence names no artifacts, so no run produces any
|
|
1107
|
+
# and `uncovered` above is empty too. `min()` then raised on an empty iterable and the
|
|
1108
|
+
# traceback replaced the finding: the publication policy grades this manifest
|
|
1109
|
+
# `claim.no_evidence`, which is the thing the author needs to be told.
|
|
1110
|
+
return (
|
|
1111
|
+
Ordering.UNCHECKED,
|
|
1112
|
+
OrderingReason.NO_EVIDENCE_OFFERED,
|
|
1113
|
+
"the claim offers no evidence, so there is nothing to order against a plan",
|
|
1114
|
+
None,
|
|
1115
|
+
)
|
|
1116
|
+
|
|
1117
|
+
authority = min((r.registration_authority for r in runs), key=lambda a: a.rank)
|
|
1118
|
+
|
|
1119
|
+
def unchecked(reason: OrderingReason, detail: str):
|
|
1120
|
+
return Ordering.UNCHECKED, reason, detail, authority
|
|
1121
|
+
|
|
1122
|
+
# More than one run claiming the same artifact with different registrations does not say
|
|
1123
|
+
# which one produced the bytes being checked.
|
|
1124
|
+
for artifact_id in sorted(produced):
|
|
1125
|
+
claimants = [r for r in runs if r.output(artifact_id) is not None]
|
|
1126
|
+
if len({(r.registered_plan, r.registered_at) for r in claimants}) > 1:
|
|
1127
|
+
return unchecked(
|
|
1128
|
+
OrderingReason.AMBIGUOUS_PRODUCING_RUN,
|
|
1129
|
+
f"{len(claimants)} runs claim to produce {artifact_id}",
|
|
1130
|
+
)
|
|
1131
|
+
|
|
1132
|
+
without_plan = sorted({r.id for r in runs if not r.registered_plan})
|
|
1133
|
+
if without_plan:
|
|
1134
|
+
return unchecked(
|
|
1135
|
+
OrderingReason.NO_REGISTERED_PLAN,
|
|
1136
|
+
f"run {', '.join(without_plan)} names no registered plan",
|
|
1137
|
+
)
|
|
1138
|
+
|
|
1139
|
+
undated = sorted({r.id for r in runs if r.registered_at is None or r.started_at is None})
|
|
1140
|
+
if undated:
|
|
1141
|
+
return unchecked(
|
|
1142
|
+
OrderingReason.TIMESTAMP_MISSING,
|
|
1143
|
+
f"run {', '.join(undated)} records no registration or start time",
|
|
1144
|
+
)
|
|
1145
|
+
|
|
1146
|
+
# The plan document, where it is a declared artifact, must be the one that was pinned.
|
|
1147
|
+
for run in runs:
|
|
1148
|
+
state = states.get(run.registered_plan)
|
|
1149
|
+
if state is None:
|
|
1150
|
+
# Undeclared, so nothing pins it: the document backing an `ordered` verdict could
|
|
1151
|
+
# have been written after the results. Declaring the plan and leaving it unpinned
|
|
1152
|
+
# reported `unchecked`, so the more transparent manifest scored strictly worse.
|
|
1153
|
+
return unchecked(
|
|
1154
|
+
OrderingReason.REGISTERED_PLAN_UNPINNED,
|
|
1155
|
+
f"plan {run.registered_plan} is not a declared artifact, so nothing pins the "
|
|
1156
|
+
f"document the ordering rests on",
|
|
1157
|
+
)
|
|
1158
|
+
if state.validity is Validity.BROKEN_PIN:
|
|
1159
|
+
return unchecked(
|
|
1160
|
+
OrderingReason.REGISTERED_PLAN_CHANGED,
|
|
1161
|
+
f"plan {run.registered_plan} is not the document that was "
|
|
1162
|
+
f"pinned, so its timestamp attests to nothing",
|
|
1163
|
+
)
|
|
1164
|
+
if state.validity is not Validity.AUTHORITATIVE:
|
|
1165
|
+
return unchecked(
|
|
1166
|
+
OrderingReason.REGISTERED_PLAN_UNPINNED,
|
|
1167
|
+
f"plan {run.registered_plan} is declared but {state.validity.value}",
|
|
1168
|
+
)
|
|
1169
|
+
|
|
1170
|
+
# Each output must be bound to the bytes actually read, not merely to a path.
|
|
1171
|
+
for run in runs:
|
|
1172
|
+
for artifact_id in sorted(produced & {o.artifact for o in run.outputs}):
|
|
1173
|
+
output = run.output(artifact_id)
|
|
1174
|
+
if output is None or output.digest is None:
|
|
1175
|
+
return unchecked(
|
|
1176
|
+
OrderingReason.RUN_OUTPUT_UNLINKED,
|
|
1177
|
+
f"run {run.id} names {artifact_id} but records no digest "
|
|
1178
|
+
f"for it, so any later file at that path would qualify",
|
|
1179
|
+
)
|
|
1180
|
+
actual = states[artifact_id].actual if artifact_id in states else None
|
|
1181
|
+
if actual is not None and actual != output.digest.value:
|
|
1182
|
+
return unchecked(
|
|
1183
|
+
OrderingReason.RUN_OUTPUT_CHANGED,
|
|
1184
|
+
f"{artifact_id} holds {actual[:12]}, but run {run.id} "
|
|
1185
|
+
f"produced {output.digest.value[:12]}",
|
|
1186
|
+
)
|
|
1187
|
+
|
|
1188
|
+
# Every run is dated by this point -- the guard above returned otherwise -- but binding
|
|
1189
|
+
# the timestamps is what makes that legible, to a reader and to a type checker.
|
|
1190
|
+
dated = [
|
|
1191
|
+
(r, r.registered_at, r.started_at)
|
|
1192
|
+
for r in runs
|
|
1193
|
+
if r.registered_at is not None and r.started_at is not None
|
|
1194
|
+
]
|
|
1195
|
+
inverted = [
|
|
1196
|
+
(r, registered, started) for r, registered, started in dated if started < registered
|
|
1197
|
+
]
|
|
1198
|
+
if inverted:
|
|
1199
|
+
run, registered_at, started_at = inverted[0]
|
|
1200
|
+
return (
|
|
1201
|
+
Ordering.VIOLATED,
|
|
1202
|
+
OrderingReason.RUN_PRECEDES_REGISTRATION,
|
|
1203
|
+
f"run {run.id} started {started_at.isoformat()}, before "
|
|
1204
|
+
f"{run.registered_plan} was registered {registered_at.isoformat()}",
|
|
1205
|
+
authority,
|
|
1206
|
+
)
|
|
1207
|
+
|
|
1208
|
+
pinned = sum(1 for r in runs if r.registered_plan in states)
|
|
1209
|
+
detail = (
|
|
1210
|
+
f"{len(runs)} run(s) started after registration"
|
|
1211
|
+
+ (f", {pinned} against a pinned plan" if pinned else "")
|
|
1212
|
+
+ f"; authority {authority.value}"
|
|
1213
|
+
)
|
|
1214
|
+
return Ordering.ORDERED, OrderingReason.RUN_FOLLOWS_REGISTRATION, detail, authority
|
|
1215
|
+
|
|
1216
|
+
|
|
1217
|
+
def artifact_states(
|
|
1218
|
+
manifest: Manifest,
|
|
1219
|
+
) -> tuple[dict[str, ArtifactState], dict[str, pathlib.Path]]:
|
|
1220
|
+
"""Each declared artifact's pin, resolved once, and the path it was read from."""
|
|
1221
|
+
states: dict[str, ArtifactState] = {}
|
|
1222
|
+
paths: dict[str, pathlib.Path] = {}
|
|
1223
|
+
for artifact in manifest.artifacts:
|
|
1224
|
+
path = manifest.resolve(artifact)
|
|
1225
|
+
paths[artifact.id] = path
|
|
1226
|
+
states[artifact.id] = _artifact_state(artifact, path)
|
|
1227
|
+
return states, paths
|
|
1228
|
+
|
|
1229
|
+
|
|
1230
|
+
def verify(
|
|
1231
|
+
manifest: Manifest, backends: tuple[Backend, ...] = DEFAULT_BACKENDS
|
|
1232
|
+
) -> VerificationReport:
|
|
1233
|
+
"""Check every evidence assertion in a manifest and report what was found.
|
|
1234
|
+
|
|
1235
|
+
Returns facts. No verdict: see `claims.policy` for whether a given set of facts is
|
|
1236
|
+
acceptable, which depends on what the project is for.
|
|
1237
|
+
|
|
1238
|
+
Executes nothing. A manifest may declare commands that regenerate its artifacts; running
|
|
1239
|
+
them is `repro.reproduce`, which has to be asked for by name.
|
|
1240
|
+
"""
|
|
1241
|
+
registry: dict[str, Backend] = {b.kind: b for b in backends}
|
|
1242
|
+
states, paths = artifact_states(manifest)
|
|
1243
|
+
|
|
1244
|
+
assessments = []
|
|
1245
|
+
# Each artifact is parsed once for the whole manifest, just after it was hashed.
|
|
1246
|
+
with reading_once():
|
|
1247
|
+
for claim in manifest.claims:
|
|
1248
|
+
decisions = tuple(
|
|
1249
|
+
check_evidence(claim, evidence, manifest, paths, states, registry)
|
|
1250
|
+
for evidence in claim.evidence
|
|
1251
|
+
)
|
|
1252
|
+
ordering, ordering_reason, ordering_detail, authority = _ordering(
|
|
1253
|
+
claim, manifest, states
|
|
1254
|
+
)
|
|
1255
|
+
assessments.append(
|
|
1256
|
+
ClaimAssessment(
|
|
1257
|
+
claim_id=claim.id,
|
|
1258
|
+
claim_digest=claim.digest.value,
|
|
1259
|
+
confirmatory=claim.confirmatory,
|
|
1260
|
+
registration=claim.registration,
|
|
1261
|
+
registration_note=claim.registration_note,
|
|
1262
|
+
availability=claim.availability,
|
|
1263
|
+
ordering=ordering,
|
|
1264
|
+
ordering_reason=ordering_reason,
|
|
1265
|
+
ordering_detail=ordering_detail,
|
|
1266
|
+
registration_authority=authority,
|
|
1267
|
+
decisions=decisions,
|
|
1268
|
+
)
|
|
1269
|
+
)
|
|
1270
|
+
|
|
1271
|
+
return VerificationReport(
|
|
1272
|
+
project=manifest.project,
|
|
1273
|
+
manifest_digest=Digest.of_text(manifest.model_dump_json()).value,
|
|
1274
|
+
artifacts=tuple(states.values()),
|
|
1275
|
+
claims=tuple(assessments),
|
|
1276
|
+
)
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def check_evidence(
|
|
1280
|
+
claim: Claim,
|
|
1281
|
+
evidence: Evidence,
|
|
1282
|
+
manifest: Manifest,
|
|
1283
|
+
paths: dict[str, pathlib.Path],
|
|
1284
|
+
states: dict[str, ArtifactState],
|
|
1285
|
+
registry: dict[str, Backend],
|
|
1286
|
+
) -> Decision:
|
|
1287
|
+
"""Evaluate one assertion against the files in `paths`.
|
|
1288
|
+
|
|
1289
|
+
`paths` is a parameter and not read off the manifest so that a re-run can ask the same
|
|
1290
|
+
question of the file it has just written: `repro.reproduce` passes the path of the fresh
|
|
1291
|
+
output in place of the pinned one, and every rule below applies unchanged.
|
|
1292
|
+
"""
|
|
1293
|
+
backend = registry.get(evidence.kind)
|
|
1294
|
+
if backend is None:
|
|
1295
|
+
raise UnknownEvidenceKindError(evidence.kind, tuple(registry))
|
|
1296
|
+
|
|
1297
|
+
base = _base(claim, evidence, backend)
|
|
1298
|
+
unchecked = {
|
|
1299
|
+
"execution": ExecutionStatus.UNAVAILABLE,
|
|
1300
|
+
"extraction": ExtractionStatus.NOT_ATTEMPTED,
|
|
1301
|
+
"comparison": ComparisonStatus.NOT_APPLICABLE,
|
|
1302
|
+
}
|
|
1303
|
+
|
|
1304
|
+
# Every artifact the assertion names, not only the first. An assertion reading two files
|
|
1305
|
+
# is unchecked if either is undeclared or absent, and no more authoritative than the worse
|
|
1306
|
+
# of the two pins.
|
|
1307
|
+
named = evidence.artifacts
|
|
1308
|
+
undeclared = [a for a in named if manifest.artifact(a) is None]
|
|
1309
|
+
if undeclared:
|
|
1310
|
+
return Decision(
|
|
1311
|
+
**base,
|
|
1312
|
+
**unchecked,
|
|
1313
|
+
reason=Reason.ARTIFACT_UNDECLARED,
|
|
1314
|
+
detail=f"manifest declares no artifact {', '.join(repr(a) for a in undeclared)}",
|
|
1315
|
+
)
|
|
1316
|
+
missing = [a for a in named if not states[a].exists]
|
|
1317
|
+
if missing:
|
|
1318
|
+
return Decision(
|
|
1319
|
+
**base,
|
|
1320
|
+
**unchecked,
|
|
1321
|
+
reason=Reason.ARTIFACT_MISSING,
|
|
1322
|
+
detail="; ".join(f"{paths[a]} does not exist" for a in missing),
|
|
1323
|
+
validity=min((states[a].validity for a in missing), key=_VALIDITY_RANK.__getitem__),
|
|
1324
|
+
)
|
|
1325
|
+
|
|
1326
|
+
try:
|
|
1327
|
+
decision = backend.check(claim, evidence, {a: paths[a] for a in named})
|
|
1328
|
+
except BackendUnavailableError as e:
|
|
1329
|
+
decision = Decision(**base, **unchecked, reason=Reason.EXTRACTOR_MISSING, detail=e.detail)
|
|
1330
|
+
except ArtifactUnreadableError as e:
|
|
1331
|
+
decision = Decision(**base, **unchecked, reason=Reason.ARTIFACT_UNREADABLE, detail=e.detail)
|
|
1332
|
+
except Exception as e:
|
|
1333
|
+
# A defect in a backend is a defect, not an abstention. Recording it as `unchecked`
|
|
1334
|
+
# would make a TypeError indistinguishable from a missing extractor, which is the
|
|
1335
|
+
# confusion this package exists to prevent.
|
|
1336
|
+
decision = Decision(
|
|
1337
|
+
**base,
|
|
1338
|
+
execution=ExecutionStatus.FAILED,
|
|
1339
|
+
extraction=ExtractionStatus.NOT_ATTEMPTED,
|
|
1340
|
+
comparison=ComparisonStatus.NOT_APPLICABLE,
|
|
1341
|
+
reason=Reason.BACKEND_DEFECT,
|
|
1342
|
+
detail=f"{type(e).__name__}: {e}",
|
|
1343
|
+
)
|
|
1344
|
+
|
|
1345
|
+
validity = min((states[a].validity for a in named), key=_VALIDITY_RANK.__getitem__)
|
|
1346
|
+
return decision.model_copy(
|
|
1347
|
+
update={
|
|
1348
|
+
"validity": validity,
|
|
1349
|
+
"artifact_digest": states[named[0]].actual if len(named) == 1 else None,
|
|
1350
|
+
# The backend knows which locator addressed which side; only the engine knows what
|
|
1351
|
+
# was at each path. Filling the digests here keeps both in one record.
|
|
1352
|
+
"sides": tuple(
|
|
1353
|
+
side.model_copy(
|
|
1354
|
+
update={
|
|
1355
|
+
"artifact_digest": states[side.artifact_id].actual,
|
|
1356
|
+
"validity": states[side.artifact_id].validity,
|
|
1357
|
+
}
|
|
1358
|
+
)
|
|
1359
|
+
for side in decision.sides
|
|
1360
|
+
),
|
|
1361
|
+
"members": tuple(
|
|
1362
|
+
member.model_copy(
|
|
1363
|
+
update={
|
|
1364
|
+
"artifact_digest": states[member.artifact_id].actual,
|
|
1365
|
+
"validity": states[member.artifact_id].validity,
|
|
1366
|
+
}
|
|
1367
|
+
)
|
|
1368
|
+
for member in decision.members
|
|
1369
|
+
),
|
|
1370
|
+
}
|
|
1371
|
+
)
|