claims-cli 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. claims/__init__.py +132 -0
  2. claims/__main__.py +5 -0
  3. claims/adapters/__init__.py +6 -0
  4. claims/adapters/array.py +212 -0
  5. claims/adapters/base.py +75 -0
  6. claims/adapters/cells.py +94 -0
  7. claims/adapters/columnar.py +188 -0
  8. claims/adapters/prose.py +261 -0
  9. claims/adapters/reads.py +47 -0
  10. claims/adapters/rows.py +116 -0
  11. claims/adapters/sheet.py +149 -0
  12. claims/adapters/sqlite.py +73 -0
  13. claims/adapters/table.py +135 -0
  14. claims/adapters/tree.py +163 -0
  15. claims/checks.py +511 -0
  16. claims/cli.py +1103 -0
  17. claims/core/__init__.py +9 -0
  18. claims/core/artifacts.py +121 -0
  19. claims/core/checks.py +132 -0
  20. claims/core/claims.py +100 -0
  21. claims/core/evidence.py +708 -0
  22. claims/core/manifest.py +349 -0
  23. claims/core/numbers.py +146 -0
  24. claims/core/outcomes.py +410 -0
  25. claims/core/report.py +503 -0
  26. claims/corpus.py +344 -0
  27. claims/exceptions.py +106 -0
  28. claims/export/__init__.py +23 -0
  29. claims/export/rocrate.py +787 -0
  30. claims/export/sarif.py +401 -0
  31. claims/manifest.py +59 -0
  32. claims/models.py +222 -0
  33. claims/numbers.py +453 -0
  34. claims/origins.py +303 -0
  35. claims/policy.py +369 -0
  36. claims/provenance.py +42 -0
  37. claims/record.py +1099 -0
  38. claims/regression.py +171 -0
  39. claims/renderers/__init__.py +10 -0
  40. claims/renderers/sarif.py +294 -0
  41. claims/reruns.py +111 -0
  42. claims/resolve.py +116 -0
  43. claims/stale.py +333 -0
  44. claims/starter.py +432 -0
  45. claims/toolchain.py +120 -0
  46. claims/verify.py +1371 -0
  47. claims/why.py +398 -0
  48. claims_cli-0.6.0.dist-info/METADATA +139 -0
  49. claims_cli-0.6.0.dist-info/RECORD +51 -0
  50. claims_cli-0.6.0.dist-info/WHEEL +4 -0
  51. claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/verify.py ADDED
@@ -0,0 +1,1371 @@
1
+ """The verification engine.
2
+
3
+ The engine reads no artifacts itself. It resolves each artifact's pin once, dispatches each
4
+ evidence assertion to the backend registered for its kind, and records what came back. What
5
+ it adds on top of the backends is the part that has to be uniform:
6
+
7
+ * **Pins are checked before evidence.** A source that moved is known before any decision is
8
+ computed against it, and those decisions are marked non-authoritative rather than being
9
+ silently reported as if the declared file had been read.
10
+ * **A backend that cannot run is not a claim that failed.** `BackendUnavailableError`
11
+ becomes `execution=unavailable` with the backend's own reason attached.
12
+ * **A defect is not a scientific outcome.** Any other exception becomes
13
+ `execution=failed`, which flattens to `error` and never to `unchecked`. Letting a
14
+ `TypeError` become an abstention is the same failure this package exists to prevent, one
15
+ level up.
16
+ * **Every decision names the toolchain that read the bytes**, and the digest of what that
17
+ toolchain produced. See `claims.toolchain` for why the backend's protocol version is not
18
+ that.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import decimal
24
+ import fractions
25
+ import math
26
+ import pathlib
27
+ import statistics
28
+ from collections.abc import Mapping
29
+ from typing import Protocol
30
+
31
+ from claims.adapters.reads import reading_once
32
+ from claims.exceptions import (
33
+ ArtifactUnreadableError,
34
+ BackendUnavailableError,
35
+ UnknownEvidenceKindError,
36
+ )
37
+ from claims.models import (
38
+ AggregateEvidence,
39
+ ArtifactRef,
40
+ ArtifactState,
41
+ Claim,
42
+ ClaimAssessment,
43
+ ComparisonMode,
44
+ ComparisonStatus,
45
+ Conversion,
46
+ CorrespondenceEvidence,
47
+ Decision,
48
+ DecisionMember,
49
+ DecisionSide,
50
+ Digest,
51
+ Evidence,
52
+ ExecutionStatus,
53
+ ExtractionStatus,
54
+ Manifest,
55
+ MetricEvidence,
56
+ Ordering,
57
+ OrderingReason,
58
+ QuoteEvidence,
59
+ Reason,
60
+ RegistrationAuthority,
61
+ Statistic,
62
+ TableCellEvidence,
63
+ Tie,
64
+ Validity,
65
+ ValueEvidence,
66
+ VerificationReport,
67
+ Warning_,
68
+ )
69
+ from claims.resolve import Resolution, resolve
70
+ from claims.toolchain import DISTRIBUTION, UNKNOWN, adapter_version, binary_version
71
+
72
+ #: The distribution that ships `claims.adapters`, which is what reads a JSON pointer, a
73
+ #: delimited cell, a SQLite row or an array element. Its name on PyPI, because that is what
74
+ #: `importlib.metadata` answers to.
75
+ ADAPTER_DISTRIBUTION = DISTRIBUTION
76
+
77
+
78
+ class Backend(Protocol):
79
+ """Evaluates one kind of evidence assertion against the artifacts it names.
80
+
81
+ `paths` holds one entry per artifact the assertion reads, keyed by id. Every kind but
82
+ `correspondence` reads one, and takes it with `paths[evidence.artifact]`. A mapping rather
83
+ than a path because an assertion whose two sides are both artifacts has no single file to
84
+ be handed, and giving it one would make the engine pick a side.
85
+ """
86
+
87
+ kind: str
88
+ version: str
89
+ """The protocol version of this interface, written by hand."""
90
+ tool: str
91
+ """What performs the extraction: a binary's name, or an installed distribution's."""
92
+
93
+ @property
94
+ def tool_version(self) -> str:
95
+ """The version `tool` reports, or `toolchain.UNKNOWN`."""
96
+ ...
97
+
98
+ def check(
99
+ self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
100
+ ) -> Decision: ...
101
+
102
+
103
+ #: Weakest first. A decision over two artifacts is no more authoritative than the worse of
104
+ #: them, and taking the better one would report a comparison against a file that moved as
105
+ #: though the declared bytes had been read.
106
+ _VALIDITY_RANK = {
107
+ Validity.ARTIFACT_ABSENT: 0,
108
+ Validity.BROKEN_PIN: 1,
109
+ Validity.UNPINNED_ARTIFACT: 2,
110
+ Validity.AUTHORITATIVE: 3,
111
+ }
112
+
113
+
114
+ def _base(
115
+ claim: Claim, evidence: Evidence, backend: Backend, extraction_digest: str = UNKNOWN
116
+ ) -> dict:
117
+ """The fields every decision carries, whichever stage it stopped at.
118
+
119
+ `extraction_digest` defaults to `unknown` rather than to an empty string: a decision the
120
+ engine produced was one an extraction was sought for, and the empty value is reserved for
121
+ a decision no backend produced.
122
+ """
123
+ named = evidence.artifacts
124
+ return {
125
+ "claim_id": claim.id,
126
+ "claim_digest": claim.digest.value,
127
+ "kind": evidence.kind,
128
+ # Left empty where an assertion reads two files or more: `artifact_id` is singular,
129
+ # and filling it with one of them would name a side the engine does not rank. They are
130
+ # recorded in `sides`, or in `members`.
131
+ "artifact_id": named[0] if len(named) == 1 else "",
132
+ "backend": backend.kind,
133
+ "backend_version": backend.version,
134
+ "tool": backend.tool,
135
+ "tool_version": backend.tool_version,
136
+ "extraction_digest": extraction_digest,
137
+ }
138
+
139
+
140
+ # -------------------------------------------------------------------------------- quotations
141
+
142
+ #: Every state `citations.verify` can return. It answers with five, and only two were mapped, so
143
+ #: a passage occurring twice in its source raised `KeyError: 'ambiguous'` from inside the backend
144
+ #: and was reported as a defect in the tool rather than as an undecided quotation.
145
+ _QUOTE_STATE = {
146
+ "found": (ExtractionStatus.EXTRACTED, ComparisonStatus.MATCH, Reason.PASSAGE_PRESENT),
147
+ "not found": (ExtractionStatus.EXTRACTED, ComparisonStatus.MISMATCH, Reason.PASSAGE_ABSENT),
148
+ # The address matched more than once, which is where `row_ambiguous` sits for a table: the
149
+ # extraction did not single out one thing to compare, so there is no comparison, and no first
150
+ # match is taken. Not `passage_ambiguous`, which says the document states two different
151
+ # numbers: a passage written twice contradicts nothing, and an author told it does goes
152
+ # looking for a defect in the manuscript when the fix is a `prefix` or `suffix` on the quote
153
+ # assertion. One that singles out no occurrence is this state too, and `citations` says so
154
+ # in the detail: the record names text on either side and still has not said which it means.
155
+ "ambiguous": (
156
+ ExtractionStatus.INVALID,
157
+ ComparisonStatus.NOT_APPLICABLE,
158
+ Reason.QUOTATION_AMBIGUOUS,
159
+ ),
160
+ # Not `passage_ambiguous` either: that says the document states two numbers where one was
161
+ # asserted. This says the extractors on this machine do not settle what text the document
162
+ # holds, which accuses nothing and asks for a better reader.
163
+ "indeterminate": (
164
+ ExtractionStatus.INVALID,
165
+ ComparisonStatus.NOT_APPLICABLE,
166
+ Reason.EXTRACTORS_DISAGREE,
167
+ ),
168
+ # Not `passage_absent`. The stages were right and the reason was not, so an author whose
169
+ # extractor never ran was told their quotation is missing from the source -- the "nothing ran"
170
+ # against "not there" collapse this package exists to keep apart.
171
+ "unchecked": (
172
+ ExtractionStatus.NOT_ATTEMPTED,
173
+ ComparisonStatus.NOT_APPLICABLE,
174
+ Reason.EXTRACTOR_MISSING,
175
+ ),
176
+ }
177
+
178
+ #: Every warning `citations.verify` can put on a result. `page unchecked` was not here, and a
179
+ #: warning with no entry is dropped, so the one warning that says a check was skipped was the
180
+ #: one `repro verify` never showed.
181
+ _QUOTE_WARNINGS = {
182
+ "short": Warning_.SHORT,
183
+ "truncated": Warning_.TRUNCATED,
184
+ "normalized": Warning_.NORMALIZED,
185
+ "page": Warning_.WRONG_PAGE,
186
+ "page unchecked": Warning_.PAGE_UNCHECKED,
187
+ }
188
+
189
+
190
+ def _page_unchecked(page: int | None, state: str, warnings: list[str], paginated: bool) -> str:
191
+ """Why a declared page was not checked on a passage that was found, or empty where it was.
192
+
193
+ `citations` gives the warning `page unchecked` for each of these. Before 0.5.6 it gave it
194
+ only where a source declares its own extractor, which a manifest cannot do, and the two
195
+ cases a manifest reaches were silent: a source that is not read page by page had its page
196
+ dropped before the passage was looked for, and a passage that matches only with spacing
197
+ ignored was returned before the page was looked at. Both are still worked out here, so
198
+ the answer is the same against an earlier `citations` of the series this package accepts.
199
+ """
200
+ if page is None or state != "found" or "page" in warnings:
201
+ return ""
202
+ if not paginated:
203
+ return f"page {page} was not checked: the artifact is not read page by page"
204
+ if "normalized" in warnings:
205
+ return f"page {page} was not checked: the passage matched only with spacing ignored"
206
+ if "page unchecked" in warnings:
207
+ return f"page {page} was not checked: the extractor cannot be asked for one page"
208
+ return ""
209
+
210
+
211
+ class QuoteBackend:
212
+ """Assertion: this passage occurs in this artifact.
213
+
214
+ A passage that is absent from a source that was read successfully is a `mismatch`: the
215
+ artifact was extracted and does not contain what the assertion says it contains. That is
216
+ a different fact from a source that could not be read, which is `unavailable`.
217
+ """
218
+
219
+ kind = "quote"
220
+ version = "1"
221
+ tool = "pdftotext"
222
+
223
+ @property
224
+ def tool_version(self) -> str:
225
+ return binary_version(self.tool)
226
+
227
+ def check(
228
+ self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
229
+ ) -> Decision:
230
+ if not isinstance(evidence, QuoteEvidence):
231
+ raise TypeError(f"QuoteBackend received {type(evidence).__name__}")
232
+ try:
233
+ # Imported here rather than at module scope so an absent `citations` reports as a
234
+ # backend that could not run, which is what the caller can act on.
235
+ from citations.exceptions import SourceUnreadableError
236
+ from citations.verify import check_one, extract, is_paginated
237
+ except ImportError as e:
238
+ raise BackendUnavailableError("quote", f"citations is not installed: {e}") from e
239
+
240
+ path = paths[evidence.artifact]
241
+ result = check_one(
242
+ evidence.text, path, evidence.page, prefix=evidence.prefix, suffix=evidence.suffix
243
+ )
244
+ # The page of a passage that was not found is not a second finding beside it.
245
+ named = [w for w in result.warnings if w != "page unchecked" or result.state == "found"]
246
+ # Asked only of a source that was read: `is_paginated` opens the file to tell text from
247
+ # a format with pages.
248
+ paginated = result.state == "found" and is_paginated(path)
249
+ unchecked = _page_unchecked(evidence.page, result.state, named, paginated)
250
+ if unchecked and "page unchecked" not in named:
251
+ named.append("page unchecked")
252
+ warnings = tuple(_QUOTE_WARNINGS[w] for w in named if w in _QUOTE_WARNINGS)
253
+
254
+ if result.state == "unchecked":
255
+ # The extractor could not produce text. Distinguishing this from an absent
256
+ # passage is the reason the three stages are separate.
257
+ raise BackendUnavailableError("quote", result.detail or "no text extracted")
258
+
259
+ try:
260
+ # `extract` memoizes per file, so this is the text `check_one` has just read
261
+ # rather than a second run of `pdftotext`. It is the whole document: a digest over
262
+ # the matched passage alone would be silent on an extractor that changed how it
263
+ # reads every other page.
264
+ extraction_digest = Digest.of_text(extract(path)).value
265
+ except SourceUnreadableError:
266
+ extraction_digest = UNKNOWN
267
+
268
+ base = _base(claim, evidence, self, extraction_digest)
269
+ extraction, comparison, reason = _QUOTE_STATE[result.state]
270
+ if Warning_.WRONG_PAGE in warnings:
271
+ # `page` is documented as verified when present, and the passage is not on the
272
+ # page the manifest asserts. Reporting `match` with a warning made the assertion
273
+ # unenforceable: no policy reads decision warnings, so it graded as verified.
274
+ comparison, reason = ComparisonStatus.MISMATCH, Reason.WRONG_PAGE
275
+ return Decision(
276
+ **base,
277
+ execution=ExecutionStatus.COMPLETED,
278
+ extraction=extraction,
279
+ comparison=comparison,
280
+ reason=reason,
281
+ detail=result.detail or unchecked,
282
+ warnings=warnings,
283
+ )
284
+
285
+
286
+ # ------------------------------------------------------------------------------- values
287
+
288
+ #: How a resolution maps onto the extraction stage and a reason. `format_unsupported` is not
289
+ #: here: a format with no adapter means the check never ran, which is an execution fact.
290
+ _RESOLUTION_STATE = {
291
+ Resolution.ABSENT: (ExtractionStatus.ABSENT, None),
292
+ Resolution.COLUMN_ABSENT: (ExtractionStatus.ABSENT, Reason.COLUMN_ABSENT),
293
+ Resolution.AMBIGUOUS: (ExtractionStatus.INVALID, Reason.ROW_AMBIGUOUS),
294
+ Resolution.NOT_SCALAR: (ExtractionStatus.INVALID, Reason.SELECTOR_NOT_SCALAR),
295
+ Resolution.SELECTOR_INVALID: (ExtractionStatus.INVALID, Reason.ROW_SELECTOR_INVALID),
296
+ Resolution.NUMBER_AS_WORD: (ExtractionStatus.INVALID, Reason.NUMBER_AS_WORD),
297
+ Resolution.PASSAGE_AMBIGUOUS: (ExtractionStatus.INVALID, Reason.PASSAGE_AMBIGUOUS),
298
+ }
299
+
300
+ #: Which "not there" reason fits each addressing scheme.
301
+ _ABSENT_REASON = {
302
+ "tree": Reason.POINTER_ABSENT,
303
+ "table": Reason.ROW_ABSENT,
304
+ "table_position": Reason.ROW_ABSENT,
305
+ "sqlite": Reason.ROW_ABSENT,
306
+ "array": Reason.POINTER_ABSENT,
307
+ "sheet": Reason.ROW_ABSENT,
308
+ "prose": Reason.PASSAGE_ABSENT,
309
+ }
310
+
311
+
312
+ def compare_decimal(
313
+ stored: decimal.Decimal, evidence: MetricEvidence | TableCellEvidence | ValueEvidence
314
+ ) -> bool:
315
+ """Does the stored value agree with the reported one, under the declared conversion and
316
+ mode?"""
317
+ return _agree(
318
+ _compared(stored, evidence.conversion),
319
+ evidence.value,
320
+ evidence.mode,
321
+ evidence.tolerance_value,
322
+ )
323
+
324
+
325
+ def _compared(stored: decimal.Decimal, conversion: Conversion | None) -> decimal.Decimal:
326
+ """The stored value in the unit it is compared in.
327
+
328
+ Converted first and rounded after: a file holding 0.0426 printed as a percentage is 4.26,
329
+ which is 4.3 at one decimal, where rounding the fraction to 0.04 first would print 4. A
330
+ non-finite value has no unit and is returned as it is, for `_agree` to refuse.
331
+ """
332
+ return conversion.apply(stored) if conversion is not None and stored.is_finite() else stored
333
+
334
+
335
+ def _held(stored: decimal.Decimal, compared: decimal.Decimal, conversion: Conversion | None) -> str:
336
+ """A stored value as a detail shows it: with what it is under a declared conversion."""
337
+ if conversion is None:
338
+ return f"{stored}"
339
+ return f"{stored} ({_plain(compared)} {conversion.said})"
340
+
341
+
342
+ def _places(count: int) -> str:
343
+ """A number of decimal places, in words a detail can use. A value printed to the tens
344
+ has fewer than none, which is said as it is."""
345
+ return f"{count} decimal place{'' if count == 1 else 's'}"
346
+
347
+
348
+ def _misplaced(
349
+ evidence: MetricEvidence | TableCellEvidence | ValueEvidence | AggregateEvidence,
350
+ ) -> str | None:
351
+ """What to say where `places` is declared and `reported` is written to another number of
352
+ them, or None. Two statements about the manuscript disagree, and no rounding of the
353
+ file's value settles which is meant: `4.20` is not a number reported to one place."""
354
+ printed_places = -_exponent(evidence.value)
355
+ if evidence.places is None or printed_places == evidence.places:
356
+ return None
357
+ return (
358
+ f"{evidence.name}: manuscript prints {evidence.reported}, which is "
359
+ f"{_places(printed_places)}, and {_places(evidence.places)} is declared"
360
+ )
361
+
362
+
363
+ def _plain(converted: decimal.Decimal) -> str:
364
+ """A converted value with no exponent: 0.5 as a percentage is written 50, where `str`
365
+ writes the same decimal as 5E+1."""
366
+ return format(converted, "f")
367
+
368
+
369
+ def _exponent(value: decimal.Decimal) -> int:
370
+ """The power of ten a finite decimal's last digit sits at.
371
+
372
+ `Decimal.as_tuple().exponent` is `int | Literal["n", "N", "F"]`: the strings tag NaN and
373
+ the infinities, which have no precision for a comparison to be coarser or finer than.
374
+ Every backend refuses a non-finite value with `value_not_numeric` before comparing, so
375
+ reaching this with one is a defect in a backend rather than a fact about an artifact, and
376
+ §5 says a defect is an `error` and never a scientific outcome. Raising is how it becomes
377
+ one. Returning a number instead would compare two values at a precision nothing chose.
378
+ """
379
+ exponent = value.as_tuple().exponent
380
+ if not isinstance(exponent, int):
381
+ raise ValueError(
382
+ f"{value} has no exponent: a non-finite value is refused before any comparison, "
383
+ f"so reaching the comparison with one is a defect"
384
+ )
385
+ return exponent
386
+
387
+
388
+ def _tie(
389
+ compared: decimal.Decimal | fractions.Fraction, reported: decimal.Decimal
390
+ ) -> tuple[decimal.Decimal, decimal.Decimal] | None:
391
+ """The two values at the precision `reported` prints that `compared` lies exactly halfway
392
+ between, lower first, or None where it is nearer one of them.
393
+
394
+ Decided in exact rationals, for a stored decimal and for a statistic that may be no
395
+ decimal at all: the value in units of the printed precision is a whole number and a half,
396
+ or it is not. No float is formed and nothing is within a tolerance of a tie, so 4.25 is
397
+ one at one place and 4.2500001 is not.
398
+ """
399
+ if isinstance(compared, decimal.Decimal) and not compared.is_finite():
400
+ return None
401
+ quantum = _exponent(reported)
402
+ units = fractions.Fraction(compared) / fractions.Fraction(10) ** quantum
403
+ if units.denominator != 2:
404
+ return None
405
+ below = math.floor(units)
406
+
407
+ def at_precision(count: int) -> decimal.Decimal:
408
+ # Zero carries no sign: the neighbor above -0.5 is 0.
409
+ return decimal.Decimal((int(count < 0), tuple(map(int, str(abs(count)))), quantum))
410
+
411
+ return at_precision(below), at_precision(below + 1)
412
+
413
+
414
+ def _tied(
415
+ compared: decimal.Decimal | fractions.Fraction,
416
+ printed: decimal.Decimal,
417
+ mode: ComparisonMode,
418
+ ) -> Tie | None:
419
+ """The record of a tie, where the comparison is at a printed precision and met one."""
420
+ if mode is not ComparisonMode.PRINTED_PRECISION:
421
+ return None
422
+ neighbors = _tie(compared, printed)
423
+ if neighbors is None:
424
+ return None
425
+ lower, upper = neighbors
426
+ return Tie(lower=_plain(lower), upper=_plain(upper), printed=_plain(printed))
427
+
428
+
429
+ def _agree(
430
+ stored: decimal.Decimal,
431
+ reported: decimal.Decimal,
432
+ mode: ComparisonMode,
433
+ tolerance: decimal.Decimal,
434
+ ) -> bool:
435
+ """Do two decimals agree under one mode? The arithmetic, with no evidence attached.
436
+
437
+ Split out because a correspondence has two extracted values and no `reported` field to
438
+ read a mode off, and duplicating the `printed_precision` rounding rule is how two
439
+ spellings of the same comparison come to disagree.
440
+ """
441
+ if not stored.is_finite():
442
+ # NaN and the infinities parse out of JSON but cannot equal a printed decimal. The
443
+ # backend flags them before reaching here; this keeps the function total.
444
+ return False
445
+ if mode is ComparisonMode.PRINTED_PRECISION:
446
+ # Round the stored value to the precision the manuscript printed. A paper reporting
447
+ # 3.2 is not contradicted by a file holding 3.20001; a paper reporting 3.20000 is.
448
+ # `quantize` raises whenever the *result* needs more digits than the context carries,
449
+ # which is not only the case where the two values differ wildly: two identical
450
+ # thirty-digit integers also exceed the default precision of 28, and were reported as
451
+ # a mismatch. That is a defect in the arithmetic emitted as a contradicted manuscript,
452
+ # which is the one thing this package must never do. Size the context to the operands.
453
+ #
454
+ # A value exactly halfway between two printed values is rounded up by half the
455
+ # software a manuscript's numbers come out of and to the even neighbor by the other
456
+ # half, so either neighbor agrees. Everywhere else the rounding is half-even, and
457
+ # which rule is used makes no difference away from a tie.
458
+ if (neighbors := _tie(stored, reported)) is not None:
459
+ return reported in neighbors
460
+ with decimal.localcontext() as context:
461
+ context.prec = max(
462
+ context.prec,
463
+ len(stored.as_tuple().digits) + abs(int(reported.as_tuple().exponent)) + 2,
464
+ )
465
+ try:
466
+ return stored.quantize(reported, rounding=decimal.ROUND_HALF_EVEN) == reported
467
+ except decimal.InvalidOperation:
468
+ # Genuinely beyond reach: the two differ by more orders of magnitude than any
469
+ # precision reconciles. They disagree, and saying so is the answer.
470
+ return False
471
+ delta = abs(stored - reported)
472
+ if mode is ComparisonMode.ABSOLUTE:
473
+ return delta <= tolerance
474
+ return delta <= tolerance * abs(reported)
475
+
476
+
477
+ def _agree_exact(
478
+ computed: fractions.Fraction,
479
+ reported: decimal.Decimal,
480
+ mode: ComparisonMode,
481
+ tolerance: decimal.Decimal,
482
+ ) -> bool:
483
+ """`_agree` for a value that may have no finite decimal expansion.
484
+
485
+ A mean of three values is a third of their sum, and dividing in decimals rounds it once
486
+ at whatever precision the context carries before `printed_precision` rounds it again. A
487
+ rational is exact, so the one rounding is the rule's own: half-even to the precision
488
+ `reported` prints, with either neighbor agreeing at an exact tie. On a value that is a
489
+ decimal the two functions agree.
490
+ """
491
+ printed = fractions.Fraction(reported)
492
+ if mode is ComparisonMode.PRINTED_PRECISION:
493
+ # The tie rule a single value is held to, by the function that holds it.
494
+ if (neighbors := _tie(computed, reported)) is not None:
495
+ return reported in neighbors
496
+ quantum = fractions.Fraction(10) ** _exponent(reported)
497
+ # `round` of a rational is exact and takes a tie to the even neighbor.
498
+ return round(computed / quantum) * quantum == printed
499
+ delta = abs(computed - printed)
500
+ if mode is ComparisonMode.ABSOLUTE:
501
+ return delta <= fractions.Fraction(tolerance)
502
+ return delta <= fractions.Fraction(tolerance) * abs(printed)
503
+
504
+
505
+ def _aggregate(statistic: Statistic, values: list[fractions.Fraction]) -> fractions.Fraction:
506
+ """One statistic of at least one value, exactly."""
507
+ if statistic is Statistic.MEAN:
508
+ return sum(values, fractions.Fraction(0)) / len(values)
509
+ if statistic is Statistic.MEDIAN:
510
+ # The middle value, or half the sum of the two middle ones, which a rational holds.
511
+ return statistics.median(values)
512
+ if statistic is Statistic.MIN:
513
+ return min(values)
514
+ if statistic is Statistic.MAX:
515
+ return max(values)
516
+ if statistic is Statistic.SUM:
517
+ return sum(values, fractions.Fraction(0))
518
+ return fractions.Fraction(len(values))
519
+
520
+
521
+ #: How many places past the printed precision a statistic with no finite expansion is shown to.
522
+ SHOWN_PLACES = 3
523
+
524
+
525
+ def _shown(computed: fractions.Fraction, reported: decimal.Decimal) -> str:
526
+ """A statistic as a report shows it: exactly where it is a terminating decimal, and
527
+ otherwise rounded half-even to `SHOWN_PLACES` past the precision the manuscript prints."""
528
+ places = 0
529
+ scaled = computed
530
+ while scaled.denominator % 2 == 0 or scaled.denominator % 5 == 0:
531
+ scaled *= 10
532
+ places += 1
533
+ if scaled.denominator != 1:
534
+ places = max(0, -_exponent(reported)) + SHOWN_PLACES
535
+ scaled = fractions.Fraction(round(computed * 10**places))
536
+ sign, digits, _ = decimal.Decimal(scaled.numerator).as_tuple()
537
+ return format(decimal.Decimal((sign, digits, -places)), "f")
538
+
539
+
540
+ class ValueBackend:
541
+ """Assertion: this artifact holds this value at this locator.
542
+
543
+ One implementation for every numeric kind. `metric` and `table` are shorthands that
544
+ expose a locator, so the addressing rules and the invariant -- exactly one scalar -- do
545
+ not depend on which spelling a manifest used.
546
+ """
547
+
548
+ kind = "value"
549
+ version = "2"
550
+ tool = ADAPTER_DISTRIBUTION
551
+
552
+ @property
553
+ def tool_version(self) -> str:
554
+ return adapter_version()
555
+
556
+ def check(
557
+ self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
558
+ ) -> Decision:
559
+ if not isinstance(evidence, (MetricEvidence, TableCellEvidence, ValueEvidence)):
560
+ raise TypeError(f"{type(self).__name__} received {type(evidence).__name__}")
561
+ path = paths[evidence.artifact]
562
+ base = _base(claim, evidence, self)
563
+ base["conversion"] = evidence.conversion
564
+ completed = {"execution": ExecutionStatus.COMPLETED}
565
+ no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
566
+
567
+ if isinstance(evidence, TableCellEvidence) and not evidence.addresses_one_row:
568
+ # Both or neither given. A manifest that does not say which row it means is
569
+ # malformed rather than a table that disagrees.
570
+ return Decision(
571
+ **base,
572
+ **completed,
573
+ **no_compare,
574
+ extraction=ExtractionStatus.INVALID,
575
+ reason=Reason.ROW_SELECTOR_INVALID,
576
+ detail="give exactly one of `row` or `where`",
577
+ )
578
+
579
+ locator = evidence.locator
580
+ base["locator_digest"] = locator.digest.value
581
+ warnings = (Warning_.POSITIONAL_ADDRESS,) if locator.kind == "table_position" else ()
582
+
583
+ resolution, extracted, detail = resolve(locator, path)
584
+
585
+ if resolution is Resolution.FORMAT_UNSUPPORTED:
586
+ # The check did not run. Reporting it as a missing value would say something
587
+ # about the artifact that was never looked into.
588
+ return Decision(
589
+ **base,
590
+ execution=ExecutionStatus.UNAVAILABLE,
591
+ extraction=ExtractionStatus.NOT_ATTEMPTED,
592
+ **no_compare,
593
+ reason=Reason.FORMAT_UNSUPPORTED,
594
+ detail=detail,
595
+ warnings=warnings,
596
+ )
597
+
598
+ if resolution is not Resolution.RESOLVED or extracted is None:
599
+ extraction, reason = _RESOLUTION_STATE[resolution]
600
+ return Decision(
601
+ **base,
602
+ **completed,
603
+ **no_compare,
604
+ extraction=extraction,
605
+ reason=reason or _ABSENT_REASON[locator.kind],
606
+ detail=detail,
607
+ warnings=warnings,
608
+ )
609
+
610
+ # The adapter produced a value, so there is an extraction to hash. Recorded before the
611
+ # comparison, and for a value that turns out not to be numeric: what the extractor
612
+ # read is a fact about the extractor whatever the comparison then makes of it.
613
+ base["extraction_digest"] = Digest.of_text(extracted.raw).value
614
+ base["extracted"] = extracted.raw
615
+
616
+ try:
617
+ stored = decimal.Decimal(extracted.raw)
618
+ except (decimal.InvalidOperation, ValueError):
619
+ return Decision(
620
+ **base,
621
+ **completed,
622
+ **no_compare,
623
+ extraction=ExtractionStatus.INVALID,
624
+ reason=Reason.VALUE_NOT_NUMERIC,
625
+ detail=f"{' '.join(extracted.trace)} holds {extracted.raw!r}",
626
+ warnings=warnings,
627
+ )
628
+ if not stored.is_finite():
629
+ # `json.loads` accepts NaN, Infinity and -Infinity by default, so a result file
630
+ # can carry one. It is not a quantity a printed value can agree with.
631
+ return Decision(
632
+ **base,
633
+ **completed,
634
+ **no_compare,
635
+ extraction=ExtractionStatus.INVALID,
636
+ reason=Reason.VALUE_NOT_NUMERIC,
637
+ detail=f"{' '.join(extracted.trace)} holds {stored}",
638
+ warnings=warnings,
639
+ )
640
+
641
+ compared = _compared(stored, evidence.conversion)
642
+ if evidence.conversion is not None:
643
+ base["converted"] = _plain(compared)
644
+ held = _held(stored, compared, evidence.conversion)
645
+ where = " ".join(extracted.trace) or locator.kind
646
+
647
+ if (misplaced := _misplaced(evidence)) is not None:
648
+ return Decision(
649
+ **base,
650
+ **completed,
651
+ extraction=ExtractionStatus.EXTRACTED,
652
+ comparison=ComparisonStatus.MISMATCH,
653
+ reason=Reason.PLACES_MISMATCH,
654
+ detail=misplaced,
655
+ warnings=warnings,
656
+ )
657
+
658
+ agrees = compare_decimal(stored, evidence)
659
+ tie = _tied(compared, evidence.value, evidence.mode)
660
+ base["tie"] = tie
661
+ between = f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""
662
+ return Decision(
663
+ **base,
664
+ **completed,
665
+ extraction=ExtractionStatus.EXTRACTED,
666
+ comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
667
+ reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
668
+ detail=(
669
+ f"{where} = {held}{between}"
670
+ + (f"; the manuscript prints {evidence.reported}" if tie else "")
671
+ if agrees
672
+ else f"{evidence.name}: manuscript prints {evidence.reported}, "
673
+ f"{path.name} holds {held} at {where}{between}"
674
+ ),
675
+ warnings=warnings,
676
+ )
677
+
678
+
679
+ class MetricBackend(ValueBackend):
680
+ """`kind: metric` -- a JSON Pointer into a structured file."""
681
+
682
+ kind = "metric"
683
+
684
+
685
+ class TableBackend(ValueBackend):
686
+ """`kind: table` -- a column plus a row selector in a delimited file."""
687
+
688
+ kind = "table"
689
+
690
+
691
+ # --------------------------------------------------------------------------- correspondences
692
+
693
+
694
+ class CorrespondenceBackend:
695
+ """Assertion: two artifacts hold the same value.
696
+
697
+ Two extractions precede one comparison, and the stages report that faithfully. A side that
698
+ does not extract makes the comparison impossible rather than false: a document that never
699
+ states the number does not contradict the file holding it, and reporting that as a
700
+ disagreement would manufacture a finding out of a gap. The decision carries the failing
701
+ side's own reason, so `pointer_absent` and `number_as_word` stay distinguishable.
702
+
703
+ When both sides extract and disagree, the decision reports both values and names neither
704
+ as wrong. Which of a specification and a test suite is in error is not something a byte
705
+ comparison establishes.
706
+ """
707
+
708
+ kind = "correspondence"
709
+ version = "1"
710
+ #: The format adapters are this distribution, as they are for `ValueBackend`. The
711
+ #: declaration is an approximation for one case: a `prose` side over a paginated source
712
+ #: reaches `citations.verify.extract`, which runs the same `pdftotext` that `QuoteBackend`
713
+ #: declares, and `Decision.tool` is one field where such an assertion has two extractors.
714
+ #: `DecisionSide.extraction_digest` is what catches a change in either of them, since it
715
+ #: moves whenever what a side read moves, whatever did the reading. Naming the tool per
716
+ #: side needs the adapters to report which one they used, which this revision does not
717
+ #: define.
718
+ tool = ADAPTER_DISTRIBUTION
719
+
720
+ @property
721
+ def tool_version(self) -> str:
722
+ return adapter_version()
723
+
724
+ def check(
725
+ self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
726
+ ) -> Decision:
727
+ if not isinstance(evidence, CorrespondenceEvidence):
728
+ raise TypeError(f"CorrespondenceBackend received {type(evidence).__name__}")
729
+ base = _base(claim, evidence, self)
730
+ no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
731
+
732
+ readings = [(side, *resolve(side.locator, paths[side.artifact])) for side in evidence.sides]
733
+ base["sides"] = tuple(
734
+ DecisionSide(
735
+ name=side.name,
736
+ artifact_id=side.artifact,
737
+ locator_digest=side.locator.digest.value,
738
+ extracted=extracted.raw if extracted is not None else None,
739
+ conversion=side.conversion,
740
+ # Hashed per side before any comparison, and for a value that turns out not
741
+ # to be numeric: what an extractor read is a fact about the extractor whatever
742
+ # the comparison then makes of it.
743
+ extraction_digest=(
744
+ Digest.of_text(extracted.raw).value if extracted is not None else UNKNOWN
745
+ ),
746
+ )
747
+ for side, _, extracted, _ in readings
748
+ )
749
+ # The decision-level field covers both extractions, in the order the manifest declares
750
+ # the sides, so it moves whenever either side's extraction moves. Which of the two
751
+ # moved is on the sides; leaving this `unknown` would say the extraction was sought
752
+ # and not obtained, and it was sought twice.
753
+ base["extraction_digest"] = Digest.of_text(
754
+ "\x00".join(s.extraction_digest for s in base["sides"])
755
+ ).value
756
+
757
+ unsupported = [(s, d) for s, r, _, d in readings if r is Resolution.FORMAT_UNSUPPORTED]
758
+ if unsupported:
759
+ side, detail = unsupported[0]
760
+ return Decision(
761
+ **base,
762
+ execution=ExecutionStatus.UNAVAILABLE,
763
+ extraction=ExtractionStatus.NOT_ATTEMPTED,
764
+ **no_compare,
765
+ reason=Reason.FORMAT_UNSUPPORTED,
766
+ detail=f"{side.name}: {detail}",
767
+ )
768
+
769
+ completed = {"execution": ExecutionStatus.COMPLETED}
770
+ unresolved = [
771
+ (s, r, d) for s, r, e, d in readings if r is not Resolution.RESOLVED or e is None
772
+ ]
773
+ if unresolved:
774
+ # Reported in the order the manifest declares the sides, which is a fact about the
775
+ # manifest and not a ranking. Every failing side appears in the detail.
776
+ side, resolution, _ = unresolved[0]
777
+ extraction, reason = _RESOLUTION_STATE[resolution]
778
+ return Decision(
779
+ **base,
780
+ **completed,
781
+ **no_compare,
782
+ extraction=extraction,
783
+ reason=reason or _ABSENT_REASON[side.locator.kind],
784
+ detail="; ".join(f"{s.name} ({s.artifact}): {d}" for s, _, d in unresolved),
785
+ )
786
+
787
+ numbers: list[decimal.Decimal] = []
788
+ for side, _, extracted, _ in readings:
789
+ raw = extracted.raw if extracted is not None else ""
790
+ try:
791
+ number = decimal.Decimal(raw)
792
+ except (decimal.InvalidOperation, ValueError):
793
+ number = decimal.Decimal("NaN")
794
+ if not number.is_finite():
795
+ return Decision(
796
+ **base,
797
+ **completed,
798
+ **no_compare,
799
+ extraction=ExtractionStatus.INVALID,
800
+ reason=Reason.VALUE_NOT_NUMERIC,
801
+ detail=f"{side.name} ({side.artifact}) holds {raw!r}",
802
+ )
803
+ numbers.append(number)
804
+
805
+ # Each side in the unit it declares, and the comparison is over those: a document
806
+ # printing 4.2 beside a file holding 0.0424 as a percentage compares 4.2 with 4.24.
807
+ stored = numbers
808
+ numbers = [
809
+ _compared(n, side.conversion) for n, side in zip(stored, evidence.sides, strict=True)
810
+ ]
811
+ base["sides"] = tuple(
812
+ recorded.model_copy(update={"converted": _plain(number)})
813
+ if recorded.conversion is not None
814
+ else recorded
815
+ for recorded, number in zip(base["sides"], numbers, strict=True)
816
+ )
817
+
818
+ left, right = evidence.sides
819
+ a, b = numbers
820
+ held = [
821
+ _held(was, number, side.conversion)
822
+ for was, number, side in zip(stored, numbers, evidence.sides, strict=True)
823
+ ]
824
+ # `printed_precision` compares at the coarser of the two precisions, so a document
825
+ # printing 0.65 agrees with a file holding 0.6478 and the answer does not depend on
826
+ # which side the manifest wrote first. A larger exponent is the coarser value.
827
+ coarse, fine = (a, b) if _exponent(a) >= _exponent(b) else (b, a)
828
+ agrees = _agree(fine, coarse, evidence.mode, evidence.tolerance_value)
829
+ tie = _tied(fine, coarse, evidence.mode)
830
+ base["tie"] = tie
831
+ return Decision(
832
+ **base,
833
+ **completed,
834
+ extraction=ExtractionStatus.EXTRACTED,
835
+ comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
836
+ reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
837
+ detail=f"{evidence.name}: {left.name} {held[0]}, {right.name} {held[1]}"
838
+ + (f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""),
839
+ warnings=(
840
+ (Warning_.POSITIONAL_ADDRESS,)
841
+ if any(side.locator.kind == "table_position" for side in evidence.sides)
842
+ else ()
843
+ ),
844
+ )
845
+
846
+
847
+ # -------------------------------------------------------------------------------- aggregates
848
+
849
+
850
+ class AggregateBackend:
851
+ """Assertion: the printed value is a statistic of the values its members hold.
852
+
853
+ As many extractions as there are members precede one comparison. A member that does not
854
+ extract makes the comparison impossible rather than false, as one side of a correspondence
855
+ does: a statistic over the members that did resolve is a statistic over a set the manifest
856
+ did not declare. The decision carries the first failing member's own reason and its detail
857
+ names every member that failed, by its position in the list and its artifact.
858
+
859
+ The statistic is computed over rationals, so a mean is the exact quotient and is rounded
860
+ once, by the comparison.
861
+ """
862
+
863
+ kind = "aggregate"
864
+ version = "1"
865
+ #: As for `CorrespondenceBackend`: one field where the members may be read by more than
866
+ #: one program, and `DecisionMember.extraction_digest` is what moves when any of them does.
867
+ tool = ADAPTER_DISTRIBUTION
868
+
869
+ @property
870
+ def tool_version(self) -> str:
871
+ return adapter_version()
872
+
873
+ def check(
874
+ self, claim: Claim, evidence: Evidence, paths: Mapping[str, pathlib.Path]
875
+ ) -> Decision:
876
+ if not isinstance(evidence, AggregateEvidence):
877
+ raise TypeError(f"AggregateBackend received {type(evidence).__name__}")
878
+ base = _base(claim, evidence, self)
879
+ base["statistic"] = evidence.statistic
880
+ base["conversion"] = evidence.conversion
881
+ no_compare = {"comparison": ComparisonStatus.NOT_APPLICABLE}
882
+
883
+ def named(position: int, artifact: str) -> str:
884
+ return f"member {position} of {evidence.count} ({artifact})"
885
+
886
+ readings = []
887
+ for position, member in enumerate(evidence.members, start=1):
888
+ try:
889
+ found = resolve(member.locator, paths[member.artifact])
890
+ except ArtifactUnreadableError as e:
891
+ # Still a file that could not be parsed, and the engine reports it as one.
892
+ # Raised again so that it says which of the members the file is.
893
+ raise ArtifactUnreadableError(
894
+ e.path, f"{named(position, member.artifact)}: {e.detail}"
895
+ ) from e
896
+ readings.append((position, member, *found))
897
+ base["members"] = tuple(
898
+ DecisionMember(
899
+ artifact_id=member.artifact,
900
+ locator_digest=member.locator.digest.value,
901
+ extracted=extracted.raw if extracted is not None else None,
902
+ extraction_digest=(
903
+ Digest.of_text(extracted.raw).value if extracted is not None else UNKNOWN
904
+ ),
905
+ )
906
+ for _, member, _, extracted, _ in readings
907
+ )
908
+ # Over every member's extraction, in the order the manifest lists them, so it moves
909
+ # whenever any member's extraction moves. Which one moved is on the member.
910
+ base["extraction_digest"] = Digest.of_text(
911
+ "\x00".join(m.extraction_digest for m in base["members"])
912
+ ).value
913
+ warnings = (
914
+ (Warning_.POSITIONAL_ADDRESS,)
915
+ if any(member.locator.kind == "table_position" for member in evidence.members)
916
+ else ()
917
+ )
918
+
919
+ unsupported = [
920
+ (p, m, d) for p, m, r, _, d in readings if r is Resolution.FORMAT_UNSUPPORTED
921
+ ]
922
+ if unsupported:
923
+ return Decision(
924
+ **base,
925
+ execution=ExecutionStatus.UNAVAILABLE,
926
+ extraction=ExtractionStatus.NOT_ATTEMPTED,
927
+ **no_compare,
928
+ reason=Reason.FORMAT_UNSUPPORTED,
929
+ detail="; ".join(f"{named(p, m.artifact)}: {d}" for p, m, d in unsupported),
930
+ warnings=warnings,
931
+ )
932
+
933
+ completed = {"execution": ExecutionStatus.COMPLETED}
934
+ unresolved = [
935
+ (p, m, r, d) for p, m, r, e, d in readings if r is not Resolution.RESOLVED or e is None
936
+ ]
937
+ if unresolved:
938
+ _, member, resolution, _ = unresolved[0]
939
+ extraction, reason = _RESOLUTION_STATE[resolution]
940
+ return Decision(
941
+ **base,
942
+ **completed,
943
+ **no_compare,
944
+ extraction=extraction,
945
+ reason=reason or _ABSENT_REASON[member.locator.kind],
946
+ detail="; ".join(f"{named(p, m.artifact)}: {d}" for p, m, _, d in unresolved),
947
+ warnings=warnings,
948
+ )
949
+
950
+ # Each member in the unit the assertion declares, and the statistic is over those.
951
+ converted: list[decimal.Decimal] = []
952
+ not_numeric = []
953
+ for position, member, _, extracted, _ in readings:
954
+ raw = extracted.raw if extracted is not None else ""
955
+ try:
956
+ number = decimal.Decimal(raw)
957
+ except (decimal.InvalidOperation, ValueError):
958
+ number = decimal.Decimal("NaN")
959
+ if number.is_finite():
960
+ converted.append(_compared(number, evidence.conversion))
961
+ else:
962
+ not_numeric.append(f"{named(position, member.artifact)} holds {raw!r}")
963
+ if not_numeric:
964
+ return Decision(
965
+ **base,
966
+ **completed,
967
+ **no_compare,
968
+ extraction=ExtractionStatus.INVALID,
969
+ reason=Reason.VALUE_NOT_NUMERIC,
970
+ detail="; ".join(not_numeric),
971
+ warnings=warnings,
972
+ )
973
+ values = [fractions.Fraction(number) for number in converted]
974
+
975
+ if evidence.conversion is not None:
976
+ base["members"] = tuple(
977
+ recorded.model_copy(update={"converted": _plain(number)})
978
+ for recorded, number in zip(base["members"], converted, strict=True)
979
+ )
980
+
981
+ if (misplaced := _misplaced(evidence)) is not None:
982
+ return Decision(
983
+ **base,
984
+ **completed,
985
+ extraction=ExtractionStatus.EXTRACTED,
986
+ comparison=ComparisonStatus.MISMATCH,
987
+ reason=Reason.PLACES_MISMATCH,
988
+ detail=misplaced,
989
+ warnings=warnings,
990
+ )
991
+
992
+ computed = _aggregate(evidence.statistic, values)
993
+ agrees = _agree_exact(computed, evidence.value, evidence.mode, evidence.tolerance_value)
994
+ shown = _shown(computed, evidence.value)
995
+ base["computed"] = shown
996
+ tie = _tied(computed, evidence.value, evidence.mode)
997
+ base["tie"] = tie
998
+ between = f", an exact tie between {tie.lower} and {tie.upper}" if tie else ""
999
+ unit = f" {evidence.conversion.said}" if evidence.conversion is not None else ""
1000
+ over = f"{evidence.statistic.value} of {evidence.count}{unit}"
1001
+ return Decision(
1002
+ **base,
1003
+ **completed,
1004
+ extraction=ExtractionStatus.EXTRACTED,
1005
+ comparison=ComparisonStatus.MATCH if agrees else ComparisonStatus.MISMATCH,
1006
+ reason=Reason.VALUE_MATCH if agrees else Reason.VALUE_MISMATCH,
1007
+ detail=(
1008
+ f"{evidence.name}: {over} = {shown}{between}"
1009
+ + (f"; the manuscript prints {evidence.reported}" if tie else "")
1010
+ if agrees
1011
+ else f"{evidence.name}: manuscript prints {evidence.reported}, "
1012
+ f"{over} is {shown}{between}"
1013
+ ),
1014
+ warnings=warnings,
1015
+ )
1016
+
1017
+
1018
+ DEFAULT_BACKENDS: tuple[Backend, ...] = (
1019
+ QuoteBackend(),
1020
+ MetricBackend(),
1021
+ TableBackend(),
1022
+ ValueBackend(),
1023
+ CorrespondenceBackend(),
1024
+ AggregateBackend(),
1025
+ )
1026
+
1027
+
1028
+ # ------------------------------------------------------------------------------------ engine
1029
+
1030
+
1031
+ def _artifact_state(artifact: ArtifactRef, path: pathlib.Path) -> ArtifactState:
1032
+ if not path.exists():
1033
+ # Not authoritative, whether or not it was pinned: there are no bytes to be the
1034
+ # declared ones. Calling it authoritative made every decision against it claim to
1035
+ # describe a file that was never read.
1036
+ return ArtifactState(
1037
+ artifact_id=artifact.id,
1038
+ validity=Validity.ARTIFACT_ABSENT,
1039
+ exists=False,
1040
+ expected=artifact.digest.value if artifact.digest else None,
1041
+ )
1042
+ # A path that exists but cannot be hashed -- a directory, a mode-000 file, a dangling
1043
+ # symlink -- is a fact about that artifact. Raising suppressed the report for every other
1044
+ # claim in the manifest because of one bad path.
1045
+ try:
1046
+ actual = Digest.of_file(path).value
1047
+ except OSError as e:
1048
+ return ArtifactState(
1049
+ artifact_id=artifact.id,
1050
+ validity=Validity.ARTIFACT_ABSENT,
1051
+ expected=artifact.digest.value if artifact.digest else None,
1052
+ exists=False,
1053
+ detail=f"cannot be read: {e.strerror or e}",
1054
+ )
1055
+ if not artifact.is_pinned or artifact.digest is None:
1056
+ return ArtifactState(
1057
+ artifact_id=artifact.id,
1058
+ validity=Validity.UNPINNED_ARTIFACT,
1059
+ actual=actual,
1060
+ )
1061
+ if actual != artifact.digest.value:
1062
+ return ArtifactState(
1063
+ artifact_id=artifact.id,
1064
+ validity=Validity.BROKEN_PIN,
1065
+ expected=artifact.digest.value,
1066
+ actual=actual,
1067
+ )
1068
+ return ArtifactState(
1069
+ artifact_id=artifact.id,
1070
+ validity=Validity.AUTHORITATIVE,
1071
+ expected=artifact.digest.value,
1072
+ actual=actual,
1073
+ )
1074
+
1075
+
1076
+ def _ordering(
1077
+ claim: Claim, manifest: Manifest, states: dict[str, ArtifactState]
1078
+ ) -> tuple[Ordering, OrderingReason, str, RegistrationAuthority | None]:
1079
+ """Did the runs producing this claim's evidence start after its plan was registered?
1080
+
1081
+ Returns an outcome, the reason it was reached, a human-readable detail, and the weakest
1082
+ registration authority among the runs that applied. Every condition that stops the check
1083
+ yields `unchecked` with its own reason rather than a verdict: an absent record is not
1084
+ evidence that a result predates its plan, and reporting it as one would manufacture a
1085
+ finding out of a gap.
1086
+ """
1087
+ if not claim.confirmatory:
1088
+ return Ordering.NOT_APPLICABLE, OrderingReason.NOT_CONFIRMATORY, "", None
1089
+
1090
+ produced = {a for e in claim.evidence for a in e.artifacts}
1091
+ runs = [r for r in manifest.runs if produced & {o.artifact for o in r.outputs}]
1092
+ covered = {o.artifact for r in runs for o in r.outputs}
1093
+ uncovered = sorted(produced - covered)
1094
+ if uncovered:
1095
+ # A claim can cite several artifacts. Taking the runs that produce *any* of them meant
1096
+ # adding a run record for one incidental artifact turned an unregistered result into
1097
+ # an ordered one, while the artifact carrying the number had no run at all.
1098
+ return (
1099
+ Ordering.UNCHECKED,
1100
+ OrderingReason.NO_RUN_RECORD,
1101
+ f"no run record produces {', '.join(uncovered)}",
1102
+ None,
1103
+ )
1104
+
1105
+ if not runs:
1106
+ # A confirmatory claim offering no evidence names no artifacts, so no run produces any
1107
+ # and `uncovered` above is empty too. `min()` then raised on an empty iterable and the
1108
+ # traceback replaced the finding: the publication policy grades this manifest
1109
+ # `claim.no_evidence`, which is the thing the author needs to be told.
1110
+ return (
1111
+ Ordering.UNCHECKED,
1112
+ OrderingReason.NO_EVIDENCE_OFFERED,
1113
+ "the claim offers no evidence, so there is nothing to order against a plan",
1114
+ None,
1115
+ )
1116
+
1117
+ authority = min((r.registration_authority for r in runs), key=lambda a: a.rank)
1118
+
1119
+ def unchecked(reason: OrderingReason, detail: str):
1120
+ return Ordering.UNCHECKED, reason, detail, authority
1121
+
1122
+ # More than one run claiming the same artifact with different registrations does not say
1123
+ # which one produced the bytes being checked.
1124
+ for artifact_id in sorted(produced):
1125
+ claimants = [r for r in runs if r.output(artifact_id) is not None]
1126
+ if len({(r.registered_plan, r.registered_at) for r in claimants}) > 1:
1127
+ return unchecked(
1128
+ OrderingReason.AMBIGUOUS_PRODUCING_RUN,
1129
+ f"{len(claimants)} runs claim to produce {artifact_id}",
1130
+ )
1131
+
1132
+ without_plan = sorted({r.id for r in runs if not r.registered_plan})
1133
+ if without_plan:
1134
+ return unchecked(
1135
+ OrderingReason.NO_REGISTERED_PLAN,
1136
+ f"run {', '.join(without_plan)} names no registered plan",
1137
+ )
1138
+
1139
+ undated = sorted({r.id for r in runs if r.registered_at is None or r.started_at is None})
1140
+ if undated:
1141
+ return unchecked(
1142
+ OrderingReason.TIMESTAMP_MISSING,
1143
+ f"run {', '.join(undated)} records no registration or start time",
1144
+ )
1145
+
1146
+ # The plan document, where it is a declared artifact, must be the one that was pinned.
1147
+ for run in runs:
1148
+ state = states.get(run.registered_plan)
1149
+ if state is None:
1150
+ # Undeclared, so nothing pins it: the document backing an `ordered` verdict could
1151
+ # have been written after the results. Declaring the plan and leaving it unpinned
1152
+ # reported `unchecked`, so the more transparent manifest scored strictly worse.
1153
+ return unchecked(
1154
+ OrderingReason.REGISTERED_PLAN_UNPINNED,
1155
+ f"plan {run.registered_plan} is not a declared artifact, so nothing pins the "
1156
+ f"document the ordering rests on",
1157
+ )
1158
+ if state.validity is Validity.BROKEN_PIN:
1159
+ return unchecked(
1160
+ OrderingReason.REGISTERED_PLAN_CHANGED,
1161
+ f"plan {run.registered_plan} is not the document that was "
1162
+ f"pinned, so its timestamp attests to nothing",
1163
+ )
1164
+ if state.validity is not Validity.AUTHORITATIVE:
1165
+ return unchecked(
1166
+ OrderingReason.REGISTERED_PLAN_UNPINNED,
1167
+ f"plan {run.registered_plan} is declared but {state.validity.value}",
1168
+ )
1169
+
1170
+ # Each output must be bound to the bytes actually read, not merely to a path.
1171
+ for run in runs:
1172
+ for artifact_id in sorted(produced & {o.artifact for o in run.outputs}):
1173
+ output = run.output(artifact_id)
1174
+ if output is None or output.digest is None:
1175
+ return unchecked(
1176
+ OrderingReason.RUN_OUTPUT_UNLINKED,
1177
+ f"run {run.id} names {artifact_id} but records no digest "
1178
+ f"for it, so any later file at that path would qualify",
1179
+ )
1180
+ actual = states[artifact_id].actual if artifact_id in states else None
1181
+ if actual is not None and actual != output.digest.value:
1182
+ return unchecked(
1183
+ OrderingReason.RUN_OUTPUT_CHANGED,
1184
+ f"{artifact_id} holds {actual[:12]}, but run {run.id} "
1185
+ f"produced {output.digest.value[:12]}",
1186
+ )
1187
+
1188
+ # Every run is dated by this point -- the guard above returned otherwise -- but binding
1189
+ # the timestamps is what makes that legible, to a reader and to a type checker.
1190
+ dated = [
1191
+ (r, r.registered_at, r.started_at)
1192
+ for r in runs
1193
+ if r.registered_at is not None and r.started_at is not None
1194
+ ]
1195
+ inverted = [
1196
+ (r, registered, started) for r, registered, started in dated if started < registered
1197
+ ]
1198
+ if inverted:
1199
+ run, registered_at, started_at = inverted[0]
1200
+ return (
1201
+ Ordering.VIOLATED,
1202
+ OrderingReason.RUN_PRECEDES_REGISTRATION,
1203
+ f"run {run.id} started {started_at.isoformat()}, before "
1204
+ f"{run.registered_plan} was registered {registered_at.isoformat()}",
1205
+ authority,
1206
+ )
1207
+
1208
+ pinned = sum(1 for r in runs if r.registered_plan in states)
1209
+ detail = (
1210
+ f"{len(runs)} run(s) started after registration"
1211
+ + (f", {pinned} against a pinned plan" if pinned else "")
1212
+ + f"; authority {authority.value}"
1213
+ )
1214
+ return Ordering.ORDERED, OrderingReason.RUN_FOLLOWS_REGISTRATION, detail, authority
1215
+
1216
+
1217
+ def artifact_states(
1218
+ manifest: Manifest,
1219
+ ) -> tuple[dict[str, ArtifactState], dict[str, pathlib.Path]]:
1220
+ """Each declared artifact's pin, resolved once, and the path it was read from."""
1221
+ states: dict[str, ArtifactState] = {}
1222
+ paths: dict[str, pathlib.Path] = {}
1223
+ for artifact in manifest.artifacts:
1224
+ path = manifest.resolve(artifact)
1225
+ paths[artifact.id] = path
1226
+ states[artifact.id] = _artifact_state(artifact, path)
1227
+ return states, paths
1228
+
1229
+
1230
+ def verify(
1231
+ manifest: Manifest, backends: tuple[Backend, ...] = DEFAULT_BACKENDS
1232
+ ) -> VerificationReport:
1233
+ """Check every evidence assertion in a manifest and report what was found.
1234
+
1235
+ Returns facts. No verdict: see `claims.policy` for whether a given set of facts is
1236
+ acceptable, which depends on what the project is for.
1237
+
1238
+ Executes nothing. A manifest may declare commands that regenerate its artifacts; running
1239
+ them is `repro.reproduce`, which has to be asked for by name.
1240
+ """
1241
+ registry: dict[str, Backend] = {b.kind: b for b in backends}
1242
+ states, paths = artifact_states(manifest)
1243
+
1244
+ assessments = []
1245
+ # Each artifact is parsed once for the whole manifest, just after it was hashed.
1246
+ with reading_once():
1247
+ for claim in manifest.claims:
1248
+ decisions = tuple(
1249
+ check_evidence(claim, evidence, manifest, paths, states, registry)
1250
+ for evidence in claim.evidence
1251
+ )
1252
+ ordering, ordering_reason, ordering_detail, authority = _ordering(
1253
+ claim, manifest, states
1254
+ )
1255
+ assessments.append(
1256
+ ClaimAssessment(
1257
+ claim_id=claim.id,
1258
+ claim_digest=claim.digest.value,
1259
+ confirmatory=claim.confirmatory,
1260
+ registration=claim.registration,
1261
+ registration_note=claim.registration_note,
1262
+ availability=claim.availability,
1263
+ ordering=ordering,
1264
+ ordering_reason=ordering_reason,
1265
+ ordering_detail=ordering_detail,
1266
+ registration_authority=authority,
1267
+ decisions=decisions,
1268
+ )
1269
+ )
1270
+
1271
+ return VerificationReport(
1272
+ project=manifest.project,
1273
+ manifest_digest=Digest.of_text(manifest.model_dump_json()).value,
1274
+ artifacts=tuple(states.values()),
1275
+ claims=tuple(assessments),
1276
+ )
1277
+
1278
+
1279
+ def check_evidence(
1280
+ claim: Claim,
1281
+ evidence: Evidence,
1282
+ manifest: Manifest,
1283
+ paths: dict[str, pathlib.Path],
1284
+ states: dict[str, ArtifactState],
1285
+ registry: dict[str, Backend],
1286
+ ) -> Decision:
1287
+ """Evaluate one assertion against the files in `paths`.
1288
+
1289
+ `paths` is a parameter and not read off the manifest so that a re-run can ask the same
1290
+ question of the file it has just written: `repro.reproduce` passes the path of the fresh
1291
+ output in place of the pinned one, and every rule below applies unchanged.
1292
+ """
1293
+ backend = registry.get(evidence.kind)
1294
+ if backend is None:
1295
+ raise UnknownEvidenceKindError(evidence.kind, tuple(registry))
1296
+
1297
+ base = _base(claim, evidence, backend)
1298
+ unchecked = {
1299
+ "execution": ExecutionStatus.UNAVAILABLE,
1300
+ "extraction": ExtractionStatus.NOT_ATTEMPTED,
1301
+ "comparison": ComparisonStatus.NOT_APPLICABLE,
1302
+ }
1303
+
1304
+ # Every artifact the assertion names, not only the first. An assertion reading two files
1305
+ # is unchecked if either is undeclared or absent, and no more authoritative than the worse
1306
+ # of the two pins.
1307
+ named = evidence.artifacts
1308
+ undeclared = [a for a in named if manifest.artifact(a) is None]
1309
+ if undeclared:
1310
+ return Decision(
1311
+ **base,
1312
+ **unchecked,
1313
+ reason=Reason.ARTIFACT_UNDECLARED,
1314
+ detail=f"manifest declares no artifact {', '.join(repr(a) for a in undeclared)}",
1315
+ )
1316
+ missing = [a for a in named if not states[a].exists]
1317
+ if missing:
1318
+ return Decision(
1319
+ **base,
1320
+ **unchecked,
1321
+ reason=Reason.ARTIFACT_MISSING,
1322
+ detail="; ".join(f"{paths[a]} does not exist" for a in missing),
1323
+ validity=min((states[a].validity for a in missing), key=_VALIDITY_RANK.__getitem__),
1324
+ )
1325
+
1326
+ try:
1327
+ decision = backend.check(claim, evidence, {a: paths[a] for a in named})
1328
+ except BackendUnavailableError as e:
1329
+ decision = Decision(**base, **unchecked, reason=Reason.EXTRACTOR_MISSING, detail=e.detail)
1330
+ except ArtifactUnreadableError as e:
1331
+ decision = Decision(**base, **unchecked, reason=Reason.ARTIFACT_UNREADABLE, detail=e.detail)
1332
+ except Exception as e:
1333
+ # A defect in a backend is a defect, not an abstention. Recording it as `unchecked`
1334
+ # would make a TypeError indistinguishable from a missing extractor, which is the
1335
+ # confusion this package exists to prevent.
1336
+ decision = Decision(
1337
+ **base,
1338
+ execution=ExecutionStatus.FAILED,
1339
+ extraction=ExtractionStatus.NOT_ATTEMPTED,
1340
+ comparison=ComparisonStatus.NOT_APPLICABLE,
1341
+ reason=Reason.BACKEND_DEFECT,
1342
+ detail=f"{type(e).__name__}: {e}",
1343
+ )
1344
+
1345
+ validity = min((states[a].validity for a in named), key=_VALIDITY_RANK.__getitem__)
1346
+ return decision.model_copy(
1347
+ update={
1348
+ "validity": validity,
1349
+ "artifact_digest": states[named[0]].actual if len(named) == 1 else None,
1350
+ # The backend knows which locator addressed which side; only the engine knows what
1351
+ # was at each path. Filling the digests here keeps both in one record.
1352
+ "sides": tuple(
1353
+ side.model_copy(
1354
+ update={
1355
+ "artifact_digest": states[side.artifact_id].actual,
1356
+ "validity": states[side.artifact_id].validity,
1357
+ }
1358
+ )
1359
+ for side in decision.sides
1360
+ ),
1361
+ "members": tuple(
1362
+ member.model_copy(
1363
+ update={
1364
+ "artifact_digest": states[member.artifact_id].actual,
1365
+ "validity": states[member.artifact_id].validity,
1366
+ }
1367
+ )
1368
+ for member in decision.members
1369
+ ),
1370
+ }
1371
+ )