claims-cli 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. claims/__init__.py +132 -0
  2. claims/__main__.py +5 -0
  3. claims/adapters/__init__.py +6 -0
  4. claims/adapters/array.py +212 -0
  5. claims/adapters/base.py +75 -0
  6. claims/adapters/cells.py +94 -0
  7. claims/adapters/columnar.py +188 -0
  8. claims/adapters/prose.py +261 -0
  9. claims/adapters/reads.py +47 -0
  10. claims/adapters/rows.py +116 -0
  11. claims/adapters/sheet.py +149 -0
  12. claims/adapters/sqlite.py +73 -0
  13. claims/adapters/table.py +135 -0
  14. claims/adapters/tree.py +163 -0
  15. claims/checks.py +511 -0
  16. claims/cli.py +1103 -0
  17. claims/core/__init__.py +9 -0
  18. claims/core/artifacts.py +121 -0
  19. claims/core/checks.py +132 -0
  20. claims/core/claims.py +100 -0
  21. claims/core/evidence.py +708 -0
  22. claims/core/manifest.py +349 -0
  23. claims/core/numbers.py +146 -0
  24. claims/core/outcomes.py +410 -0
  25. claims/core/report.py +503 -0
  26. claims/corpus.py +344 -0
  27. claims/exceptions.py +106 -0
  28. claims/export/__init__.py +23 -0
  29. claims/export/rocrate.py +787 -0
  30. claims/export/sarif.py +401 -0
  31. claims/manifest.py +59 -0
  32. claims/models.py +222 -0
  33. claims/numbers.py +453 -0
  34. claims/origins.py +303 -0
  35. claims/policy.py +369 -0
  36. claims/provenance.py +42 -0
  37. claims/record.py +1099 -0
  38. claims/regression.py +171 -0
  39. claims/renderers/__init__.py +10 -0
  40. claims/renderers/sarif.py +294 -0
  41. claims/reruns.py +111 -0
  42. claims/resolve.py +116 -0
  43. claims/stale.py +333 -0
  44. claims/starter.py +432 -0
  45. claims/toolchain.py +120 -0
  46. claims/verify.py +1371 -0
  47. claims/why.py +398 -0
  48. claims_cli-0.6.0.dist-info/METADATA +139 -0
  49. claims_cli-0.6.0.dist-info/RECORD +51 -0
  50. claims_cli-0.6.0.dist-info/WHEEL +4 -0
  51. claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/record.py ADDED
@@ -0,0 +1,1099 @@
1
+ """One checked assertion, as a record another program can read.
2
+
3
+ `repro verify` prints a report and `repro why` an account, and each is shaped for its reader.
4
+ This is the same evaluation stated once per assertion and for no reader in particular: the
5
+ checker and its version read this file, with this digest, at this address, obtained this
6
+ value, compared it with the value the manuscript reports under this rule, and returned this
7
+ outcome. An exporter to another format reads these records and nothing else, so a fact an
8
+ exporter needs is added here and reaches every format.
9
+
10
+ Nothing is evaluated here that `repro verify` does not evaluate. Each assertion goes through
11
+ `claims.verify.check_evidence`, each pin through `claims.verify.artifact_states`, and what is
12
+ known of the commands upstream comes from `claims.stale.stale` and the record of re-runs.
13
+ Executes nothing and writes nothing.
14
+
15
+ **Experimental.** The version is in every record, and a field may change before `1.0`.
16
+
17
+ A conversion, where the file stores a fraction and the manuscript prints a percentage, is a
18
+ second value beside `stored` on the file read: `converted`, which is then the one compared.
19
+ An aggregate, a statistic over several runs, is one entry in `reads` per member, each with
20
+ its own `stored`, and the statistic and what it computed in `comparison`.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import collections
26
+ import datetime
27
+ import enum
28
+ import json
29
+ import pathlib
30
+ import re
31
+ from collections.abc import Iterable, Mapping
32
+ from typing import Any
33
+
34
+ import yaml
35
+ from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, ValidationError
36
+
37
+ from claims.adapters.reads import reading_once
38
+ from claims.exceptions import ReproError
39
+ from claims.models import (
40
+ AggregateEvidence,
41
+ ArtifactState,
42
+ Claim,
43
+ ComparisonMode,
44
+ ComparisonStatus,
45
+ CorrespondenceEvidence,
46
+ Decision,
47
+ Digest,
48
+ Evidence,
49
+ ExecutionStatus,
50
+ ExtractionStatus,
51
+ Freshness,
52
+ Manifest,
53
+ MetricEvidence,
54
+ Origin,
55
+ Outcome,
56
+ QuoteEvidence,
57
+ Reason,
58
+ Regeneration,
59
+ RegenerationState,
60
+ StalenessReport,
61
+ TableCellEvidence,
62
+ Validity,
63
+ ValueEvidence,
64
+ )
65
+ from claims.reruns import reruns, upstream
66
+ from claims.resolve import stored
67
+ from claims.stale import stale
68
+ from claims.toolchain import adapter_version
69
+ from claims.verify import (
70
+ DEFAULT_BACKENDS,
71
+ Backend,
72
+ artifact_states,
73
+ check_evidence,
74
+ )
75
+
76
+ RECORD_VERSION = "reproducible.science/claim-check/0.1-experimental"
77
+ """The version every record written here carries. A common format for claim checks, named
78
+ under the domain that publishes its schema and its RO-Crate profile."""
79
+
80
+ LEGACY_RECORD_VERSION = "repro.check/0.1-experimental"
81
+ """The version 0.5.6 wrote, for the same format under its earlier name. Read throughout the
82
+ 0.6 series and never written. The next minor release after 0.6 drops it, and a record that
83
+ carries it is then a record of a version this reader does not know."""
84
+
85
+ #: Every version a record may carry and be read here.
86
+ READ_RECORD_VERSIONS = (RECORD_VERSION, LEGACY_RECORD_VERSION)
87
+
88
+ #: Where the JSON Schema of the record is published. `docs/schemas/` holds the file and the
89
+ #: site serves a copy of it under `/schemas/`.
90
+ SCHEMA_URL = "https://reproducible.science/schemas/claim-check-0.1-experimental.schema.json"
91
+
92
+ CHECKER = "repro"
93
+
94
+ #: Where the checker is described. A record names it so a reader can find out what the tool
95
+ #: is without knowing the name already.
96
+ CHECKER_URL = "https://github.com/elliottower/reproducible-science"
97
+
98
+
99
+ class CheckType(enum.StrEnum):
100
+ """Which proposition a record is about. Closed, and defined once, in SPEC 7.10.
101
+
102
+ Two tools can both say `verified` and have checked different things, so a record names
103
+ the proposition as well as the outcome. The first three are the ones this tool checks. The
104
+ rest are reserved for a record written from another tool's result, and this tool emits
105
+ none of them.
106
+ """
107
+
108
+ VALUE_COMPARISON = "value-comparison"
109
+ """A value the manuscript prints, against the value at an address in a file."""
110
+ PASSAGE_PRESENCE = "passage-presence"
111
+ """A passage occurs in a file."""
112
+ CORRESPONDENCE = "correspondence"
113
+ """The values at two addresses, in two files, agree."""
114
+ AGGREGATE_COMPARISON = "aggregate-comparison"
115
+ """A value the manuscript prints, against a statistic computed over the values at
116
+ several addresses. No one stored value is the value compared."""
117
+ ARTIFACT_IDENTITY = "artifact-identity"
118
+ """A file's bytes are the bytes that were recorded. No value in it is compared."""
119
+ LINEAGE_CHECK = "lineage-check"
120
+ """A file was produced by a recorded process from recorded inputs."""
121
+ RERUN_COMPARISON = "rerun-comparison"
122
+ """What a command wrote when run again, against what it wrote before."""
123
+ COVERAGE_CHECK = "coverage-check"
124
+ """Whether something the manuscript states has a claim declared for it."""
125
+ OTHER = "other"
126
+ """A proposition none of the above names. Never read as a comparison of values."""
127
+
128
+
129
+ class NormalizedOutcome(enum.StrEnum):
130
+ """What a check found, in five words defined by their preconditions. SPEC 7.10.
131
+
132
+ The emitting tool's own word travels beside it in `native_outcome`.
133
+ """
134
+
135
+ MATCH = "match"
136
+ """The file was read, the address resolved, a comparison ran, and the two agreed."""
137
+ MISMATCH = "mismatch"
138
+ """The file was read, the address resolved, a comparison ran, and the two differed."""
139
+ NOT_FOUND = "not-found"
140
+ """The file was readable and the address did not resolve to exactly one value of the kind
141
+ the assertion needs. Nothing was compared."""
142
+ NOT_CHECKED = "not-checked"
143
+ """No comparison occurred, because something it needed was absent or was skipped."""
144
+ ERROR = "error"
145
+ """The tool failed."""
146
+
147
+
148
+ #: The proposition each evidence kind asserts.
149
+ CHECK_TYPES: dict[str, CheckType] = {
150
+ "metric": CheckType.VALUE_COMPARISON,
151
+ "aggregate": CheckType.AGGREGATE_COMPARISON,
152
+ "table": CheckType.VALUE_COMPARISON,
153
+ "value": CheckType.VALUE_COMPARISON,
154
+ "quote": CheckType.PASSAGE_PRESENCE,
155
+ "correspondence": CheckType.CORRESPONDENCE,
156
+ }
157
+
158
+ #: The check types whose `match` this version can read as a comparison that held.
159
+ UNDERSTOOD_CHECK_TYPES = frozenset(CHECK_TYPES.values())
160
+
161
+ #: The address kinds this version knows: every locator of SPEC 3.6, and a quotation's.
162
+ KNOWN_ADDRESS_KINDS = frozenset(
163
+ {"tree", "table", "table_position", "array", "sheet", "sqlite", "prose", "passage"}
164
+ )
165
+
166
+ #: The outcome words each checker is known to emit, by checker name. A word from a checker
167
+ #: that is not listed here is not understood, whatever it looks like.
168
+ NATIVE_OUTCOMES: dict[str, frozenset[str]] = {
169
+ CHECKER: frozenset(outcome.value for outcome in Outcome),
170
+ }
171
+
172
+
173
+ class UnmappedError(ReproError):
174
+ """An outcome or an evidence kind has no place in the record's vocabulary yet."""
175
+
176
+ def __init__(self, what: str, value: str) -> None:
177
+ self.what = what
178
+ self.value = value
179
+ super().__init__(f"{what} {value!r} has no mapping into a check record")
180
+
181
+
182
+ def normalized(outcome: Outcome) -> NormalizedOutcome:
183
+ """The record's outcome for one of this tool's own. The one place the mapping is written.
184
+
185
+ Total by a test over every member and never by a default: an outcome added to the engine
186
+ and not placed here raises, since any fallback would be a guess about what a check found.
187
+ `not_offered` is here for that reason, though no record is written for a claim that offers
188
+ no evidence.
189
+ """
190
+ mapping = {
191
+ Outcome.VERIFIED: NormalizedOutcome.MATCH,
192
+ Outcome.MISMATCH: NormalizedOutcome.MISMATCH,
193
+ Outcome.NOT_FOUND: NormalizedOutcome.NOT_FOUND,
194
+ Outcome.UNCHECKED: NormalizedOutcome.NOT_CHECKED,
195
+ Outcome.NOT_OFFERED: NormalizedOutcome.NOT_CHECKED,
196
+ Outcome.ERROR: NormalizedOutcome.ERROR,
197
+ }
198
+ if outcome not in mapping:
199
+ raise UnmappedError("outcome", str(outcome))
200
+ return mapping[outcome]
201
+
202
+
203
+ def check_type(kind: str) -> CheckType:
204
+ """The proposition an evidence kind asserts, or `UnmappedError` for a kind not placed."""
205
+ if kind not in CHECK_TYPES:
206
+ raise UnmappedError("evidence kind", kind)
207
+ return CHECK_TYPES[kind]
208
+
209
+
210
+ #: The address kind of a quotation, which has no locator: the passage is the address.
211
+ PASSAGE = "passage"
212
+
213
+ #: Suffixes read as text when a quoted passage is looked for by line. Anything else is left
214
+ #: without a line, since a line in an extraction is not a line in the file.
215
+ TEXT_SUFFIXES = (".txt", ".md", ".markdown", ".tex", ".rst")
216
+
217
+ _SHA256_HEX = r"^[a-f0-9]{64}$"
218
+
219
+ _CONFIG = ConfigDict(frozen=True, extra="allow")
220
+
221
+
222
+ class Checker(BaseModel):
223
+ model_config = _CONFIG
224
+
225
+ name: str
226
+ version: str
227
+ url: str = ""
228
+ """Where the checker is described."""
229
+
230
+
231
+ class ManifestRef(BaseModel):
232
+ """The manifest the assertion was read from."""
233
+
234
+ model_config = _CONFIG
235
+
236
+ path: str = ""
237
+ """As named from the directory the checker ran in where the manifest lies under it, and
238
+ its file name otherwise. Never absolute: records are committed and published."""
239
+ digest: str | None = Field(default=None, pattern=_SHA256_HEX)
240
+ """sha256 of the manifest's bytes. None for a manifest that was not read from a file."""
241
+
242
+
243
+ class Passage(BaseModel):
244
+ """A passage one of the claim's quote assertions names, and where it sits."""
245
+
246
+ model_config = _CONFIG
247
+
248
+ assertion: int
249
+ """Index of the quote assertion within the claim."""
250
+ artifact: str
251
+ path: str = ""
252
+ text: str
253
+ line: int | None = Field(default=None, ge=1)
254
+ """The line the passage starts on, where the artifact is a text file and holds the
255
+ passage exactly once up to whitespace. None otherwise: a line is never guessed."""
256
+
257
+
258
+ class ClaimRef(BaseModel):
259
+ model_config = _CONFIG
260
+
261
+ id: str
262
+ text: str = ""
263
+ where: str | None = None
264
+ """Where the manuscript makes the claim, in the author's words."""
265
+ digest: str
266
+ line: int | None = Field(default=None, ge=1)
267
+ """The line of the manifest the claim is declared on."""
268
+ passages: tuple[Passage, ...] = ()
269
+
270
+
271
+ class AssertionRef(BaseModel):
272
+ model_config = _CONFIG
273
+
274
+ kind: str
275
+ name: str = ""
276
+ position: int = Field(default=0, ge=0)
277
+ """Which of the claim's assertions of this kind and name it is, from 0. With the claim's
278
+ id, the kind and the name, this is the identity an assertion keeps across edits, and the
279
+ one `.repro/checks.jsonl` records a history under."""
280
+ index: int = Field(ge=0)
281
+ """Where it is among all of the claim's assertions, from 0, as the manifest orders them."""
282
+ line: int | None = Field(default=None, ge=1)
283
+ """The line of the manifest the assertion is declared on."""
284
+
285
+
286
+ class Address(BaseModel):
287
+ """Where in a file an assertion looks, tagged by `kind`.
288
+
289
+ The fields beside `kind` are those of the locator: `{"kind": "tree", "pointer": ...}`,
290
+ `{"kind": "table", "column": ..., "where": {...}}`, and so on for each kind SPEC 3.6
291
+ defines. A quotation is `{"kind": "passage"}` with the page or the neighbouring text it
292
+ names. A kind this version does not know is kept as it was written.
293
+ """
294
+
295
+ model_config = _CONFIG
296
+
297
+ kind: str
298
+
299
+
300
+ class FileState(BaseModel):
301
+ """One declared file: its pin, and what is on disk."""
302
+
303
+ model_config = _CONFIG
304
+
305
+ artifact: str
306
+ path: str = ""
307
+ media_type: str = ""
308
+ validity: Validity | None = None
309
+ """None where the manifest declares no such artifact."""
310
+ pinned: str | None = None
311
+ on_disk: str | None = None
312
+ origin: Origin | None = None
313
+ """The repository, commit and path the manifest says the bytes were copied from. Recorded
314
+ as declared; whether the bytes there are the pinned ones is `claims origins`'s question."""
315
+
316
+
317
+ class FileRead(FileState):
318
+ """One file an assertion reads, the address it reads, and the value there."""
319
+
320
+ side: str = ""
321
+ """Which side of a `correspondence` this is, in the author's word for it."""
322
+ address: Address | None = None
323
+ """None where the assertion names no one address."""
324
+ stored: str | None = None
325
+ """The value at the address, as the file stores it. None where none was read."""
326
+ converted: str | None = None
327
+ """`stored` under the conversion `comparison` names, which is then the value compared.
328
+ None where no conversion is declared or no number was read."""
329
+
330
+
331
+ class Tie(BaseModel):
332
+ """A compared value exactly halfway between two values at the printed precision, either
333
+ of which a manuscript may print."""
334
+
335
+ model_config = _CONFIG
336
+
337
+ lower: str
338
+ upper: str
339
+ printed: str
340
+
341
+
342
+ class Comparison(BaseModel):
343
+ """The rule a printed value was compared under, and what it was compared with where that
344
+ is not a stored value as it stands."""
345
+
346
+ model_config = _CONFIG
347
+
348
+ mode: ComparisonMode
349
+ tolerance: str = "0"
350
+ places: int | None = None
351
+ """The decimal places the assertion declares the manuscript reports the value to."""
352
+ conversion: str | None = None
353
+ """The conversion declared between what a file stores and what is printed: `percent` or
354
+ `fraction`. A side of a `correspondence` that declares one says so in its address."""
355
+ statistic: str | None = None
356
+ """What an `aggregate` computes over the values in `reads`."""
357
+ count: int | None = None
358
+ """How many members an `aggregate` declares, which is how many entries `reads` has."""
359
+ computed: str | None = None
360
+ """That statistic of the values read, which is what was compared with `printed`."""
361
+ tie: Tie | None = None
362
+
363
+
364
+ class Stages(BaseModel):
365
+ """The three stages SPEC 4 reports separately, from which the outcome is derived."""
366
+
367
+ model_config = _CONFIG
368
+
369
+ execution: ExecutionStatus
370
+ extraction: ExtractionStatus
371
+ comparison: ComparisonStatus
372
+
373
+
374
+ class Reader(BaseModel):
375
+ """What read the bytes: the backend, and the program that extracted from the file."""
376
+
377
+ model_config = _CONFIG
378
+
379
+ backend: str = ""
380
+ backend_version: str = ""
381
+ tool: str = ""
382
+ tool_version: str = ""
383
+ extraction_digest: str = ""
384
+
385
+
386
+ class Rerun(BaseModel):
387
+ """One witnessed re-run of a command, as `repro reproduce` recorded it."""
388
+
389
+ model_config = _CONFIG
390
+
391
+ started_at: str = ""
392
+ ended_at: str = ""
393
+ """When the invocation that ran the command started and ended. The command ran between
394
+ the two; `duration_seconds` is its own."""
395
+ state: Regeneration
396
+ reason: str = ""
397
+ checker_version: str = ""
398
+ exit_code: int | None = None
399
+ duration_seconds: float | None = None
400
+ output_digest: str | None = None
401
+ """The digest of the output the re-run wrote."""
402
+
403
+
404
+ class Command(BaseModel):
405
+ """A declared command upstream of what the assertion reads."""
406
+
407
+ model_config = _CONFIG
408
+
409
+ id: str
410
+ command: tuple[str, ...]
411
+ steps: int = Field(ge=0)
412
+ """How many commands lie between the assertion and this one. 0 for a command that writes
413
+ a file the assertion reads."""
414
+ output: FileState
415
+ inputs: tuple[FileState, ...] = ()
416
+ state: Freshness
417
+ causes: tuple[str, ...] = ()
418
+ last_rerun: Rerun | None = None
419
+ """The latest re-run of the command that the record of re-runs holds, whatever it found.
420
+ None where none is recorded."""
421
+
422
+
423
+ class Execution(BaseModel):
424
+ """What is known of the commands that produce what the assertion reads."""
425
+
426
+ model_config = _CONFIG
427
+
428
+ state: Freshness
429
+ """The assertion's state against the record of re-runs, as `repro stale` gives it."""
430
+ commands: tuple[Command, ...]
431
+ """Nearest first."""
432
+
433
+
434
+ class History(BaseModel):
435
+ """A pointer into the record of checks, for the assertion a record is about."""
436
+
437
+ model_config = _CONFIG
438
+
439
+ states: int = Field(ge=0)
440
+ """How many states the record of checks holds for the assertion."""
441
+ last_changed_at: str | None = None
442
+ """When a check last found it different from the state recorded before."""
443
+
444
+
445
+ class ExportRecord(BaseModel):
446
+ """One assertion, as it was checked. Frozen, and carries no verdict."""
447
+
448
+ model_config = ConfigDict(
449
+ frozen=True,
450
+ extra="allow",
451
+ json_schema_extra={
452
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
453
+ "$id": SCHEMA_URL,
454
+ },
455
+ )
456
+
457
+ record: str = Field(
458
+ default=RECORD_VERSION, json_schema_extra={"enum": list(READ_RECORD_VERSIONS)}
459
+ )
460
+ """Any string is held as written, and `problems` names one this reader does not know."""
461
+ checker: Checker
462
+ checked_at: AwareDatetime
463
+ manifest: ManifestRef = ManifestRef()
464
+ claim: ClaimRef
465
+ assertion: AssertionRef
466
+ check_type: CheckType
467
+ reads: tuple[FileRead, ...] = ()
468
+ printed: str | None = None
469
+ """The value as the manuscript prints it, the manifest's `reported`, or the passage a
470
+ quotation asserts. None for a `correspondence`, which compares two files and prints no
471
+ value of its own."""
472
+ comparison: Comparison | None = None
473
+ """None for a quotation, which is present or absent."""
474
+ outcome: NormalizedOutcome
475
+ native_outcome: str
476
+ """The checker's own word for what it found, as it prints it."""
477
+ native_reason: str
478
+ """The checker's own reason for the outcome."""
479
+ detail: str = ""
480
+ stages: Stages | None = None
481
+ """The three stages this tool derives its outcome from. None in a record written from a
482
+ tool that reports none."""
483
+ reader: Reader = Reader()
484
+ warnings: tuple[str, ...] = ()
485
+ """The checker's own notes on the evidence, as it words them."""
486
+ execution: Execution | None = None
487
+ """Present only where a declared command writes a file the assertion reads."""
488
+ history: History | None = None
489
+ """What the project's record of checks holds for this assertion. Reserved: nothing writes
490
+ it yet, and a record is derived when it is exported, from no stored state."""
491
+
492
+ def to_json(self) -> str:
493
+ """One line. A field with nothing known is left out."""
494
+ return json.dumps(self.model_dump(mode="json", exclude_none=True), ensure_ascii=False)
495
+
496
+
497
+ # ---------------------------------------------------------------------------------- building
498
+
499
+
500
+ def _state(artifact_id: str, manifest: Manifest, states: dict[str, ArtifactState]) -> dict:
501
+ declared, state = manifest.artifact(artifact_id), states.get(artifact_id)
502
+ return {
503
+ "artifact": artifact_id,
504
+ "path": declared.path.as_posix() if declared else "",
505
+ "media_type": declared.media_type if declared else "",
506
+ "validity": state.validity if state else None,
507
+ "pinned": state.expected if state else None,
508
+ "on_disk": state.actual if state else None,
509
+ "origin": declared.origin if declared else None,
510
+ }
511
+
512
+
513
+ def _address(
514
+ evidence: QuoteEvidence | MetricEvidence | TableCellEvidence | ValueEvidence,
515
+ ) -> Address | None:
516
+ if isinstance(evidence, QuoteEvidence):
517
+ named = evidence.model_dump(
518
+ mode="json", exclude={"kind", "artifact", "text"}, exclude_defaults=True
519
+ )
520
+ return Address(kind=PASSAGE, **named)
521
+ if isinstance(evidence, TableCellEvidence) and not evidence.addresses_one_row:
522
+ # Neither `row` nor `where`, or both: there is no one address to state.
523
+ return None
524
+ return Address.model_validate(evidence.locator.model_dump(mode="json"))
525
+
526
+
527
+ def _reads(
528
+ evidence: Evidence,
529
+ decision: Decision,
530
+ manifest: Manifest,
531
+ states: dict[str, ArtifactState],
532
+ paths: dict[str, pathlib.Path],
533
+ ) -> tuple[FileRead, ...]:
534
+ # A value is stated only where the check ran. An address may still resolve where a
535
+ # backend failed or a format on the other side has no reader, and a record that held the
536
+ # value would report a reading no check was made with.
537
+ ran = decision.execution is ExecutionStatus.COMPLETED
538
+ if isinstance(evidence, CorrespondenceEvidence):
539
+ read = {side.name: side for side in decision.sides}
540
+ return tuple(
541
+ FileRead(
542
+ **_state(side.artifact, manifest, states),
543
+ side=side.name,
544
+ address=Address.model_validate(
545
+ side.locator.model_dump(mode="json")
546
+ | ({"conversion": side.conversion.value} if side.conversion else {})
547
+ ),
548
+ stored=read[side.name].extracted if ran and side.name in read else None,
549
+ converted=read[side.name].converted if ran and side.name in read else None,
550
+ )
551
+ for side in evidence.sides
552
+ )
553
+ if isinstance(evidence, AggregateEvidence):
554
+ # No member was read where a file is absent or undeclared: the engine stops before
555
+ # the backend, and the decision lists none.
556
+ found = [(member.extracted, member.converted) for member in decision.members]
557
+ return tuple(
558
+ FileRead(
559
+ **_state(member.artifact, manifest, states),
560
+ address=Address.model_validate(member.locator.model_dump(mode="json")),
561
+ stored=value if ran else None,
562
+ converted=converted if ran else None,
563
+ )
564
+ for member, (value, converted) in zip(
565
+ evidence.members, found or [(None, None)] * evidence.count, strict=True
566
+ )
567
+ )
568
+ path = paths.get(evidence.artifact)
569
+ return (
570
+ FileRead(
571
+ **_state(evidence.artifact, manifest, states),
572
+ address=_address(evidence),
573
+ stored=stored(evidence, path) if ran and path is not None else None,
574
+ converted=decision.converted if ran else None,
575
+ ),
576
+ )
577
+
578
+
579
+ def _comparison(evidence: Evidence, decision: Decision) -> Comparison | None:
580
+ if isinstance(evidence, QuoteEvidence):
581
+ return None
582
+ conversion = getattr(evidence, "conversion", None)
583
+ return Comparison(
584
+ mode=evidence.mode,
585
+ tolerance=evidence.tolerance,
586
+ places=getattr(evidence, "places", None),
587
+ conversion=conversion.value if conversion else None,
588
+ statistic=decision.statistic.value if decision.statistic else None,
589
+ count=evidence.count if isinstance(evidence, AggregateEvidence) else None,
590
+ computed=decision.computed,
591
+ tie=Tie(**decision.tie.model_dump()) if decision.tie else None,
592
+ )
593
+
594
+
595
+ def _declared_lines(path: pathlib.Path) -> dict[int, tuple[int, tuple[int, ...]]]:
596
+ """For each claim by position in the manifest: its line, and the line of each assertion.
597
+
598
+ Read off the YAML as written, so a line is where an editor shows the claim. Empty where
599
+ the file does not parse into the expected shape; the manifest loader has then already
600
+ refused it.
601
+ """
602
+ loader = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
603
+ try:
604
+ root = yaml.compose(path.read_text(encoding="utf-8"), Loader=loader)
605
+ except (OSError, yaml.YAMLError):
606
+ return {}
607
+ if not isinstance(root, yaml.MappingNode):
608
+ return {}
609
+
610
+ def child(node: yaml.MappingNode, key: str) -> yaml.Node | None:
611
+ return next((value for name, value in node.value if name.value == key), None)
612
+
613
+ claims = child(root, "claims")
614
+ if not isinstance(claims, yaml.SequenceNode):
615
+ return {}
616
+ lines = {}
617
+ for index, claim in enumerate(claims.value):
618
+ evidence = child(claim, "evidence") if isinstance(claim, yaml.MappingNode) else None
619
+ lines[index] = (
620
+ claim.start_mark.line + 1,
621
+ tuple(item.start_mark.line + 1 for item in evidence.value)
622
+ if isinstance(evidence, yaml.SequenceNode)
623
+ else (),
624
+ )
625
+ return lines
626
+
627
+
628
+ def _passage_line(
629
+ path: pathlib.Path, passage: str, texts: dict[pathlib.Path, str | None]
630
+ ) -> int | None:
631
+ """The line a passage starts on in a text file that holds it exactly once, else None.
632
+
633
+ Matched up to whitespace and no further. A passage found only after the normalization a
634
+ quotation is checked under gets no line, and neither does one that occurs twice.
635
+ """
636
+ if path.suffix.lower() not in TEXT_SUFFIXES or not passage.split():
637
+ return None
638
+ if path not in texts:
639
+ try:
640
+ texts[path] = path.read_text(encoding="utf-8")
641
+ except (OSError, UnicodeDecodeError):
642
+ texts[path] = None
643
+ text = texts[path]
644
+ if text is None:
645
+ return None
646
+ found = list(re.finditer(r"\s+".join(re.escape(word) for word in passage.split()), text))
647
+ return text.count("\n", 0, found[0].start()) + 1 if len(found) == 1 else None
648
+
649
+
650
+ def _rerun(when: tuple[str, str, str], entry: RegenerationState) -> Rerun:
651
+ started_at, ended_at, version = when
652
+ return Rerun(
653
+ started_at=started_at,
654
+ ended_at=ended_at,
655
+ state=entry.state,
656
+ reason=entry.reason.value,
657
+ checker_version=version,
658
+ exit_code=entry.exit_code,
659
+ duration_seconds=entry.duration_seconds,
660
+ output_digest=entry.actual,
661
+ )
662
+
663
+
664
+ def _execution(
665
+ evidence: Evidence,
666
+ state: Freshness,
667
+ manifest: Manifest,
668
+ states: dict[str, ArtifactState],
669
+ report: StalenessReport,
670
+ latest: dict[str, Rerun],
671
+ ) -> Execution | None:
672
+ """The commands upstream of what one assertion reads, nearest first, or None where no
673
+ declared command writes a file it reads."""
674
+ records = {record.id: record for record in manifest.regenerations}
675
+ producers = {record.output.artifact: record for record in manifest.regenerations}
676
+ steps: dict[str, int] = {}
677
+ for name in evidence.artifacts:
678
+ producer = producers.get(name)
679
+ if producer is None:
680
+ continue
681
+ for record_id, distance in {**upstream(producer, producers), producer.id: 0}.items():
682
+ steps[record_id] = min(distance, steps.get(record_id, distance))
683
+ if not steps:
684
+ return None
685
+ nodes = {node.id: node for node in report.regenerations}
686
+ return Execution(
687
+ state=state,
688
+ commands=tuple(
689
+ Command(
690
+ id=record_id,
691
+ command=records[record_id].command,
692
+ steps=steps[record_id],
693
+ output=FileState(**_state(records[record_id].output.artifact, manifest, states)),
694
+ inputs=tuple(
695
+ FileState(**_state(wanted.artifact, manifest, states))
696
+ for wanted in records[record_id].inputs
697
+ ),
698
+ state=nodes[record_id].state,
699
+ causes=tuple(cause.value for cause in nodes[record_id].causes),
700
+ last_rerun=latest.get(record_id),
701
+ )
702
+ for record_id in sorted(steps, key=steps.__getitem__)
703
+ ),
704
+ )
705
+
706
+
707
+ def _manifest_ref(manifest: Manifest) -> ManifestRef:
708
+ if manifest.path is None:
709
+ return ManifestRef()
710
+ here = pathlib.Path.cwd().resolve()
711
+ resolved = manifest.path.resolve()
712
+ return ManifestRef(
713
+ path=resolved.relative_to(here).as_posix()
714
+ if resolved.is_relative_to(here)
715
+ else resolved.name,
716
+ digest=Digest.of_file(manifest.path).value,
717
+ )
718
+
719
+
720
+ def check_records(
721
+ manifest: Manifest,
722
+ checked_at: datetime.datetime | None = None,
723
+ backends: tuple[Backend, ...] = DEFAULT_BACKENDS,
724
+ ) -> tuple[ExportRecord, ...]:
725
+ """Check every assertion in a manifest and return one record for each, in the order the
726
+ manifest declares them.
727
+
728
+ `checked_at` is the time every record carries; now, where none is given. Executes nothing
729
+ and writes nothing.
730
+ """
731
+ when = checked_at or datetime.datetime.now(datetime.UTC).replace(microsecond=0)
732
+ checker = Checker(name=CHECKER, version=adapter_version(), url=CHECKER_URL)
733
+ reference = _manifest_ref(manifest)
734
+ lines = _declared_lines(manifest.path) if manifest.path else {}
735
+
736
+ report = stale(manifest)
737
+ root = manifest.path.parent if manifest.path else pathlib.Path.cwd()
738
+ latest: dict[str, Rerun] = {}
739
+ for invocation in reruns(root):
740
+ stamp = (invocation.started_at, invocation.ended_at, invocation.tool_version)
741
+ for entry in invocation.regenerations:
742
+ if entry.executed:
743
+ latest[entry.regeneration_id] = _rerun(stamp, entry)
744
+
745
+ states, paths = artifact_states(manifest)
746
+ registry: dict[str, Backend] = {b.kind: b for b in backends}
747
+ texts: dict[pathlib.Path, str | None] = {}
748
+ records = []
749
+ with reading_once():
750
+ for position, (claim, node) in enumerate(zip(manifest.claims, report.claims, strict=True)):
751
+ claim_line, evidence_lines = lines.get(position, (None, ()))
752
+ reference_to_claim = ClaimRef(
753
+ id=claim.id,
754
+ text=claim.text,
755
+ where=claim.where,
756
+ digest=claim.digest.value,
757
+ line=claim_line,
758
+ passages=_passages(claim, manifest, paths, texts),
759
+ )
760
+ seen: collections.Counter[tuple[str, str]] = collections.Counter()
761
+ for index, (evidence, freshness) in enumerate(
762
+ zip(claim.evidence, node.assertions, strict=True)
763
+ ):
764
+ decision = check_evidence(claim, evidence, manifest, paths, states, registry)
765
+ records.append(
766
+ ExportRecord(
767
+ checker=checker,
768
+ checked_at=when,
769
+ manifest=reference,
770
+ claim=reference_to_claim,
771
+ assertion=AssertionRef(
772
+ kind=evidence.kind,
773
+ name=getattr(evidence, "name", ""),
774
+ position=seen[evidence.kind, getattr(evidence, "name", "")],
775
+ index=index,
776
+ line=evidence_lines[index] if index < len(evidence_lines) else None,
777
+ ),
778
+ check_type=check_type(evidence.kind),
779
+ reads=_reads(evidence, decision, manifest, states, paths),
780
+ printed=evidence.text
781
+ if isinstance(evidence, QuoteEvidence)
782
+ else getattr(evidence, "reported", None),
783
+ comparison=_comparison(evidence, decision),
784
+ outcome=normalized(decision.outcome),
785
+ native_outcome=decision.outcome.value,
786
+ native_reason=decision.reason.value,
787
+ detail=decision.detail,
788
+ stages=Stages(
789
+ execution=decision.execution,
790
+ extraction=decision.extraction,
791
+ comparison=decision.comparison,
792
+ ),
793
+ reader=Reader(
794
+ backend=decision.backend,
795
+ backend_version=decision.backend_version,
796
+ tool=decision.tool,
797
+ tool_version=decision.tool_version,
798
+ extraction_digest=decision.extraction_digest,
799
+ ),
800
+ warnings=tuple(warning.value for warning in decision.warnings),
801
+ execution=_execution(
802
+ evidence, freshness.state, manifest, states, report, latest
803
+ ),
804
+ )
805
+ )
806
+ seen[evidence.kind, getattr(evidence, "name", "")] += 1
807
+ return tuple(records)
808
+
809
+
810
+ def _passages(
811
+ claim: Claim,
812
+ manifest: Manifest,
813
+ paths: dict[str, pathlib.Path],
814
+ texts: dict[pathlib.Path, str | None],
815
+ ) -> tuple[Passage, ...]:
816
+ found = []
817
+ for index, evidence in enumerate(claim.evidence):
818
+ if not isinstance(evidence, QuoteEvidence):
819
+ continue
820
+ declared, path = manifest.artifact(evidence.artifact), paths.get(evidence.artifact)
821
+ found.append(
822
+ Passage(
823
+ assertion=index,
824
+ artifact=evidence.artifact,
825
+ path=declared.path.as_posix() if declared else "",
826
+ text=evidence.text,
827
+ line=_passage_line(path, evidence.text, texts) if path is not None else None,
828
+ )
829
+ )
830
+ return tuple(found)
831
+
832
+
833
+ def to_jsonl(records: Iterable[ExportRecord]) -> str:
834
+ """JSON Lines: one record per line, in the order given."""
835
+ return "".join(f"{record.to_json()}\n" for record in records)
836
+
837
+
838
+ # -------------------------------------------------------------------------------- validating
839
+
840
+
841
+ class InvalidRecordError(ReproError):
842
+ """A record does not say something a check could have found."""
843
+
844
+ def __init__(self, problems: tuple[str, ...]) -> None:
845
+ self.problems = problems
846
+ super().__init__("; ".join(problems))
847
+
848
+
849
+ def _flattened(stages: Stages) -> Outcome:
850
+ """The outcome SPEC 4 derives from three stages, by the rule `Decision` derives it."""
851
+ return Decision(
852
+ claim_id="",
853
+ claim_digest="",
854
+ kind="",
855
+ execution=stages.execution,
856
+ extraction=stages.extraction,
857
+ comparison=stages.comparison,
858
+ reason=Reason.NOT_OFFERED,
859
+ ).outcome
860
+
861
+
862
+ def _pin_problems(file: FileState) -> list[str]:
863
+ """Validity is a fact about a file and its pin, and the two digests have to bear it out."""
864
+ name = f"file {file.artifact!r}"
865
+ if file.validity is Validity.AUTHORITATIVE and (not file.pinned or file.pinned != file.on_disk):
866
+ return [f"{name} is authoritative, so its pinned digest must be the one on disk"]
867
+ if file.validity is Validity.BROKEN_PIN and (
868
+ not file.pinned or not file.on_disk or file.pinned == file.on_disk
869
+ ):
870
+ return [f"{name} is a broken pin, so it must carry two digests that differ"]
871
+ if file.validity is Validity.UNPINNED_ARTIFACT and file.pinned:
872
+ return [f"{name} is unpinned, so it must carry no pinned digest"]
873
+ if file.validity is Validity.ARTIFACT_ABSENT and file.on_disk:
874
+ return [f"{name} is absent, so it must carry no digest on disk"]
875
+ return []
876
+
877
+
878
+ def understood(record: ExportRecord) -> bool:
879
+ """Whether this version knows what the record's outcome is an outcome of.
880
+
881
+ It does where the checker's own word is one that checker is known to emit, the check type
882
+ is one whose comparison this version can read, and every address is of a kind it knows.
883
+ """
884
+ return (
885
+ record.native_outcome in NATIVE_OUTCOMES.get(record.checker.name, frozenset())
886
+ and record.check_type in UNDERSTOOD_CHECK_TYPES
887
+ and all(
888
+ read.address is None or read.address.kind in KNOWN_ADDRESS_KINDS
889
+ for read in record.reads
890
+ )
891
+ )
892
+
893
+
894
+ def read_outcome(record: ExportRecord) -> NormalizedOutcome:
895
+ """The outcome a consumer may act on: the record's, except that a `match` it does not
896
+ understand is `not-checked`.
897
+
898
+ Unknown stays unknown. A word, a check type or an address kind this version has never
899
+ seen may be a success in the tool that wrote it, and nothing here can tell, so it is never
900
+ passed on as one. Every exporter reads the outcome through this.
901
+ """
902
+ if record.outcome is NormalizedOutcome.MATCH and not understood(record):
903
+ return NormalizedOutcome.NOT_CHECKED
904
+ return record.outcome
905
+
906
+
907
+ def _a(word: str) -> str:
908
+ return "an" if word[:1] in "aeiou" else "a"
909
+
910
+
911
+ def _value_problems(record: ExportRecord) -> list[str]:
912
+ """What each check type has to carry for its outcome to be one a check could reach."""
913
+ values = [read.stored for read in record.reads if read.stored is not None]
914
+ converted = [read.converted for read in record.reads if read.converted is not None]
915
+ compared = record.outcome in (NormalizedOutcome.MATCH, NormalizedOutcome.MISMATCH)
916
+ comparison = record.comparison
917
+ word = record.outcome.value
918
+ found = []
919
+ if comparison is not None and comparison.tie is not None:
920
+ tie = comparison.tie
921
+ if record.outcome is NormalizedOutcome.MATCH and tie.printed not in (tie.lower, tie.upper):
922
+ found.append("a match on a tie prints one of the two values the tie lies between")
923
+ if record.check_type is CheckType.VALUE_COMPARISON:
924
+ if compared and (record.printed is None or not values):
925
+ found.append(f"a value-comparison that is a {word} needs a stored and a printed value")
926
+ elif compared and comparison is not None and comparison.conversion and not converted:
927
+ found.append(
928
+ f"a {word} under the conversion {comparison.conversion!r} needs the converted "
929
+ f"value that was compared"
930
+ )
931
+ elif record.check_type is CheckType.AGGREGATE_COMPARISON:
932
+ if comparison is None or not comparison.statistic or comparison.count is None:
933
+ found.append("an aggregate-comparison names its statistic and its count")
934
+ elif comparison.count != len(record.reads):
935
+ found.append(
936
+ f"an aggregate-comparison of {comparison.count} members reads "
937
+ f"{len(record.reads)} files"
938
+ )
939
+ elif compared and (
940
+ record.printed is None or comparison.computed is None or len(values) != comparison.count
941
+ ):
942
+ found.append(
943
+ f"an aggregate-comparison that is a {word} needs the printed value, every "
944
+ f"member's stored value and what the statistic computed"
945
+ )
946
+ elif record.check_type is CheckType.PASSAGE_PRESENCE:
947
+ if compared and not record.printed:
948
+ found.append(f"a passage-presence that is a {word} needs the passage looked for")
949
+ if values:
950
+ found.append("a passage-presence compares no value, so none can be stored")
951
+ elif record.check_type is CheckType.CORRESPONDENCE:
952
+ if len(record.reads) != 2 or record.printed is not None:
953
+ found.append("a correspondence reads two files and prints no value of its own")
954
+ elif compared and len(values) != 2:
955
+ found.append(f"a correspondence that is a {word} needs the value stored on each side")
956
+ elif record.check_type is not CheckType.RERUN_COMPARISON and (
957
+ values or record.printed is not None or record.comparison is not None
958
+ ):
959
+ # Identity, lineage and coverage are facts about a file or a manuscript as a whole.
960
+ # A pair of values on one of them would say a value was compared, and none was.
961
+ found.append(f"a {record.check_type.value} compares no values, so it carries none")
962
+ return found
963
+
964
+
965
+ def problems(record: ExportRecord) -> tuple[str, ...]:
966
+ """What a record says that no check could have found. Empty where it is consistent.
967
+
968
+ A schema says which fields a record may hold. This says which records are possible: an
969
+ outcome whose preconditions the rest of the record contradicts, fields its check type
970
+ cannot have, a `match` under a word this version does not know.
971
+ """
972
+ found = []
973
+ if record.record not in READ_RECORD_VERSIONS:
974
+ versions = " and ".join(repr(version) for version in READ_RECORD_VERSIONS)
975
+ found.append(f"record is {record.record!r}; this reader knows {versions}")
976
+
977
+ known = record.native_outcome in NATIVE_OUTCOMES.get(record.checker.name, frozenset())
978
+ if not known and record.outcome is NormalizedOutcome.MATCH:
979
+ found.append(
980
+ f"{record.checker.name!r} is not known to emit {record.native_outcome!r}, and a "
981
+ f"word that is not understood is not-checked at best, never a match"
982
+ )
983
+ if known and record.checker.name == CHECKER:
984
+ expected = normalized(Outcome(record.native_outcome))
985
+ if expected is not record.outcome:
986
+ found.append(
987
+ f"outcome {record.outcome.value!r} is not what {record.native_outcome!r} "
988
+ f"normalizes to, which is {expected.value!r}"
989
+ )
990
+ if record.stages is not None and _flattened(record.stages).value != record.native_outcome:
991
+ found.append(f"{record.native_outcome!r} does not follow from the stages recorded")
992
+
993
+ placed = CHECK_TYPES.get(record.assertion.kind)
994
+ if record.checker.name == CHECKER and placed is not None and placed is not record.check_type:
995
+ found.append(
996
+ f"{_a(record.assertion.kind)} {record.assertion.kind!r} assertion is {_a(placed.value)} "
997
+ f"{placed.value}, and the record "
998
+ f"says {record.check_type.value}"
999
+ )
1000
+
1001
+ values = [read.stored for read in record.reads if read.stored is not None]
1002
+ stages = record.stages
1003
+ if record.outcome in (NormalizedOutcome.MATCH, NormalizedOutcome.MISMATCH):
1004
+ # The file was read, the address resolved and a comparison ran.
1005
+ if any(read.validity is Validity.ARTIFACT_ABSENT for read in record.reads):
1006
+ found.append(f"a {record.outcome.value} needs every file to have been read")
1007
+ if stages is not None and (
1008
+ stages.execution is not ExecutionStatus.COMPLETED
1009
+ or stages.extraction is not ExtractionStatus.EXTRACTED
1010
+ or stages.comparison.value != record.outcome.value
1011
+ ):
1012
+ found.append(f"a {record.outcome.value} needs a comparison that ran and found one")
1013
+ else:
1014
+ if stages is not None and stages.comparison is not ComparisonStatus.NOT_APPLICABLE:
1015
+ found.append(f"{record.outcome.value} means nothing was compared")
1016
+ if record.outcome is NormalizedOutcome.NOT_FOUND:
1017
+ if any(read.validity is Validity.ARTIFACT_ABSENT for read in record.reads):
1018
+ found.append("not-found needs a readable file; an absent one is not-checked")
1019
+ if stages is not None and stages.execution is not ExecutionStatus.COMPLETED:
1020
+ found.append("not-found needs a check that ran")
1021
+ if record.outcome in (NormalizedOutcome.NOT_CHECKED, NormalizedOutcome.ERROR) and values:
1022
+ found.append(f"{record.outcome.value} means no check ran, so no stored value was read")
1023
+ if (
1024
+ record.outcome is NormalizedOutcome.ERROR
1025
+ and stages is not None
1026
+ and stages.execution is not ExecutionStatus.FAILED
1027
+ ):
1028
+ found.append("error means the tool failed, and the stages say it did not")
1029
+
1030
+ found += _value_problems(record)
1031
+ for read in record.reads:
1032
+ found += _pin_problems(read)
1033
+ if record.execution is not None:
1034
+ if not record.execution.commands:
1035
+ found.append("an execution block names one or more commands, or is left out")
1036
+ for command in record.execution.commands:
1037
+ for file in (command.output, *command.inputs):
1038
+ found += _pin_problems(file)
1039
+ return tuple(found)
1040
+
1041
+
1042
+ def validate(data: Mapping[str, Any]) -> ExportRecord:
1043
+ """Read one record, or raise `InvalidRecordError` saying everything wrong with it."""
1044
+ try:
1045
+ record = ExportRecord.model_validate(data)
1046
+ except ValidationError as e:
1047
+ raise InvalidRecordError(
1048
+ tuple(
1049
+ f"{'.'.join(str(x) for x in error['loc']) or '(top level)'}: {error['msg']}"
1050
+ for error in e.errors()
1051
+ )
1052
+ ) from e
1053
+ if found := problems(record):
1054
+ raise InvalidRecordError(found)
1055
+ return record
1056
+
1057
+
1058
+ __all__ = [
1059
+ "CHECKER",
1060
+ "CHECKER_URL",
1061
+ "CHECK_TYPES",
1062
+ "KNOWN_ADDRESS_KINDS",
1063
+ "LEGACY_RECORD_VERSION",
1064
+ "NATIVE_OUTCOMES",
1065
+ "PASSAGE",
1066
+ "READ_RECORD_VERSIONS",
1067
+ "RECORD_VERSION",
1068
+ "SCHEMA_URL",
1069
+ "UNDERSTOOD_CHECK_TYPES",
1070
+ "Address",
1071
+ "AssertionRef",
1072
+ "CheckType",
1073
+ "Checker",
1074
+ "ClaimRef",
1075
+ "Command",
1076
+ "Comparison",
1077
+ "Execution",
1078
+ "ExportRecord",
1079
+ "FileRead",
1080
+ "FileState",
1081
+ "History",
1082
+ "InvalidRecordError",
1083
+ "ManifestRef",
1084
+ "NormalizedOutcome",
1085
+ "Passage",
1086
+ "Reader",
1087
+ "Rerun",
1088
+ "Stages",
1089
+ "Tie",
1090
+ "UnmappedError",
1091
+ "check_records",
1092
+ "check_type",
1093
+ "normalized",
1094
+ "problems",
1095
+ "read_outcome",
1096
+ "to_jsonl",
1097
+ "understood",
1098
+ "validate",
1099
+ ]