claims-cli 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. claims/__init__.py +132 -0
  2. claims/__main__.py +5 -0
  3. claims/adapters/__init__.py +6 -0
  4. claims/adapters/array.py +212 -0
  5. claims/adapters/base.py +75 -0
  6. claims/adapters/cells.py +94 -0
  7. claims/adapters/columnar.py +188 -0
  8. claims/adapters/prose.py +261 -0
  9. claims/adapters/reads.py +47 -0
  10. claims/adapters/rows.py +116 -0
  11. claims/adapters/sheet.py +149 -0
  12. claims/adapters/sqlite.py +73 -0
  13. claims/adapters/table.py +135 -0
  14. claims/adapters/tree.py +163 -0
  15. claims/checks.py +511 -0
  16. claims/cli.py +1103 -0
  17. claims/core/__init__.py +9 -0
  18. claims/core/artifacts.py +121 -0
  19. claims/core/checks.py +132 -0
  20. claims/core/claims.py +100 -0
  21. claims/core/evidence.py +708 -0
  22. claims/core/manifest.py +349 -0
  23. claims/core/numbers.py +146 -0
  24. claims/core/outcomes.py +410 -0
  25. claims/core/report.py +503 -0
  26. claims/corpus.py +344 -0
  27. claims/exceptions.py +106 -0
  28. claims/export/__init__.py +23 -0
  29. claims/export/rocrate.py +787 -0
  30. claims/export/sarif.py +401 -0
  31. claims/manifest.py +59 -0
  32. claims/models.py +222 -0
  33. claims/numbers.py +453 -0
  34. claims/origins.py +303 -0
  35. claims/policy.py +369 -0
  36. claims/provenance.py +42 -0
  37. claims/record.py +1099 -0
  38. claims/regression.py +171 -0
  39. claims/renderers/__init__.py +10 -0
  40. claims/renderers/sarif.py +294 -0
  41. claims/reruns.py +111 -0
  42. claims/resolve.py +116 -0
  43. claims/stale.py +333 -0
  44. claims/starter.py +432 -0
  45. claims/toolchain.py +120 -0
  46. claims/verify.py +1371 -0
  47. claims/why.py +398 -0
  48. claims_cli-0.6.0.dist-info/METADATA +139 -0
  49. claims_cli-0.6.0.dist-info/RECORD +51 -0
  50. claims_cli-0.6.0.dist-info/WHEEL +4 -0
  51. claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/resolve.py ADDED
@@ -0,0 +1,116 @@
1
+ """Dispatching a locator to the adapter for its format.
2
+
3
+ One contract over several formats, rather than one pointer syntax pretending every file is a
4
+ tree. Each adapter in `claims.adapters` translates a locator into the addressing its format
5
+ already has -- a JSON Pointer into JSON, a column and key predicate into a table, a sheet as
6
+ well into a workbook, a dataset name and index into an array -- and every adapter enforces the same invariant:
7
+
8
+ 0 matches -> ABSENT
9
+ 1 scalar -> RESOLVED
10
+ 2 or more -> AMBIGUOUS
11
+ a container -> NOT_SCALAR
12
+
13
+ No adapter takes the first match, and no adapter falls back to searching a file for the
14
+ printed number. A search would find the number wherever it appears and call that
15
+ verification, which is the failure the whole package exists to prevent. A format with no
16
+ adapter reports `FORMAT_UNSUPPORTED` and stops.
17
+
18
+ **Values come back as text.** The original representation is preserved rather than parsed to
19
+ a binary float, because converting `0.1` to a float and back reintroduces exactly the
20
+ third-significant-figure disagreements the comparison is meant to measure.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import pathlib
26
+ from collections.abc import Callable
27
+ from typing import Any
28
+
29
+ from claims.adapters.array import _resolve_array
30
+ from claims.adapters.base import ExtractedValue, Found, Resolution, _no
31
+ from claims.adapters.prose import CARDINALS as CARDINALS
32
+ from claims.adapters.prose import _resolve_prose
33
+ from claims.adapters.sheet import _resolve_sheet
34
+ from claims.adapters.sqlite import _resolve_sqlite
35
+ from claims.adapters.table import _resolve_table, _resolve_table_position, predicate_text
36
+ from claims.adapters.table import read_table as read_table
37
+ from claims.adapters.table import sniff_delimiter as sniff_delimiter
38
+ from claims.adapters.tree import _resolve_tree
39
+ from claims.adapters.tree import resolve_pointer as resolve_pointer
40
+ from claims.exceptions import ArtifactUnreadableError
41
+ from claims.models import Evidence, ValueLocator
42
+
43
+ #: Formats named here have no adapter. Listing them means an unsupported artifact is reported
44
+ #: as unsupported rather than as an unreadable one.
45
+ UNSUPPORTED = {
46
+ ".mat": "MATLAB",
47
+ ".rda": "RData",
48
+ ".rdata": "RData",
49
+ ".sas7bdat": "SAS",
50
+ }
51
+
52
+ #: Formats no adapter will ever open, because reading one runs code. A pickle is a program
53
+ #: whose last instruction returns the object, and `joblib` and PyTorch checkpoints are pickles.
54
+ NEVER_OPENED = {
55
+ ".pkl": "a pickle",
56
+ ".pickle": "a pickle",
57
+ ".joblib": "a joblib file, which is a pickle",
58
+ ".pt": "a PyTorch checkpoint, which is a pickle",
59
+ }
60
+
61
+ #: Keyed by `Locator.kind`, which is what makes the dispatch below sound: each adapter takes
62
+ #: the one variant whose discriminator selects it. The annotation is deliberately loose,
63
+ #: because a mapping cannot express "the value whose parameter type matches this key".
64
+ _ADAPTERS: dict[str, Callable[[Any, pathlib.Path], Found]] = {
65
+ "tree": _resolve_tree,
66
+ "table": _resolve_table,
67
+ "table_position": _resolve_table_position,
68
+ "sqlite": _resolve_sqlite,
69
+ "array": _resolve_array,
70
+ "sheet": _resolve_sheet,
71
+ "prose": _resolve_prose,
72
+ }
73
+
74
+
75
+ def resolve(locator: ValueLocator, path: pathlib.Path) -> Found:
76
+ """Resolve one locator against one artifact, to exactly one value or a reason."""
77
+ if program := NEVER_OPENED.get(path.suffix.lower()):
78
+ return _no(
79
+ Resolution.FORMAT_UNSUPPORTED,
80
+ f"{path.name} is {program}, and reading it runs whatever it was written to run; "
81
+ f"export the value to a format that is data",
82
+ )
83
+ if named := UNSUPPORTED.get(path.suffix.lower()):
84
+ return _no(
85
+ Resolution.FORMAT_UNSUPPORTED,
86
+ f"{named} has no adapter in this release; addressing it by guesswork "
87
+ f"would not be verification",
88
+ )
89
+ return _ADAPTERS[locator.kind](locator, path)
90
+
91
+
92
+ def stored(evidence: Evidence, path: pathlib.Path) -> str | None:
93
+ """The value an assertion addresses in one file, as stored, or None where there is none."""
94
+ locator = getattr(evidence, "locator", None)
95
+ if locator is None or not path.is_file():
96
+ return None
97
+ try:
98
+ _, extracted, _ = resolve(locator, path)
99
+ except (ArtifactUnreadableError, OSError, ValueError):
100
+ return None
101
+ return extracted.raw if extracted is not None else None
102
+
103
+
104
+ __all__ = [
105
+ "NEVER_OPENED",
106
+ "UNSUPPORTED",
107
+ "ExtractedValue",
108
+ "Found",
109
+ "Resolution",
110
+ "predicate_text",
111
+ "read_table",
112
+ "resolve",
113
+ "resolve_pointer",
114
+ "sniff_delimiter",
115
+ "stored",
116
+ ]
claims/stale.py ADDED
@@ -0,0 +1,333 @@
1
+ """Is what each claim reads still what a recorded re-run produced?
2
+
3
+ `repro verify` compares a manuscript with its files. It passes a number that sits in its file
4
+ unchanged while the data or the script that file was computed from has since been edited:
5
+ the output is still the pinned one, and nothing in a pin says what it was made from.
6
+ `repro reproduce` would find out, by running every command. This reads what `reproduce` has
7
+ already recorded, executes nothing and writes nothing.
8
+
9
+ **The reference is the record of re-runs, not the manifest.** A regeneration record carries a
10
+ digest for each input and for its output, and a project that rebuilds its manifest from the
11
+ files on disk rewrites all of them at once, so they agree after a rebuild whether or not the
12
+ command ran. What a rebuild cannot write is `.repro/reproductions.jsonl`, where each
13
+ `repro reproduce` appends the command it started, the digest of every input it copied into
14
+ the sandbox and the digest of the output the command wrote.
15
+
16
+ A regeneration is `current` where one of those re-runs, among the ones that executed and
17
+ reproduced, ran today's command over the inputs now on disk and its output is the output now
18
+ on disk: the file the re-run was compared against, or the one it wrote. An artifact takes the
19
+ state of the record that writes it, joined with every record upstream.
20
+
21
+ `current` is a statement about the record. It says the files are the ones a re-run that
22
+ reproduced was made over, and not that the command would reproduce now. An input the
23
+ manifest does not declare is not compared at all.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import collections
29
+ import pathlib
30
+ from collections.abc import Iterable
31
+
32
+ from claims.models import (
33
+ ArtifactNode,
34
+ AssertionNode,
35
+ ClaimNode,
36
+ Digest,
37
+ Edge,
38
+ Freshness,
39
+ Manifest,
40
+ PinDisagreement,
41
+ Regeneration,
42
+ RegenerationNode,
43
+ RegenerationRecord,
44
+ RegenerationState,
45
+ StaleCause,
46
+ StaleFinding,
47
+ StalenessReport,
48
+ )
49
+ from claims.reruns import RecordUnreadableError as RecordUnreadableError
50
+ from claims.reruns import canonical_digest, upstream
51
+ from claims.reruns import reruns as reruns
52
+ from claims.verify import artifact_states
53
+
54
+ #: One regeneration's entry in the record, with when its invocation started.
55
+ Rerun = tuple[str, RegenerationState]
56
+
57
+
58
+ def worst(states: Iterable[Freshness]) -> Freshness:
59
+ """The state of a thing that rests on several.
60
+
61
+ A file with no producer says nothing either way, so it never outranks a finding. Beside
62
+ something current it leaves the whole `partly_outside_the_record`.
63
+ """
64
+ found = set(states)
65
+ for state in (Freshness.STALE, Freshness.UNDETERMINED, Freshness.NEVER_RERUN):
66
+ if state in found:
67
+ return state
68
+ if found <= {Freshness.NO_PRODUCER}:
69
+ return Freshness.NO_PRODUCER
70
+ return Freshness.CURRENT if found == {Freshness.CURRENT} else Freshness.PARTLY_OUTSIDE
71
+
72
+
73
+ def _read(entry: RegenerationState) -> dict[str, str | None]:
74
+ return {reading.artifact_id: reading.observed for reading in entry.inputs}
75
+
76
+
77
+ def _compare(
78
+ record: RegenerationRecord,
79
+ recorded: list[Rerun],
80
+ attempted: list[Rerun],
81
+ disk: dict[str, str | None],
82
+ written: str | None,
83
+ loops: bool,
84
+ ) -> tuple[Freshness, Rerun | None, list[StaleFinding]]:
85
+ """One record against its own re-runs, saying nothing of the records upstream: its state,
86
+ the re-run it was compared with, and each difference from that re-run."""
87
+ output = record.output.artifact
88
+
89
+ def finding(cause: StaleCause, artifact: str, **digests: str | None) -> StaleFinding:
90
+ return StaleFinding(cause=cause, regeneration=record.id, artifact=artifact, **digests)
91
+
92
+ if loops:
93
+ return Freshness.UNDETERMINED, None, [finding(StaleCause.CYCLE, output)]
94
+ if not recorded:
95
+ return Freshness.NEVER_RERUN, None, []
96
+
97
+ missing = [
98
+ finding(StaleCause.INPUT_MISSING, wanted.artifact)
99
+ for wanted in record.inputs
100
+ if disk.get(wanted.artifact) is None
101
+ ]
102
+ if written is None:
103
+ missing.append(finding(StaleCause.OUTPUT_MISSING, output))
104
+ if missing:
105
+ return Freshness.UNDETERMINED, recorded[-1], missing
106
+
107
+ declared = {wanted.artifact: disk[wanted.artifact] for wanted in record.inputs}
108
+ for rerun in reversed(recorded):
109
+ entry = rerun[1]
110
+ if (
111
+ entry.command == record.command
112
+ and _read(entry) == declared
113
+ # A re-run that wrote other bytes holding the same numbers reproduced the file it
114
+ # was compared against, so that file is as much its output as the one it wrote.
115
+ and written in (entry.expected, entry.actual)
116
+ ):
117
+ # A later re-run of the same command over the same inputs that did not reproduce
118
+ # leaves two witnessed runs that disagree, and the record does not say which the
119
+ # file on disk is the output of.
120
+ later = attempted[attempted.index(rerun) + 1 :]
121
+ disagreeing = [
122
+ finding(
123
+ StaleCause.LATER_RERUN_DID_NOT_REPRODUCE,
124
+ output,
125
+ observed=other.actual,
126
+ on_disk=written,
127
+ rerun_at=when,
128
+ )
129
+ for when, other in later
130
+ if other.state is not Regeneration.REPRODUCED
131
+ and other.command == record.command
132
+ and _read(other) == declared
133
+ ]
134
+ if disagreeing:
135
+ return Freshness.UNDETERMINED, rerun, disagreeing[-1:]
136
+ return Freshness.CURRENT, rerun, []
137
+
138
+ when, last = recorded[-1]
139
+ found = []
140
+ if last.command != record.command:
141
+ found.append(finding(StaleCause.COMMAND_CHANGED, output, rerun_at=when))
142
+ read = _read(last)
143
+ found += [
144
+ finding(
145
+ StaleCause.INPUT_CHANGED,
146
+ name,
147
+ observed=read.get(name),
148
+ on_disk=declared.get(name),
149
+ rerun_at=when,
150
+ )
151
+ for name in sorted(read.keys() | declared.keys())
152
+ if read.get(name) != declared.get(name)
153
+ ]
154
+ if written not in (last.expected, last.actual):
155
+ found.append(
156
+ finding(
157
+ StaleCause.OUTPUT_CHANGED,
158
+ output,
159
+ observed=last.actual,
160
+ on_disk=written,
161
+ rerun_at=when,
162
+ )
163
+ )
164
+ return Freshness.STALE, recorded[-1], found
165
+
166
+
167
+ def stale(manifest: Manifest) -> StalenessReport:
168
+ """Compare every regeneration with the project's record of re-runs, and say for each
169
+ artifact and each claim whether what it rests on is still what a re-run produced.
170
+
171
+ Executes nothing and writes nothing.
172
+ """
173
+ root = manifest.path.parent if manifest.path else pathlib.Path.cwd()
174
+ log = reruns(root)
175
+ recorded: dict[str, list[Rerun]] = collections.defaultdict(list)
176
+ attempted: dict[str, list[Rerun]] = collections.defaultdict(list)
177
+ # Where each invocation ran, by when it started, which is how a re-run is named.
178
+ ran_in = {report.started_at: report.witness for report in log}
179
+ for report in log:
180
+ for entry in report.regenerations:
181
+ if not entry.executed:
182
+ continue
183
+ attempted[entry.regeneration_id].append((report.started_at, entry))
184
+ if entry.state is Regeneration.REPRODUCED:
185
+ recorded[entry.regeneration_id].append((report.started_at, entry))
186
+
187
+ states, paths = artifact_states(manifest)
188
+ disk = {name: state.actual for name, state in states.items()}
189
+ producers = {record.output.artifact: record for record in manifest.regenerations}
190
+ above = {record.id: upstream(record, producers) for record in manifest.regenerations}
191
+
192
+ local: dict[str, Freshness] = {}
193
+ compared: dict[str, Rerun | None] = {}
194
+ written: dict[str, str | None] = {}
195
+ differences: dict[str, list[StaleFinding]] = {}
196
+ for record in manifest.regenerations:
197
+ output = record.output.artifact
198
+ # The digest `reproduce` compares: the file's own, or the canonical one where the
199
+ # record names volatile fields.
200
+ written[record.id] = (
201
+ canonical_digest(paths[output], record.volatile).value
202
+ if disk.get(output) is not None
203
+ else None
204
+ )
205
+ local[record.id], compared[record.id], differences[record.id] = _compare(
206
+ record,
207
+ recorded[record.id],
208
+ attempted[record.id],
209
+ disk,
210
+ written[record.id],
211
+ loops=record.id in above[record.id],
212
+ )
213
+
214
+ nodes: dict[str, RegenerationNode] = {}
215
+ for record in manifest.regenerations:
216
+ own = local[record.id]
217
+ others = above[record.id]
218
+ # A record on a cycle has no upstream to take a state from: it is its own.
219
+ state = own if record.id in others else worst([own, *(local[name] for name in others)])
220
+ nearest = (
221
+ min((distance, name) for name, distance in others.items() if local[name] is state)
222
+ if state is not own
223
+ else None
224
+ )
225
+ rerun = compared[record.id]
226
+ nodes[record.id] = RegenerationNode(
227
+ id=record.id,
228
+ artifact=record.output.artifact,
229
+ state=state,
230
+ causes=(
231
+ tuple(dict.fromkeys(found.cause for found in differences[record.id]))
232
+ if nearest is None
233
+ else ((StaleCause.UPSTREAM_STALE,) if state is Freshness.STALE else ())
234
+ ),
235
+ upstream=nearest[1] if nearest else None,
236
+ distance=nearest[0] if nearest else None,
237
+ rerun_at=rerun[0] if rerun else None,
238
+ witness=ran_in.get(rerun[0]) if rerun else None,
239
+ )
240
+
241
+ def of(artifact_id: str) -> Freshness:
242
+ producer = producers.get(artifact_id)
243
+ return nodes[producer.id].state if producer else Freshness.NO_PRODUCER
244
+
245
+ claims = []
246
+ edges = []
247
+ for claim in manifest.claims:
248
+ assertions = tuple(
249
+ AssertionNode(
250
+ kind=evidence.kind,
251
+ artifacts=evidence.artifacts,
252
+ state=worst(of(name) for name in evidence.artifacts),
253
+ )
254
+ for evidence in claim.evidence
255
+ )
256
+ claims.append(
257
+ ClaimNode(id=claim.id, state=worst(a.state for a in assertions), assertions=assertions)
258
+ )
259
+ edges += [
260
+ Edge(kind="reads", source=claim.id, target=name, on_disk=disk.get(name))
261
+ for name in dict.fromkeys(name for a in assertions for name in a.artifacts)
262
+ ]
263
+
264
+ disagreements = []
265
+ for record in manifest.regenerations:
266
+ rerun = compared[record.id]
267
+ edges.append(
268
+ Edge(
269
+ kind="produced_by",
270
+ source=record.output.artifact,
271
+ target=record.id,
272
+ recorded=record.output.digest.value if record.output.digest else None,
273
+ observed=rerun[1].actual if rerun else None,
274
+ on_disk=written[record.id],
275
+ )
276
+ )
277
+ read = _read(rerun[1]) if rerun else {}
278
+ for wanted in record.inputs:
279
+ edges.append(
280
+ Edge(
281
+ kind="consumes",
282
+ source=record.id,
283
+ target=wanted.artifact,
284
+ recorded=wanted.digest.value if wanted.digest else None,
285
+ observed=read.get(wanted.artifact),
286
+ on_disk=disk.get(wanted.artifact),
287
+ )
288
+ )
289
+ pinned = manifest.artifact(wanted.artifact)
290
+ if (
291
+ wanted.digest is not None
292
+ and pinned is not None
293
+ and pinned.digest is not None
294
+ and wanted.digest != pinned.digest
295
+ ):
296
+ disagreements.append(
297
+ PinDisagreement(
298
+ regeneration=record.id,
299
+ artifact=wanted.artifact,
300
+ recorded=wanted.digest.value,
301
+ pinned=pinned.digest.value,
302
+ )
303
+ )
304
+
305
+ return StalenessReport(
306
+ project=manifest.project,
307
+ manifest_digest=Digest.of_text(manifest.model_dump_json()).value,
308
+ reruns=len(log),
309
+ claims=tuple(claims),
310
+ artifacts=tuple(
311
+ ArtifactNode(
312
+ id=artifact.id,
313
+ path=artifact.path.as_posix(),
314
+ state=of(artifact.id),
315
+ producer=producers[artifact.id].id if artifact.id in producers else None,
316
+ on_disk=disk.get(artifact.id),
317
+ )
318
+ for artifact in manifest.artifacts
319
+ ),
320
+ regenerations=tuple(nodes.values()),
321
+ edges=tuple(edges),
322
+ findings=tuple(
323
+ found.model_copy(update={"witness": ran_in.get(found.rerun_at)})
324
+ if found.rerun_at is not None
325
+ else found
326
+ for each in differences.values()
327
+ for found in each
328
+ ),
329
+ record_disagrees_with_pin=tuple(disagreements),
330
+ )
331
+
332
+
333
+ __all__ = ["RecordUnreadableError", "reruns", "stale", "worst"]