claims-cli 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claims/__init__.py +132 -0
- claims/__main__.py +5 -0
- claims/adapters/__init__.py +6 -0
- claims/adapters/array.py +212 -0
- claims/adapters/base.py +75 -0
- claims/adapters/cells.py +94 -0
- claims/adapters/columnar.py +188 -0
- claims/adapters/prose.py +261 -0
- claims/adapters/reads.py +47 -0
- claims/adapters/rows.py +116 -0
- claims/adapters/sheet.py +149 -0
- claims/adapters/sqlite.py +73 -0
- claims/adapters/table.py +135 -0
- claims/adapters/tree.py +163 -0
- claims/checks.py +511 -0
- claims/cli.py +1103 -0
- claims/core/__init__.py +9 -0
- claims/core/artifacts.py +121 -0
- claims/core/checks.py +132 -0
- claims/core/claims.py +100 -0
- claims/core/evidence.py +708 -0
- claims/core/manifest.py +349 -0
- claims/core/numbers.py +146 -0
- claims/core/outcomes.py +410 -0
- claims/core/report.py +503 -0
- claims/corpus.py +344 -0
- claims/exceptions.py +106 -0
- claims/export/__init__.py +23 -0
- claims/export/rocrate.py +787 -0
- claims/export/sarif.py +401 -0
- claims/manifest.py +59 -0
- claims/models.py +222 -0
- claims/numbers.py +453 -0
- claims/origins.py +303 -0
- claims/policy.py +369 -0
- claims/provenance.py +42 -0
- claims/record.py +1099 -0
- claims/regression.py +171 -0
- claims/renderers/__init__.py +10 -0
- claims/renderers/sarif.py +294 -0
- claims/reruns.py +111 -0
- claims/resolve.py +116 -0
- claims/stale.py +333 -0
- claims/starter.py +432 -0
- claims/toolchain.py +120 -0
- claims/verify.py +1371 -0
- claims/why.py +398 -0
- claims_cli-0.6.0.dist-info/METADATA +139 -0
- claims_cli-0.6.0.dist-info/RECORD +51 -0
- claims_cli-0.6.0.dist-info/WHEEL +4 -0
- claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/regression.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""Running a recorded finding against the revisions it was recorded at.
|
|
2
|
+
|
|
3
|
+
A regression entry names a repository, a revision where a finding reproduced, and optionally a
|
|
4
|
+
revision where it stopped. Running one checks out each revision in turn and verifies a
|
|
5
|
+
manifest against it.
|
|
6
|
+
|
|
7
|
+
The manifest lives in this corpus rather than in the repository under test, because a finding
|
|
8
|
+
is often recorded about a project that has not adopted this tool. Artifact paths in such a
|
|
9
|
+
manifest are relative to the repository root, and are resolved to absolute paths against the
|
|
10
|
+
checkout before verification, so nothing is written into the project being audited.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import pathlib
|
|
16
|
+
import tempfile
|
|
17
|
+
|
|
18
|
+
import yaml
|
|
19
|
+
from pydantic import BaseModel, ConfigDict
|
|
20
|
+
|
|
21
|
+
from claims.corpus import (
|
|
22
|
+
DEFAULT_CACHE,
|
|
23
|
+
CorpusEntry,
|
|
24
|
+
FindingState,
|
|
25
|
+
Regression,
|
|
26
|
+
_is_pinned_revision,
|
|
27
|
+
ensure,
|
|
28
|
+
)
|
|
29
|
+
from claims.manifest import load
|
|
30
|
+
from claims.models import VerificationReport
|
|
31
|
+
from claims.verify import verify
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RevisionResult(BaseModel):
|
|
35
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
36
|
+
|
|
37
|
+
commit: str
|
|
38
|
+
available: bool
|
|
39
|
+
counts: dict[str, int] = {}
|
|
40
|
+
expected: dict[str, int] = {}
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def matches_expected(self) -> bool:
|
|
44
|
+
return bool(self.expected) and self.counts == self.expected
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class FindingResult(BaseModel):
|
|
48
|
+
model_config = ConfigDict(frozen=True, extra="forbid")
|
|
49
|
+
|
|
50
|
+
name: str
|
|
51
|
+
state: FindingState
|
|
52
|
+
before: RevisionResult | None = None
|
|
53
|
+
after: RevisionResult | None = None
|
|
54
|
+
detail: str = ""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _manifest_against(manifest_path: pathlib.Path, root: pathlib.Path) -> VerificationReport:
|
|
58
|
+
"""Verify a manifest whose artifact paths are relative to `root`.
|
|
59
|
+
|
|
60
|
+
The rewritten copy is written to a temporary directory; the repository under test is only
|
|
61
|
+
read.
|
|
62
|
+
"""
|
|
63
|
+
raw = yaml.safe_load(manifest_path.read_text()) or {}
|
|
64
|
+
for artifact in raw.get("artifacts", []):
|
|
65
|
+
p = pathlib.Path(artifact["path"])
|
|
66
|
+
if not p.is_absolute():
|
|
67
|
+
artifact["path"] = str((root / p).resolve())
|
|
68
|
+
with tempfile.TemporaryDirectory() as td:
|
|
69
|
+
tmp = pathlib.Path(td) / "repro.yaml"
|
|
70
|
+
tmp.write_text(yaml.safe_dump(raw, sort_keys=False))
|
|
71
|
+
return verify(load(tmp))
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _at(
|
|
75
|
+
entry: Regression, revision, corpus_dir: pathlib.Path, cache: pathlib.Path
|
|
76
|
+
) -> RevisionResult:
|
|
77
|
+
if revision is None or not revision.commit:
|
|
78
|
+
return RevisionResult(commit="", available=False)
|
|
79
|
+
root = ensure(
|
|
80
|
+
CorpusEntry(
|
|
81
|
+
name=entry.name,
|
|
82
|
+
repository=entry.repository,
|
|
83
|
+
commit=revision.commit,
|
|
84
|
+
local_path=entry.local_path,
|
|
85
|
+
),
|
|
86
|
+
corpus_dir,
|
|
87
|
+
cache,
|
|
88
|
+
)
|
|
89
|
+
if root is None:
|
|
90
|
+
return RevisionResult(commit=revision.commit, available=False, expected=revision.expected)
|
|
91
|
+
# `ensure` returns an existing checkout unchanged where an entry has no remote, or under
|
|
92
|
+
# REPRO_OFFLINE. Nothing then re-checked which revision it was at, so counts measured at
|
|
93
|
+
# whatever HEAD happened to be were reported under the pinned commit -- and a finding
|
|
94
|
+
# could be called fixed on the strength of a revision never checked out.
|
|
95
|
+
if not _is_pinned_revision(
|
|
96
|
+
CorpusEntry(
|
|
97
|
+
name=entry.name,
|
|
98
|
+
repository=entry.repository,
|
|
99
|
+
commit=revision.commit,
|
|
100
|
+
local_path=entry.local_path,
|
|
101
|
+
),
|
|
102
|
+
root,
|
|
103
|
+
):
|
|
104
|
+
return RevisionResult(commit=revision.commit, available=False, expected=revision.expected)
|
|
105
|
+
manifest = corpus_dir / revision.manifest
|
|
106
|
+
if not manifest.is_file():
|
|
107
|
+
return RevisionResult(commit=revision.commit, available=False, expected=revision.expected)
|
|
108
|
+
report = _manifest_against(manifest, root)
|
|
109
|
+
return RevisionResult(
|
|
110
|
+
commit=revision.commit, available=True, counts=report.counts, expected=revision.expected
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def run(
|
|
115
|
+
entry: Regression, corpus_dir: pathlib.Path, cache: pathlib.Path = DEFAULT_CACHE
|
|
116
|
+
) -> FindingResult:
|
|
117
|
+
"""Check a recorded finding at the revisions it names."""
|
|
118
|
+
before = _at(entry, entry.before, corpus_dir, cache)
|
|
119
|
+
after = _at(entry, entry.after, corpus_dir, cache) if entry.after else None
|
|
120
|
+
|
|
121
|
+
if not before.available:
|
|
122
|
+
return FindingResult(
|
|
123
|
+
name=entry.name,
|
|
124
|
+
state=FindingState.UNAVAILABLE,
|
|
125
|
+
before=before,
|
|
126
|
+
after=after,
|
|
127
|
+
detail="the revision that showed it is unavailable",
|
|
128
|
+
)
|
|
129
|
+
if not before.matches_expected:
|
|
130
|
+
# The finding no longer reproduces where it was recorded. That is not a pass: either
|
|
131
|
+
# it was fixed without an `after` being recorded, or the entry is wrong.
|
|
132
|
+
return FindingResult(
|
|
133
|
+
name=entry.name,
|
|
134
|
+
state=FindingState.UNREPRODUCED,
|
|
135
|
+
before=before,
|
|
136
|
+
after=after,
|
|
137
|
+
detail=f"expected {before.expected} at {before.commit[:12]}, got {before.counts}",
|
|
138
|
+
)
|
|
139
|
+
if after is None:
|
|
140
|
+
return FindingResult(
|
|
141
|
+
name=entry.name,
|
|
142
|
+
state=FindingState.OPEN,
|
|
143
|
+
before=before,
|
|
144
|
+
detail="reproduced; no revision recorded as fixing it",
|
|
145
|
+
)
|
|
146
|
+
if not after.available:
|
|
147
|
+
return FindingResult(
|
|
148
|
+
name=entry.name,
|
|
149
|
+
state=FindingState.UNAVAILABLE,
|
|
150
|
+
before=before,
|
|
151
|
+
after=after,
|
|
152
|
+
detail="the revision that fixed it is unavailable",
|
|
153
|
+
)
|
|
154
|
+
if after.matches_expected:
|
|
155
|
+
return FindingResult(
|
|
156
|
+
name=entry.name,
|
|
157
|
+
state=FindingState.FIXED,
|
|
158
|
+
before=before,
|
|
159
|
+
after=after,
|
|
160
|
+
detail="reproduced at before, absent at after",
|
|
161
|
+
)
|
|
162
|
+
return FindingResult(
|
|
163
|
+
name=entry.name,
|
|
164
|
+
state=FindingState.UNREPRODUCED,
|
|
165
|
+
before=before,
|
|
166
|
+
after=after,
|
|
167
|
+
detail=f"expected {after.expected} at {after.commit[:12]}, got {after.counts}",
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
__all__ = ["FindingResult", "RevisionResult", "run"]
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Ways of presenting a report.
|
|
2
|
+
|
|
3
|
+
A renderer reads a `VerificationReport` and an `Assessment` and produces bytes. None computes
|
|
4
|
+
anything: two renderers over one report always say the same thing in different formats, which
|
|
5
|
+
is the property that makes it safe to add a third.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from claims.renderers.sarif import to_sarif
|
|
9
|
+
|
|
10
|
+
__all__ = ["to_sarif"]
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
"""SARIF 2.1.0 output.
|
|
2
|
+
|
|
3
|
+
SARIF is what code-scanning tools emit and what GitHub renders inline on a pull request, so a
|
|
4
|
+
report in this format shows up where a reviewer already looks rather than in a log they have
|
|
5
|
+
to open.
|
|
6
|
+
|
|
7
|
+
Two parts of the format fit this work unusually well. `artifacts[].hashes` carries a sha-256
|
|
8
|
+
per file, so the digests a report is computed against travel with it rather than being
|
|
9
|
+
described in prose. And `result.kind` distinguishes `fail`, `informational` and
|
|
10
|
+
`notApplicable` from `level`, which lets a check that could not run stay separate from one
|
|
11
|
+
that ran and failed -- the distinction this package exists to preserve, and one most report
|
|
12
|
+
formats cannot express.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from claims.models import Ordering, Outcome, Validity, VerificationReport
|
|
20
|
+
from claims.policy import Assessment, Severity
|
|
21
|
+
|
|
22
|
+
SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json"
|
|
23
|
+
VERSION = "2.1.0"
|
|
24
|
+
|
|
25
|
+
#: SARIF splits an outcome into `kind` (what happened) and `level` (how bad). A check that did
|
|
26
|
+
#: not run is `notApplicable`, never a `fail` at a lower level, because severity is not the
|
|
27
|
+
#: axis on which "unchecked" differs from "mismatch".
|
|
28
|
+
_KIND = {
|
|
29
|
+
Outcome.VERIFIED: ("pass", "none"),
|
|
30
|
+
Outcome.MISMATCH: ("fail", "error"),
|
|
31
|
+
Outcome.NOT_FOUND: ("fail", "warning"),
|
|
32
|
+
Outcome.UNCHECKED: ("notApplicable", "none"),
|
|
33
|
+
Outcome.ERROR: ("fail", "error"),
|
|
34
|
+
Outcome.NOT_OFFERED: ("informational", "note"),
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
_LEVEL = {Severity.ERROR: "error", Severity.WARNING: "warning", Severity.IGNORE: "note"}
|
|
38
|
+
_SEVERITY_ORDER = {"note": 0, "warning": 1, "error": 2}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _rules() -> list[dict[str, Any]]:
|
|
42
|
+
return (
|
|
43
|
+
[
|
|
44
|
+
{
|
|
45
|
+
"id": f"repro/{outcome.value}",
|
|
46
|
+
"name": outcome.value.replace("_", " ").title().replace(" ", ""),
|
|
47
|
+
"shortDescription": {"text": text},
|
|
48
|
+
"defaultConfiguration": {"level": _KIND[outcome][1]},
|
|
49
|
+
}
|
|
50
|
+
for outcome, text in [
|
|
51
|
+
(Outcome.VERIFIED, "The artifact holds what the claim says it holds."),
|
|
52
|
+
(Outcome.MISMATCH, "The artifact was read and holds something else."),
|
|
53
|
+
(Outcome.NOT_FOUND, "The artifact was read and holds no such value."),
|
|
54
|
+
(Outcome.UNCHECKED, "No comparison was made."),
|
|
55
|
+
(Outcome.ERROR, "A verifier failed."),
|
|
56
|
+
(Outcome.NOT_OFFERED, "The claim offers no evidence."),
|
|
57
|
+
]
|
|
58
|
+
]
|
|
59
|
+
+ [
|
|
60
|
+
{
|
|
61
|
+
# Artifact-level results carry these ids. A ruleId with no matching rule object is
|
|
62
|
+
# what a strict SARIF consumer rejects, so they are declared beside the outcomes.
|
|
63
|
+
"id": f"repro/{validity.value}",
|
|
64
|
+
"name": validity.value.replace("_", " ").title().replace(" ", ""),
|
|
65
|
+
"shortDescription": {"text": text},
|
|
66
|
+
"defaultConfiguration": {"level": "error"},
|
|
67
|
+
}
|
|
68
|
+
for validity, text in [
|
|
69
|
+
(Validity.BROKEN_PIN, "The file read is not the file that was pinned."),
|
|
70
|
+
(Validity.ARTIFACT_ABSENT, "Nothing exists at the declared path."),
|
|
71
|
+
]
|
|
72
|
+
]
|
|
73
|
+
+ [
|
|
74
|
+
{
|
|
75
|
+
"id": f"repro/ordering_{ordering.value}",
|
|
76
|
+
"name": f"Ordering{ordering.value.title()}",
|
|
77
|
+
"shortDescription": {"text": text},
|
|
78
|
+
"defaultConfiguration": {"level": level},
|
|
79
|
+
}
|
|
80
|
+
for ordering, text, level in [
|
|
81
|
+
(
|
|
82
|
+
Ordering.VIOLATED,
|
|
83
|
+
"A confirmatory run started before the plan it names was registered.",
|
|
84
|
+
"error",
|
|
85
|
+
),
|
|
86
|
+
(
|
|
87
|
+
Ordering.UNCHECKED,
|
|
88
|
+
"A confirmatory claim's ordering could not be established.",
|
|
89
|
+
"warning",
|
|
90
|
+
),
|
|
91
|
+
]
|
|
92
|
+
]
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def to_sarif(
|
|
97
|
+
report: VerificationReport, assessment: Assessment | None = None, version: str = "0"
|
|
98
|
+
) -> dict[str, Any]:
|
|
99
|
+
"""One SARIF run over a verification report.
|
|
100
|
+
|
|
101
|
+
`assessment` raises a result's level to what a policy says it is worth. Without one, each
|
|
102
|
+
result carries the default level for its outcome, so the output describes what was found
|
|
103
|
+
and takes no position on whether it is acceptable.
|
|
104
|
+
"""
|
|
105
|
+
by_subject: dict[str, str] = {}
|
|
106
|
+
if assessment is not None:
|
|
107
|
+
for violation in assessment.violations:
|
|
108
|
+
# Last-write-wins on a key two decisions can share: `claim_id/kind` is not
|
|
109
|
+
# unique across a claim's evidence, so an error-level mismatch rendered at the
|
|
110
|
+
# warning level of a `not_found` beside it. Keep the gravest.
|
|
111
|
+
existing = by_subject.get(violation.subject)
|
|
112
|
+
level = _LEVEL[violation.severity]
|
|
113
|
+
if existing is None or _SEVERITY_ORDER[level] > _SEVERITY_ORDER[existing]:
|
|
114
|
+
by_subject[violation.subject] = level
|
|
115
|
+
|
|
116
|
+
artifacts = [
|
|
117
|
+
{
|
|
118
|
+
"location": {"uri": state.artifact_id},
|
|
119
|
+
**({"hashes": {"sha-256": state.actual}} if state.actual else {}),
|
|
120
|
+
"description": {"text": state.validity.value},
|
|
121
|
+
}
|
|
122
|
+
for state in report.artifacts
|
|
123
|
+
]
|
|
124
|
+
index = {state.artifact_id: i for i, state in enumerate(report.artifacts)}
|
|
125
|
+
|
|
126
|
+
# SARIF results are heterogeneous by design -- a result carries different keys depending
|
|
127
|
+
# on what it reports -- so the value type is annotated rather than inferred from whichever
|
|
128
|
+
# entry happens to be appended first.
|
|
129
|
+
results: list[dict[str, Any]] = []
|
|
130
|
+
for claim in report.claims:
|
|
131
|
+
if not claim.decisions:
|
|
132
|
+
# Claim-level violations are keyed by `claim_id`, and this branch ignored them: a
|
|
133
|
+
# confirmatory claim offering no evidence, which the publication policy grades an
|
|
134
|
+
# error, rendered as an informational note. Code scanning shows notes and does not
|
|
135
|
+
# fail a build on them, so the gravest finding in the report was the one nobody saw.
|
|
136
|
+
level = by_subject.get(claim.claim_id, "note")
|
|
137
|
+
results.append(
|
|
138
|
+
{
|
|
139
|
+
"ruleId": f"repro/{Outcome.NOT_OFFERED.value}",
|
|
140
|
+
"kind": "informational" if level == "note" else "fail",
|
|
141
|
+
"level": level,
|
|
142
|
+
"message": {"text": f"{claim.claim_id}: no evidence offered"},
|
|
143
|
+
"partialFingerprints": {"claimDigest": claim.claim_digest},
|
|
144
|
+
}
|
|
145
|
+
)
|
|
146
|
+
continue
|
|
147
|
+
for decision in claim.decisions:
|
|
148
|
+
kind, level = _KIND[decision.outcome]
|
|
149
|
+
subject = f"{claim.claim_id}/{decision.kind}"
|
|
150
|
+
entry: dict[str, Any] = {
|
|
151
|
+
"ruleId": f"repro/{decision.outcome.value}",
|
|
152
|
+
"kind": kind,
|
|
153
|
+
"level": by_subject.get(subject, level),
|
|
154
|
+
"message": {
|
|
155
|
+
"text": f"{claim.claim_id}: {decision.detail or decision.reason.value}"
|
|
156
|
+
},
|
|
157
|
+
"properties": {
|
|
158
|
+
"execution": decision.execution.value,
|
|
159
|
+
"extraction": decision.extraction.value,
|
|
160
|
+
"comparison": decision.comparison.value,
|
|
161
|
+
"reason": decision.reason.value,
|
|
162
|
+
"validity": decision.validity.value,
|
|
163
|
+
"backend": f"{decision.backend}/{decision.backend_version}",
|
|
164
|
+
# The toolchain that read the bytes, kept apart from the backend's
|
|
165
|
+
# protocol version: a renderer that showed only the latter would repeat
|
|
166
|
+
# in the output the gap the fields exist to close.
|
|
167
|
+
"extractor": f"{decision.tool}/{decision.tool_version}",
|
|
168
|
+
"extractionDigest": decision.extraction_digest,
|
|
169
|
+
},
|
|
170
|
+
# A fingerprint over the claim and the bytes read, so a result is the same
|
|
171
|
+
# result across runs and a different one when either changes.
|
|
172
|
+
"partialFingerprints": {
|
|
173
|
+
"claimDigest": decision.claim_digest,
|
|
174
|
+
**(
|
|
175
|
+
{"artifactDigest": decision.artifact_digest}
|
|
176
|
+
if decision.artifact_digest
|
|
177
|
+
else {}
|
|
178
|
+
),
|
|
179
|
+
},
|
|
180
|
+
}
|
|
181
|
+
# A correspondence reads two artifacts and names neither in `artifact_id`, so its
|
|
182
|
+
# locations come from the sides. SARIF takes a list, and a finding about two files
|
|
183
|
+
# that pointed at one of them would send a reader to the wrong half. An aggregate
|
|
184
|
+
# is located at each file a member reads, once.
|
|
185
|
+
located = (
|
|
186
|
+
[s.artifact_id for s in decision.sides]
|
|
187
|
+
or list(dict.fromkeys(m.artifact_id for m in decision.members))
|
|
188
|
+
or [decision.artifact_id]
|
|
189
|
+
)
|
|
190
|
+
entry["locations"] = [
|
|
191
|
+
{"physicalLocation": {"artifactLocation": {"uri": uri, "index": index[uri]}}}
|
|
192
|
+
for uri in located
|
|
193
|
+
if uri in index
|
|
194
|
+
]
|
|
195
|
+
if not entry["locations"]:
|
|
196
|
+
del entry["locations"]
|
|
197
|
+
if decision.warnings:
|
|
198
|
+
entry["properties"]["warnings"] = [w.value for w in decision.warnings]
|
|
199
|
+
if decision.validity is not Validity.AUTHORITATIVE:
|
|
200
|
+
entry["properties"]["nonAuthoritative"] = True
|
|
201
|
+
results.append(entry)
|
|
202
|
+
|
|
203
|
+
for claim in report.claims:
|
|
204
|
+
if claim.ordering is Ordering.VIOLATED:
|
|
205
|
+
kind, level = "fail", "error"
|
|
206
|
+
elif claim.ordering is Ordering.UNCHECKED:
|
|
207
|
+
# Nothing is asserted either way, which is `notApplicable` rather than a
|
|
208
|
+
# low-severity failure -- the same distinction the outcome kinds preserve.
|
|
209
|
+
kind, level = "notApplicable", "warning"
|
|
210
|
+
else:
|
|
211
|
+
continue
|
|
212
|
+
results.append(
|
|
213
|
+
{
|
|
214
|
+
"ruleId": f"repro/ordering_{claim.ordering.value}",
|
|
215
|
+
"kind": kind,
|
|
216
|
+
"level": by_subject.get(claim.claim_id, level),
|
|
217
|
+
"message": {"text": f"{claim.claim_id}: {claim.ordering_detail}"},
|
|
218
|
+
"properties": {
|
|
219
|
+
"claimDigest": claim.claim_digest,
|
|
220
|
+
"orderingReason": claim.ordering_reason.value,
|
|
221
|
+
**(
|
|
222
|
+
{"registrationAuthority": claim.registration_authority.value}
|
|
223
|
+
if claim.registration_authority
|
|
224
|
+
else {}
|
|
225
|
+
),
|
|
226
|
+
},
|
|
227
|
+
}
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
for state in report.artifacts:
|
|
231
|
+
if state.validity is Validity.BROKEN_PIN:
|
|
232
|
+
message = (
|
|
233
|
+
f"{state.artifact_id}: pinned {(state.expected or '')[:12]}, "
|
|
234
|
+
f"found {(state.actual or '')[:12]}"
|
|
235
|
+
)
|
|
236
|
+
elif state.validity is Validity.ARTIFACT_ABSENT:
|
|
237
|
+
message = f"{state.artifact_id}: nothing at the declared path"
|
|
238
|
+
else:
|
|
239
|
+
continue
|
|
240
|
+
results.append(
|
|
241
|
+
{
|
|
242
|
+
"ruleId": f"repro/{state.validity.value}",
|
|
243
|
+
"kind": "fail",
|
|
244
|
+
"level": "error",
|
|
245
|
+
"message": {"text": message},
|
|
246
|
+
"locations": [
|
|
247
|
+
{
|
|
248
|
+
"physicalLocation": {
|
|
249
|
+
"artifactLocation": {
|
|
250
|
+
"uri": state.artifact_id,
|
|
251
|
+
"index": index[state.artifact_id],
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
],
|
|
256
|
+
}
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
return {
|
|
260
|
+
"$schema": SCHEMA,
|
|
261
|
+
"version": VERSION,
|
|
262
|
+
"runs": [
|
|
263
|
+
{
|
|
264
|
+
"tool": {
|
|
265
|
+
"driver": {
|
|
266
|
+
"name": "repro",
|
|
267
|
+
"version": version,
|
|
268
|
+
"informationUri": "https://github.com/elliottower/reproducible-science",
|
|
269
|
+
"rules": [
|
|
270
|
+
*_rules(),
|
|
271
|
+
{
|
|
272
|
+
"id": "repro/broken_pin",
|
|
273
|
+
"shortDescription": {
|
|
274
|
+
"text": "The file read is not the file that was pinned."
|
|
275
|
+
},
|
|
276
|
+
"defaultConfiguration": {"level": "error"},
|
|
277
|
+
},
|
|
278
|
+
],
|
|
279
|
+
}
|
|
280
|
+
},
|
|
281
|
+
"artifacts": artifacts,
|
|
282
|
+
"results": results,
|
|
283
|
+
"properties": {
|
|
284
|
+
"project": report.project,
|
|
285
|
+
"manifestDigest": report.manifest_digest,
|
|
286
|
+
**(
|
|
287
|
+
{"policy": assessment.policy, "passed": assessment.passed}
|
|
288
|
+
if assessment
|
|
289
|
+
else {}
|
|
290
|
+
),
|
|
291
|
+
},
|
|
292
|
+
}
|
|
293
|
+
],
|
|
294
|
+
}
|
claims/reruns.py
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""The record of re-runs, read: where it is, what it holds, and what a record's files rest on.
|
|
2
|
+
|
|
3
|
+
`repro reproduce` runs each declared command and appends what it found to
|
|
4
|
+
`.repro/reproductions.jsonl`. Nothing here runs a command or writes that file. This is the
|
|
5
|
+
half a reader of the record needs: the path, the entries parsed back into the model they were
|
|
6
|
+
written from, the digest an output is compared by, and which records lie upstream of one.
|
|
7
|
+
`claims.stale`, `claims.why` and `claims.record` read the record through this module, and
|
|
8
|
+
`repro.reproduce` and `repro.regenerate`, which write it, take the same names from here.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import collections
|
|
14
|
+
import json
|
|
15
|
+
import pathlib
|
|
16
|
+
|
|
17
|
+
from pydantic import ValidationError
|
|
18
|
+
|
|
19
|
+
from claims.exceptions import ReproError
|
|
20
|
+
from claims.models import Digest, RegenerationRecord, ReproductionReport
|
|
21
|
+
from claims.resolve import resolve_pointer
|
|
22
|
+
|
|
23
|
+
#: Where each invocation's record is appended, beside the manifest. One JSON object per line.
|
|
24
|
+
RECORD = pathlib.Path(".repro") / "reproductions.jsonl"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class RecordUnreadableError(ReproError):
|
|
28
|
+
"""A line of the record of re-runs is not a report `repro reproduce` wrote."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, path: pathlib.Path, line: int, detail: str) -> None:
|
|
31
|
+
self.path = path
|
|
32
|
+
self.line = line
|
|
33
|
+
super().__init__(f"{path}:{line}: {detail}")
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def reruns(root: pathlib.Path) -> tuple[ReproductionReport, ...]:
|
|
37
|
+
"""Every invocation in the project's record of re-runs, oldest first. Empty where the
|
|
38
|
+
project has no record."""
|
|
39
|
+
path = root / RECORD
|
|
40
|
+
if not path.is_file():
|
|
41
|
+
return ()
|
|
42
|
+
reports = []
|
|
43
|
+
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
|
44
|
+
if not line.strip():
|
|
45
|
+
continue
|
|
46
|
+
try:
|
|
47
|
+
reports.append(ReproductionReport.model_validate_json(line))
|
|
48
|
+
except ValidationError as e:
|
|
49
|
+
first = e.errors()[0]
|
|
50
|
+
where = ".".join(str(x) for x in first["loc"]) or "(top level)"
|
|
51
|
+
raise RecordUnreadableError(path, number, f"{where}: {first['msg']}") from e
|
|
52
|
+
return tuple(reports)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _drop(document: object, pointer: str) -> object:
|
|
56
|
+
"""Remove one JSON Pointer's target, if it resolves. Returns the document."""
|
|
57
|
+
if pointer in ("", "/"):
|
|
58
|
+
return document
|
|
59
|
+
parent_pointer, _, last = pointer.rpartition("/")
|
|
60
|
+
parent = resolve_pointer(document, parent_pointer)
|
|
61
|
+
key = last.replace("~1", "/").replace("~0", "~")
|
|
62
|
+
if isinstance(parent, dict):
|
|
63
|
+
parent.pop(key, None)
|
|
64
|
+
elif isinstance(parent, list) and key.isdigit() and int(key) < len(parent):
|
|
65
|
+
parent.pop(int(key))
|
|
66
|
+
return document
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def canonical_digest(path: pathlib.Path, volatile: tuple[str, ...]) -> Digest:
|
|
70
|
+
"""The artifact's digest, after removing any volatile fields.
|
|
71
|
+
|
|
72
|
+
Falls back to the file's own digest where the artifact is not JSON or names no volatile
|
|
73
|
+
fields, so the strict comparison stays the default.
|
|
74
|
+
"""
|
|
75
|
+
if not volatile:
|
|
76
|
+
return Digest.of_file(path)
|
|
77
|
+
try:
|
|
78
|
+
document = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
79
|
+
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
|
|
80
|
+
# Volatile fields are addressed as JSON Pointers, so a non-JSON artifact cannot have
|
|
81
|
+
# them removed. Comparing the raw bytes is the honest fallback.
|
|
82
|
+
return Digest.of_file(path)
|
|
83
|
+
for pointer in volatile:
|
|
84
|
+
document = _drop(document, pointer)
|
|
85
|
+
return Digest.of_text(
|
|
86
|
+
json.dumps(document, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def upstream(
|
|
91
|
+
record: RegenerationRecord, producers: dict[str, RegenerationRecord]
|
|
92
|
+
) -> dict[str, int]:
|
|
93
|
+
"""Every record upstream of this one, by id, with how many steps away it is.
|
|
94
|
+
|
|
95
|
+
`producers` maps an artifact to the record that writes it. Breadth first and each record
|
|
96
|
+
once, so the distance is the shortest, a record reached by two routes is listed once, and a
|
|
97
|
+
cycle ends the walk. The record itself is listed only where a cycle leads back to it.
|
|
98
|
+
"""
|
|
99
|
+
found: dict[str, int] = {}
|
|
100
|
+
pending = collections.deque([(record, 0)])
|
|
101
|
+
while pending:
|
|
102
|
+
current, distance = pending.popleft()
|
|
103
|
+
for wanted in current.inputs:
|
|
104
|
+
producer = producers.get(wanted.artifact)
|
|
105
|
+
if producer is not None and producer.id not in found:
|
|
106
|
+
found[producer.id] = distance + 1
|
|
107
|
+
pending.append((producer, distance + 1))
|
|
108
|
+
return found
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
__all__ = ["RECORD", "RecordUnreadableError", "canonical_digest", "reruns", "upstream"]
|