claims-cli 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claims/__init__.py +132 -0
- claims/__main__.py +5 -0
- claims/adapters/__init__.py +6 -0
- claims/adapters/array.py +212 -0
- claims/adapters/base.py +75 -0
- claims/adapters/cells.py +94 -0
- claims/adapters/columnar.py +188 -0
- claims/adapters/prose.py +261 -0
- claims/adapters/reads.py +47 -0
- claims/adapters/rows.py +116 -0
- claims/adapters/sheet.py +149 -0
- claims/adapters/sqlite.py +73 -0
- claims/adapters/table.py +135 -0
- claims/adapters/tree.py +163 -0
- claims/checks.py +511 -0
- claims/cli.py +1103 -0
- claims/core/__init__.py +9 -0
- claims/core/artifacts.py +121 -0
- claims/core/checks.py +132 -0
- claims/core/claims.py +100 -0
- claims/core/evidence.py +708 -0
- claims/core/manifest.py +349 -0
- claims/core/numbers.py +146 -0
- claims/core/outcomes.py +410 -0
- claims/core/report.py +503 -0
- claims/corpus.py +344 -0
- claims/exceptions.py +106 -0
- claims/export/__init__.py +23 -0
- claims/export/rocrate.py +787 -0
- claims/export/sarif.py +401 -0
- claims/manifest.py +59 -0
- claims/models.py +222 -0
- claims/numbers.py +453 -0
- claims/origins.py +303 -0
- claims/policy.py +369 -0
- claims/provenance.py +42 -0
- claims/record.py +1099 -0
- claims/regression.py +171 -0
- claims/renderers/__init__.py +10 -0
- claims/renderers/sarif.py +294 -0
- claims/reruns.py +111 -0
- claims/resolve.py +116 -0
- claims/stale.py +333 -0
- claims/starter.py +432 -0
- claims/toolchain.py +120 -0
- claims/verify.py +1371 -0
- claims/why.py +398 -0
- claims_cli-0.6.0.dist-info/METADATA +139 -0
- claims_cli-0.6.0.dist-info/RECORD +51 -0
- claims_cli-0.6.0.dist-info/WHEEL +4 -0
- claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/export/sarif.py
ADDED
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
"""The check records as a SARIF 2.1.0 log, for GitHub code scanning.
|
|
2
|
+
|
|
3
|
+
A projection for display, and a lossy one. SARIF has a place for a tool, a rule, a message, a
|
|
4
|
+
file and a line, and a digest per file. It has none for an address inside a file, a stored
|
|
5
|
+
value beside a reported one, a comparison rule, or which proposition was checked, so those
|
|
6
|
+
travel in each result's property bag under `repro/checkRecord`, where the whole record sits.
|
|
7
|
+
`NOT_CARRIED_NATIVELY` lists them.
|
|
8
|
+
|
|
9
|
+
What becomes a result is what a reader has to act on: `mismatch`, `not-found` and `error`.
|
|
10
|
+
A `match` is a result only where asked for, as `kind: pass`: GitHub does not read `kind` and
|
|
11
|
+
opens a warning alert for a pass, so it is not the default. A `not-checked` assertion is
|
|
12
|
+
never a result, since a result reads as a finding and a `pass` reads as a success, and it is
|
|
13
|
+
neither: it is a notification on the invocation, counted in the run's properties.
|
|
14
|
+
|
|
15
|
+
Reads `claims.record.ExportRecord` and nothing else, and every outcome through
|
|
16
|
+
`claims.record.read_outcome`, so a word this version does not know is never shown as a pass.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import collections
|
|
22
|
+
import datetime
|
|
23
|
+
import hashlib
|
|
24
|
+
import json
|
|
25
|
+
import posixpath
|
|
26
|
+
import warnings
|
|
27
|
+
from collections.abc import Sequence
|
|
28
|
+
from typing import Any
|
|
29
|
+
|
|
30
|
+
from claims.record import CheckType, ExportRecord, NormalizedOutcome, read_outcome
|
|
31
|
+
|
|
32
|
+
SCHEMA = "https://json.schemastore.org/sarif-2.1.0.json"
|
|
33
|
+
VERSION = "2.1.0"
|
|
34
|
+
|
|
35
|
+
#: Where the whole record sits in a result's or a notification's property bag.
|
|
36
|
+
RECORD_PROPERTY = "repro/checkRecord"
|
|
37
|
+
|
|
38
|
+
#: The partial fingerprint this exporter defines: a digest over the claim id and what
|
|
39
|
+
#: identifies the assertion within it. It does not move when a line does.
|
|
40
|
+
FINGERPRINT = "reproAssertion/v1"
|
|
41
|
+
|
|
42
|
+
#: The one partial fingerprint GitHub reads. Supplied, and set from `FINGERPRINT`: left out,
|
|
43
|
+
#: the upload action hashes the text of the annotated line, which for an assertion annotated
|
|
44
|
+
#: at its manifest entry is `- kind: metric` on every one of them.
|
|
45
|
+
GITHUB_FINGERPRINT = "primaryLocationLineHash"
|
|
46
|
+
|
|
47
|
+
#: GitHub keeps this many results of a run and drops the rest by severity.
|
|
48
|
+
GITHUB_RESULT_LIMIT = 5000
|
|
49
|
+
|
|
50
|
+
#: The fields of a record SARIF has no property for. Each is in the property bag and nowhere
|
|
51
|
+
#: else in the log.
|
|
52
|
+
NOT_CARRIED_NATIVELY = (
|
|
53
|
+
"record",
|
|
54
|
+
"manifest.digest",
|
|
55
|
+
"claim.text",
|
|
56
|
+
"claim.digest",
|
|
57
|
+
"claim.where",
|
|
58
|
+
"claim.passages",
|
|
59
|
+
"assertion.kind",
|
|
60
|
+
"assertion.name",
|
|
61
|
+
"assertion.position",
|
|
62
|
+
"assertion.index",
|
|
63
|
+
"check_type",
|
|
64
|
+
"reads.validity",
|
|
65
|
+
"reads.pinned",
|
|
66
|
+
"reads.side",
|
|
67
|
+
"reads.address",
|
|
68
|
+
"reads.stored",
|
|
69
|
+
"reads.converted",
|
|
70
|
+
"printed",
|
|
71
|
+
"comparison",
|
|
72
|
+
"outcome",
|
|
73
|
+
"native_outcome",
|
|
74
|
+
"native_reason",
|
|
75
|
+
"stages",
|
|
76
|
+
"reader",
|
|
77
|
+
"warnings",
|
|
78
|
+
"execution",
|
|
79
|
+
"history",
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
#: `kind` and `level` for each outcome that is a result. SARIF 3.27.9: `fail` is "a problem
|
|
83
|
+
#: whose severity is specified by the level property", `pass` is "evaluated, and no problem
|
|
84
|
+
#: was found". 3.27.10: a level other than `none` is only for `fail`.
|
|
85
|
+
_RESULT = {
|
|
86
|
+
NormalizedOutcome.MISMATCH: ("fail", "error"),
|
|
87
|
+
NormalizedOutcome.NOT_FOUND: ("fail", "warning"),
|
|
88
|
+
NormalizedOutcome.ERROR: ("fail", "error"),
|
|
89
|
+
NormalizedOutcome.MATCH: ("pass", "none"),
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class ResultLimitWarning(UserWarning):
|
|
94
|
+
"""A log holds more results than GitHub keeps."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
_SAID = {
|
|
98
|
+
NormalizedOutcome.MATCH: "The file was read, the address resolved, and the two agreed.",
|
|
99
|
+
NormalizedOutcome.MISMATCH: (
|
|
100
|
+
"The file was read and the address resolved, and what it holds is not what the "
|
|
101
|
+
"manuscript reports."
|
|
102
|
+
),
|
|
103
|
+
NormalizedOutcome.NOT_FOUND: (
|
|
104
|
+
"The file was read and the declared address did not resolve to one value. Nothing "
|
|
105
|
+
"was compared."
|
|
106
|
+
),
|
|
107
|
+
NormalizedOutcome.NOT_CHECKED: (
|
|
108
|
+
"No comparison occurred: a file, a reader or a format it needed was absent."
|
|
109
|
+
),
|
|
110
|
+
NormalizedOutcome.ERROR: "The checker failed on this assertion. Nothing was compared.",
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
_HELP = {
|
|
114
|
+
NormalizedOutcome.MATCH: "Nothing to do.",
|
|
115
|
+
NormalizedOutcome.MISMATCH: (
|
|
116
|
+
"Either the manuscript or the file is out of date. `{program} why <claim>` prints the "
|
|
117
|
+
"stored value beside the reported one, and what produced the file."
|
|
118
|
+
),
|
|
119
|
+
NormalizedOutcome.NOT_FOUND: (
|
|
120
|
+
"Correct the address in repro.yaml, or the file. A file that is silent on a value "
|
|
121
|
+
"does not contradict it, so this is not a mismatch."
|
|
122
|
+
),
|
|
123
|
+
NormalizedOutcome.NOT_CHECKED: (
|
|
124
|
+
"Supply what the check needs: the file, or the extra that reads its format."
|
|
125
|
+
),
|
|
126
|
+
NormalizedOutcome.ERROR: "A defect in the checker or a backend. Report it with the detail.",
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _rule_id(outcome: NormalizedOutcome, record: ExportRecord) -> str:
|
|
131
|
+
return f"repro/{outcome.value}/{record.native_reason}"
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _descriptor(
|
|
135
|
+
outcome: NormalizedOutcome, record: ExportRecord, program: str = "repro"
|
|
136
|
+
) -> dict[str, Any]:
|
|
137
|
+
reason = record.native_reason.replace("_", " ")
|
|
138
|
+
descriptor: dict[str, Any] = {
|
|
139
|
+
"id": _rule_id(outcome, record),
|
|
140
|
+
"name": "".join(
|
|
141
|
+
part.capitalize()
|
|
142
|
+
for part in f"{outcome.value}_{record.native_reason}".replace("-", "_").split("_")
|
|
143
|
+
),
|
|
144
|
+
"shortDescription": {"text": f"{outcome.value}: {reason}"},
|
|
145
|
+
"fullDescription": {"text": _SAID[outcome]},
|
|
146
|
+
"help": {"text": _HELP[outcome].format(program=program)},
|
|
147
|
+
}
|
|
148
|
+
if outcome in _RESULT and _RESULT[outcome][0] == "fail":
|
|
149
|
+
descriptor["defaultConfiguration"] = {"level": _RESULT[outcome][1]}
|
|
150
|
+
return descriptor
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _uri(record: ExportRecord, path: str) -> str:
|
|
154
|
+
"""A declared path as named from where the checker ran: the manifest's directory, then
|
|
155
|
+
the path the manifest gives."""
|
|
156
|
+
return posixpath.normpath(posixpath.join(posixpath.dirname(record.manifest.path), path))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _physical(uri: str, line: int | None) -> dict[str, Any]:
|
|
160
|
+
location: dict[str, Any] = {"artifactLocation": {"uri": uri}}
|
|
161
|
+
if line is not None:
|
|
162
|
+
location["region"] = {"startLine": line}
|
|
163
|
+
return location
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _location(record: ExportRecord) -> dict[str, Any] | None:
|
|
167
|
+
"""The one place a result is shown, or None where the record names no file.
|
|
168
|
+
|
|
169
|
+
The sentence, where the claim quotes one from a text file and it was found on one line:
|
|
170
|
+
the assertion's own passage first, then another of the claim's. Otherwise the assertion's
|
|
171
|
+
entry in the manifest. A line is never estimated.
|
|
172
|
+
"""
|
|
173
|
+
located = [passage for passage in record.claim.passages if passage.line is not None]
|
|
174
|
+
own = [passage for passage in located if passage.assertion == record.assertion.index]
|
|
175
|
+
for passage in (*own, *located):
|
|
176
|
+
return {"physicalLocation": _physical(_uri(record, passage.path), passage.line)}
|
|
177
|
+
if not record.manifest.path:
|
|
178
|
+
return None
|
|
179
|
+
return {
|
|
180
|
+
"physicalLocation": _physical(
|
|
181
|
+
record.manifest.path, record.assertion.line or record.claim.line
|
|
182
|
+
)
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _related(record: ExportRecord) -> list[dict[str, Any]]:
|
|
187
|
+
"""Each file the assertion reads, and its entry in the manifest where the result is shown
|
|
188
|
+
at the quoted sentence and not there."""
|
|
189
|
+
related = []
|
|
190
|
+
for read in record.reads:
|
|
191
|
+
if not read.path:
|
|
192
|
+
continue
|
|
193
|
+
held = f" holds {read.stored}" if read.stored is not None else ""
|
|
194
|
+
side = f"{read.side}: " if read.side else ""
|
|
195
|
+
related.append(
|
|
196
|
+
{
|
|
197
|
+
"id": len(related) + 1,
|
|
198
|
+
"physicalLocation": _physical(_uri(record, read.path), None),
|
|
199
|
+
"message": {"text": f"{side}{read.artifact}{held}"},
|
|
200
|
+
}
|
|
201
|
+
)
|
|
202
|
+
quoted = any(passage.line is not None for passage in record.claim.passages)
|
|
203
|
+
if quoted and record.manifest.path and record.assertion.line is not None:
|
|
204
|
+
related.append(
|
|
205
|
+
{
|
|
206
|
+
"id": len(related) + 1,
|
|
207
|
+
"physicalLocation": _physical(record.manifest.path, record.assertion.line),
|
|
208
|
+
"message": {"text": "declared here"},
|
|
209
|
+
}
|
|
210
|
+
)
|
|
211
|
+
return related
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _message(record: ExportRecord, outcome: NormalizedOutcome) -> str:
|
|
215
|
+
subject = f"{record.assertion.kind} {record.assertion.name}".strip()
|
|
216
|
+
said = record.detail or record.native_reason.replace("_", " ")
|
|
217
|
+
return f"{record.claim.id}: {subject}: {outcome.value}. {said}"
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _fingerprint(record: ExportRecord) -> str:
|
|
221
|
+
"""A digest over the assertion's identity (the claim's id, its kind, its name and its
|
|
222
|
+
position among the claim's assertions of that kind and name) and what it declares: the
|
|
223
|
+
files and addresses it reads, and for a quotation the passage.
|
|
224
|
+
|
|
225
|
+
Not the index, which moves when an assertion of another kind is inserted above, and not a
|
|
226
|
+
line. An address or a passage that is edited is another alert, as it is another assertion.
|
|
227
|
+
"""
|
|
228
|
+
identity = [
|
|
229
|
+
record.claim.id,
|
|
230
|
+
record.assertion.kind,
|
|
231
|
+
record.assertion.name,
|
|
232
|
+
record.assertion.position,
|
|
233
|
+
[
|
|
234
|
+
[read.artifact, read.address.model_dump(mode="json") if read.address else None]
|
|
235
|
+
for read in record.reads
|
|
236
|
+
],
|
|
237
|
+
record.printed if record.check_type is CheckType.PASSAGE_PRESENCE else None,
|
|
238
|
+
]
|
|
239
|
+
canonical = json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
240
|
+
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _properties(record: ExportRecord) -> dict[str, Any]:
|
|
244
|
+
return {RECORD_PROPERTY: record.model_dump(mode="json", exclude_none=True)}
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def to_sarif_log(
|
|
248
|
+
records: Sequence[ExportRecord], include_matches: bool = False, program: str = "repro"
|
|
249
|
+
) -> str:
|
|
250
|
+
"""One SARIF run over the records, as JSON text.
|
|
251
|
+
|
|
252
|
+
A result for each `mismatch`, `not-found` and `error`, and for each `match` where
|
|
253
|
+
`include_matches` is set. A notification for each `not-checked`, and for each `error`
|
|
254
|
+
beside its result.
|
|
255
|
+
"""
|
|
256
|
+
shown = [
|
|
257
|
+
(record, outcome)
|
|
258
|
+
for record in records
|
|
259
|
+
for outcome in (read_outcome(record),)
|
|
260
|
+
if outcome in _RESULT and (include_matches or outcome is not NormalizedOutcome.MATCH)
|
|
261
|
+
]
|
|
262
|
+
noted = [
|
|
263
|
+
(record, outcome)
|
|
264
|
+
for record in records
|
|
265
|
+
for outcome in (read_outcome(record),)
|
|
266
|
+
if outcome in (NormalizedOutcome.NOT_CHECKED, NormalizedOutcome.ERROR)
|
|
267
|
+
]
|
|
268
|
+
|
|
269
|
+
rules = {
|
|
270
|
+
_rule_id(outcome, record): _descriptor(outcome, record, program)
|
|
271
|
+
for record, outcome in shown
|
|
272
|
+
}
|
|
273
|
+
rule_index = {rule: index for index, rule in enumerate(sorted(rules))}
|
|
274
|
+
descriptors = {
|
|
275
|
+
_rule_id(outcome, record): _descriptor(outcome, record, program)
|
|
276
|
+
for record, outcome in noted
|
|
277
|
+
}
|
|
278
|
+
descriptor_index = {name: index for index, name in enumerate(sorted(descriptors))}
|
|
279
|
+
|
|
280
|
+
results = []
|
|
281
|
+
for record, outcome in shown:
|
|
282
|
+
kind, level = _RESULT[outcome]
|
|
283
|
+
fingerprint = _fingerprint(record)
|
|
284
|
+
result: dict[str, Any] = {
|
|
285
|
+
"ruleId": _rule_id(outcome, record),
|
|
286
|
+
"ruleIndex": rule_index[_rule_id(outcome, record)],
|
|
287
|
+
"kind": kind,
|
|
288
|
+
"level": level,
|
|
289
|
+
"message": {"text": _message(record, outcome)},
|
|
290
|
+
"partialFingerprints": {
|
|
291
|
+
FINGERPRINT: fingerprint,
|
|
292
|
+
GITHUB_FINGERPRINT: f"{fingerprint[:16]}:1",
|
|
293
|
+
},
|
|
294
|
+
"properties": _properties(record),
|
|
295
|
+
}
|
|
296
|
+
if (location := _location(record)) is not None:
|
|
297
|
+
result["locations"] = [location]
|
|
298
|
+
if related := _related(record):
|
|
299
|
+
result["relatedLocations"] = related
|
|
300
|
+
results.append(result)
|
|
301
|
+
|
|
302
|
+
if len(results) > GITHUB_RESULT_LIMIT:
|
|
303
|
+
warnings.warn(
|
|
304
|
+
f"{len(results)} results: GitHub keeps the {GITHUB_RESULT_LIMIT} most severe of a "
|
|
305
|
+
f"run and drops the rest. Leave out --include-matches, or split the manifest.",
|
|
306
|
+
ResultLimitWarning,
|
|
307
|
+
stacklevel=2,
|
|
308
|
+
)
|
|
309
|
+
|
|
310
|
+
notifications = []
|
|
311
|
+
for record, outcome in noted:
|
|
312
|
+
failed = outcome is NormalizedOutcome.ERROR
|
|
313
|
+
notification: dict[str, Any] = {
|
|
314
|
+
"descriptor": {
|
|
315
|
+
"id": _rule_id(outcome, record),
|
|
316
|
+
"index": descriptor_index[_rule_id(outcome, record)],
|
|
317
|
+
},
|
|
318
|
+
# 3.58.6: `error` is a condition that left the results "incorrect or incomplete",
|
|
319
|
+
# and `warning` one where "the analysis might be incomplete but the results that
|
|
320
|
+
# were generated are probably valid".
|
|
321
|
+
"level": "error" if failed else "warning",
|
|
322
|
+
"message": {"text": _message(record, outcome)},
|
|
323
|
+
"properties": _properties(record),
|
|
324
|
+
}
|
|
325
|
+
if (location := _location(record)) is not None:
|
|
326
|
+
notification["locations"] = [location]
|
|
327
|
+
notifications.append(notification)
|
|
328
|
+
|
|
329
|
+
files: dict[str, dict[str, Any]] = {}
|
|
330
|
+
for record in records:
|
|
331
|
+
for read in record.reads:
|
|
332
|
+
if read.path:
|
|
333
|
+
files.setdefault(
|
|
334
|
+
_uri(record, read.path),
|
|
335
|
+
{
|
|
336
|
+
"location": {"uri": _uri(record, read.path)},
|
|
337
|
+
**({"mimeType": read.media_type} if read.media_type else {}),
|
|
338
|
+
**({"hashes": {"sha-256": read.on_disk}} if read.on_disk else {}),
|
|
339
|
+
},
|
|
340
|
+
)
|
|
341
|
+
|
|
342
|
+
counts = collections.Counter(read_outcome(record).value for record in records)
|
|
343
|
+
unchecked = collections.Counter(
|
|
344
|
+
record.native_reason
|
|
345
|
+
for record in records
|
|
346
|
+
if read_outcome(record) is NormalizedOutcome.NOT_CHECKED
|
|
347
|
+
)
|
|
348
|
+
first = records[0] if records else None
|
|
349
|
+
run: dict[str, Any] = {
|
|
350
|
+
"tool": {
|
|
351
|
+
"driver": {
|
|
352
|
+
"name": first.checker.name if first else "repro",
|
|
353
|
+
**({"version": first.checker.version} if first else {}),
|
|
354
|
+
**({"informationUri": first.checker.url} if first and first.checker.url else {}),
|
|
355
|
+
"rules": [rules[name] for name in sorted(rules)],
|
|
356
|
+
"notifications": [descriptors[name] for name in sorted(descriptors)],
|
|
357
|
+
}
|
|
358
|
+
},
|
|
359
|
+
"invocations": [
|
|
360
|
+
{
|
|
361
|
+
# 3.20.21: a notification at level `error` "SHALL mean that the run failed".
|
|
362
|
+
"executionSuccessful": not any(
|
|
363
|
+
outcome is NormalizedOutcome.ERROR for _, outcome in noted
|
|
364
|
+
),
|
|
365
|
+
**(
|
|
366
|
+
{
|
|
367
|
+
"endTimeUtc": first.checked_at.astimezone(datetime.UTC).strftime(
|
|
368
|
+
"%Y-%m-%dT%H:%M:%SZ"
|
|
369
|
+
)
|
|
370
|
+
}
|
|
371
|
+
if first
|
|
372
|
+
else {}
|
|
373
|
+
),
|
|
374
|
+
"toolExecutionNotifications": notifications,
|
|
375
|
+
}
|
|
376
|
+
],
|
|
377
|
+
"artifacts": [files[uri] for uri in sorted(files)],
|
|
378
|
+
"results": results,
|
|
379
|
+
"columnKind": "unicodeCodePoints",
|
|
380
|
+
"properties": {
|
|
381
|
+
"repro/recordVersion": first.record if first else "",
|
|
382
|
+
"repro/assertions": len(records),
|
|
383
|
+
"repro/outcomes": {
|
|
384
|
+
outcome.value: counts[outcome.value] for outcome in NormalizedOutcome
|
|
385
|
+
},
|
|
386
|
+
"repro/notCheckedReasons": dict(sorted(unchecked.items())),
|
|
387
|
+
"repro/matchesIncluded": "yes" if include_matches else "no",
|
|
388
|
+
},
|
|
389
|
+
}
|
|
390
|
+
return json.dumps({"$schema": SCHEMA, "version": VERSION, "runs": [run]}, indent=2) + "\n"
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
__all__ = [
|
|
394
|
+
"FINGERPRINT",
|
|
395
|
+
"GITHUB_FINGERPRINT",
|
|
396
|
+
"GITHUB_RESULT_LIMIT",
|
|
397
|
+
"NOT_CARRIED_NATIVELY",
|
|
398
|
+
"RECORD_PROPERTY",
|
|
399
|
+
"ResultLimitWarning",
|
|
400
|
+
"to_sarif_log",
|
|
401
|
+
]
|
claims/manifest.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Reading a manifest off disk.
|
|
2
|
+
|
|
3
|
+
Validation happens once, here, at the edge. A malformed file names itself and the field at
|
|
4
|
+
fault rather than raising a `KeyError` from inside the engine, because the person who has to
|
|
5
|
+
fix it is looking at the file, not at the traceback.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import pathlib
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
import yaml
|
|
14
|
+
from pydantic import ValidationError
|
|
15
|
+
|
|
16
|
+
from claims.exceptions import ManifestError
|
|
17
|
+
from claims.models import Manifest
|
|
18
|
+
|
|
19
|
+
DEFAULT_NAME = "repro.yaml"
|
|
20
|
+
|
|
21
|
+
#: libyaml's loader where PyYAML was built with it, which every published wheel is. It reads the
|
|
22
|
+
#: same documents as the Python loader and reads a manifest of a thousand claims in a tenth of
|
|
23
|
+
#: the time, which was most of what verifying one took.
|
|
24
|
+
_LOADER = getattr(yaml, "CSafeLoader", yaml.SafeLoader)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load(path: pathlib.Path | str) -> Manifest:
|
|
28
|
+
"""Read one manifest, or raise `ManifestError` naming the file and the field."""
|
|
29
|
+
path = pathlib.Path(path)
|
|
30
|
+
try:
|
|
31
|
+
raw: Any = yaml.load(path.read_text(), Loader=_LOADER) or {}
|
|
32
|
+
except yaml.YAMLError as e:
|
|
33
|
+
raise ManifestError(path, f"not valid YAML: {e}") from e
|
|
34
|
+
except OSError as e:
|
|
35
|
+
raise ManifestError(path, f"could not be read: {e}") from e
|
|
36
|
+
if not isinstance(raw, dict):
|
|
37
|
+
raise ManifestError(
|
|
38
|
+
path, f"expected a mapping at the top level, found {type(raw).__name__}"
|
|
39
|
+
)
|
|
40
|
+
# `path` is set at construction because a Manifest is frozen: where it was read from is
|
|
41
|
+
# part of what it is, and an object that can be repointed afterwards resolves relative
|
|
42
|
+
# artifact paths against whichever directory was assigned last.
|
|
43
|
+
raw["path"] = path
|
|
44
|
+
try:
|
|
45
|
+
return Manifest.model_validate(raw)
|
|
46
|
+
except ValidationError as e:
|
|
47
|
+
first = e.errors()[0]
|
|
48
|
+
where = ".".join(str(x) for x in first["loc"]) or "(top level)"
|
|
49
|
+
raise ManifestError(path, f"{where}: {first['msg']}") from e
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def find(start: pathlib.Path | None = None) -> pathlib.Path | None:
|
|
53
|
+
"""The manifest governing `start`, found by walking up the way git finds `.git`."""
|
|
54
|
+
here = (start or pathlib.Path.cwd()).resolve()
|
|
55
|
+
for directory in [here, *here.parents]:
|
|
56
|
+
candidate = directory / DEFAULT_NAME
|
|
57
|
+
if candidate.is_file():
|
|
58
|
+
return candidate
|
|
59
|
+
return None
|
claims/models.py
ADDED
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""The evidence contract.
|
|
2
|
+
|
|
3
|
+
A quotation in a manuscript and a number in a results file are the same kind of object: a
|
|
4
|
+
typed value asserted to be extractable from a pinned artifact. This package verifies that
|
|
5
|
+
assertion and nothing beyond it.
|
|
6
|
+
|
|
7
|
+
**What a decision evaluates.** Each decision evaluates an *evidence assertion* -- "this passage
|
|
8
|
+
occurs in this artifact", "this artifact holds this value at this location" -- and never the
|
|
9
|
+
truth of the manuscript claim the evidence is offered for. A source can contain a quotation
|
|
10
|
+
verbatim while the quotation fails to support the sentence citing it, and a passage can be
|
|
11
|
+
absent while the claim is true and supported elsewhere. Entailment between evidence and claim
|
|
12
|
+
is a separate question, addressed by the claim-verification literature with model-based
|
|
13
|
+
methods, and out of scope here.
|
|
14
|
+
|
|
15
|
+
That distinction governs the vocabulary. `SUPPORTED` and `REFUTED` name a semantic relation
|
|
16
|
+
between evidence and claim; this package reports `MATCH` and `MISMATCH`, which name what a
|
|
17
|
+
byte comparison found.
|
|
18
|
+
|
|
19
|
+
**Three stages, reported separately.** A check has to execute, extract, and compare, and each
|
|
20
|
+
can fail differently:
|
|
21
|
+
|
|
22
|
+
execution did the check run at all? completed | unavailable | failed
|
|
23
|
+
extraction was the value located? extracted | absent | invalid | not_attempted
|
|
24
|
+
comparison did it match what was claimed? match | mismatch | not_applicable
|
|
25
|
+
|
|
26
|
+
Collapsing these is how a missing PDF extractor becomes a quotation that failed to check out,
|
|
27
|
+
and how a results file that is silent on a value becomes one that contradicts it. A selector
|
|
28
|
+
that resolves to nothing is `completed / absent / not_applicable`: the check ran, the value is
|
|
29
|
+
not there, and no comparison was possible. A missing extractor is
|
|
30
|
+
`unavailable / not_attempted / not_applicable`: nothing ran.
|
|
31
|
+
|
|
32
|
+
**Availability is a claim-level fact, not a check outcome.** Whether a claim offers evidence
|
|
33
|
+
cannot be the result of checking evidence, because there is none to pass to a backend. It
|
|
34
|
+
lives on the claim.
|
|
35
|
+
|
|
36
|
+
The contract is defined in `claims.core`, one module per group. This module re-exports it,
|
|
37
|
+
so `from claims.models import X` continues to name the same objects.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
from claims.core.artifacts import SCHEMA_VERSION, ArtifactRef, Digest, Origin
|
|
43
|
+
from claims.core.checks import CheckedAssertion, CheckRecord, RecordSetAside, RecordState
|
|
44
|
+
from claims.core.claims import Availability, Claim, Registration
|
|
45
|
+
from claims.core.evidence import (
|
|
46
|
+
AggregateEvidence,
|
|
47
|
+
AggregateMember,
|
|
48
|
+
ArrayLocator,
|
|
49
|
+
ComparisonMode,
|
|
50
|
+
Conversion,
|
|
51
|
+
CorrespondenceEvidence,
|
|
52
|
+
CorrespondenceSide,
|
|
53
|
+
Evidence,
|
|
54
|
+
Locator,
|
|
55
|
+
MetricEvidence,
|
|
56
|
+
NumberForm,
|
|
57
|
+
PredicateValue,
|
|
58
|
+
ProseLocator,
|
|
59
|
+
QuoteEvidence,
|
|
60
|
+
SheetLocator,
|
|
61
|
+
SqliteLocator,
|
|
62
|
+
Statistic,
|
|
63
|
+
TableCellEvidence,
|
|
64
|
+
TableLocator,
|
|
65
|
+
TablePositionLocator,
|
|
66
|
+
TreeLocator,
|
|
67
|
+
ValueEvidence,
|
|
68
|
+
ValueLocator,
|
|
69
|
+
)
|
|
70
|
+
from claims.core.manifest import (
|
|
71
|
+
ClaimReading,
|
|
72
|
+
InputReading,
|
|
73
|
+
Manifest,
|
|
74
|
+
Provenance,
|
|
75
|
+
Reading,
|
|
76
|
+
Regeneration,
|
|
77
|
+
RegenerationReason,
|
|
78
|
+
RegenerationRecord,
|
|
79
|
+
RegenerationState,
|
|
80
|
+
RunOutput,
|
|
81
|
+
RunRecord,
|
|
82
|
+
SetAside,
|
|
83
|
+
)
|
|
84
|
+
from claims.core.numbers import (
|
|
85
|
+
NumberRule,
|
|
86
|
+
NumbersReport,
|
|
87
|
+
PrintedNumber,
|
|
88
|
+
SetAsideCount,
|
|
89
|
+
Standing,
|
|
90
|
+
TextInventory,
|
|
91
|
+
)
|
|
92
|
+
from claims.core.outcomes import (
|
|
93
|
+
ComparisonStatus,
|
|
94
|
+
Decision,
|
|
95
|
+
DecisionMember,
|
|
96
|
+
DecisionSide,
|
|
97
|
+
ExecutionStatus,
|
|
98
|
+
ExtractionStatus,
|
|
99
|
+
Ordering,
|
|
100
|
+
OrderingReason,
|
|
101
|
+
Outcome,
|
|
102
|
+
Reason,
|
|
103
|
+
RegistrationAuthority,
|
|
104
|
+
Tie,
|
|
105
|
+
Validity,
|
|
106
|
+
Warning_,
|
|
107
|
+
)
|
|
108
|
+
from claims.core.report import (
|
|
109
|
+
ArtifactNode,
|
|
110
|
+
ArtifactState,
|
|
111
|
+
AssertionAccount,
|
|
112
|
+
AssertionNode,
|
|
113
|
+
ChainNode,
|
|
114
|
+
ClaimAccount,
|
|
115
|
+
ClaimAssessment,
|
|
116
|
+
ClaimNode,
|
|
117
|
+
Edge,
|
|
118
|
+
Environment,
|
|
119
|
+
FileRead,
|
|
120
|
+
Freshness,
|
|
121
|
+
NotEstablished,
|
|
122
|
+
NotEstablishedKind,
|
|
123
|
+
PinDisagreement,
|
|
124
|
+
RegenerationNode,
|
|
125
|
+
ReproductionReport,
|
|
126
|
+
StaleCause,
|
|
127
|
+
StaleFinding,
|
|
128
|
+
StalenessReport,
|
|
129
|
+
VerificationReport,
|
|
130
|
+
Witness,
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
__all__ = [
|
|
134
|
+
"SCHEMA_VERSION",
|
|
135
|
+
"AggregateEvidence",
|
|
136
|
+
"AggregateMember",
|
|
137
|
+
"ArrayLocator",
|
|
138
|
+
"ArtifactNode",
|
|
139
|
+
"ArtifactRef",
|
|
140
|
+
"ArtifactState",
|
|
141
|
+
"AssertionAccount",
|
|
142
|
+
"AssertionNode",
|
|
143
|
+
"Availability",
|
|
144
|
+
"ChainNode",
|
|
145
|
+
"CheckRecord",
|
|
146
|
+
"CheckedAssertion",
|
|
147
|
+
"Claim",
|
|
148
|
+
"ClaimAccount",
|
|
149
|
+
"ClaimAssessment",
|
|
150
|
+
"ClaimNode",
|
|
151
|
+
"ClaimReading",
|
|
152
|
+
"ComparisonMode",
|
|
153
|
+
"ComparisonStatus",
|
|
154
|
+
"Conversion",
|
|
155
|
+
"CorrespondenceEvidence",
|
|
156
|
+
"CorrespondenceSide",
|
|
157
|
+
"Decision",
|
|
158
|
+
"DecisionMember",
|
|
159
|
+
"DecisionSide",
|
|
160
|
+
"Digest",
|
|
161
|
+
"Edge",
|
|
162
|
+
"Environment",
|
|
163
|
+
"Evidence",
|
|
164
|
+
"ExecutionStatus",
|
|
165
|
+
"ExtractionStatus",
|
|
166
|
+
"FileRead",
|
|
167
|
+
"Freshness",
|
|
168
|
+
"InputReading",
|
|
169
|
+
"Locator",
|
|
170
|
+
"Manifest",
|
|
171
|
+
"MetricEvidence",
|
|
172
|
+
"NotEstablished",
|
|
173
|
+
"NotEstablishedKind",
|
|
174
|
+
"NumberForm",
|
|
175
|
+
"NumberRule",
|
|
176
|
+
"NumbersReport",
|
|
177
|
+
"Ordering",
|
|
178
|
+
"OrderingReason",
|
|
179
|
+
"Origin",
|
|
180
|
+
"Outcome",
|
|
181
|
+
"PinDisagreement",
|
|
182
|
+
"PredicateValue",
|
|
183
|
+
"PrintedNumber",
|
|
184
|
+
"ProseLocator",
|
|
185
|
+
"Provenance",
|
|
186
|
+
"QuoteEvidence",
|
|
187
|
+
"Reading",
|
|
188
|
+
"Reason",
|
|
189
|
+
"RecordSetAside",
|
|
190
|
+
"RecordState",
|
|
191
|
+
"Regeneration",
|
|
192
|
+
"RegenerationNode",
|
|
193
|
+
"RegenerationReason",
|
|
194
|
+
"RegenerationRecord",
|
|
195
|
+
"RegenerationState",
|
|
196
|
+
"Registration",
|
|
197
|
+
"RegistrationAuthority",
|
|
198
|
+
"ReproductionReport",
|
|
199
|
+
"RunOutput",
|
|
200
|
+
"RunRecord",
|
|
201
|
+
"SetAside",
|
|
202
|
+
"SetAsideCount",
|
|
203
|
+
"SheetLocator",
|
|
204
|
+
"SqliteLocator",
|
|
205
|
+
"StaleCause",
|
|
206
|
+
"StaleFinding",
|
|
207
|
+
"StalenessReport",
|
|
208
|
+
"Standing",
|
|
209
|
+
"Statistic",
|
|
210
|
+
"TableCellEvidence",
|
|
211
|
+
"TableLocator",
|
|
212
|
+
"TablePositionLocator",
|
|
213
|
+
"TextInventory",
|
|
214
|
+
"Tie",
|
|
215
|
+
"TreeLocator",
|
|
216
|
+
"Validity",
|
|
217
|
+
"ValueEvidence",
|
|
218
|
+
"ValueLocator",
|
|
219
|
+
"VerificationReport",
|
|
220
|
+
"Warning_",
|
|
221
|
+
"Witness",
|
|
222
|
+
]
|