claims-cli 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. claims/__init__.py +132 -0
  2. claims/__main__.py +5 -0
  3. claims/adapters/__init__.py +6 -0
  4. claims/adapters/array.py +212 -0
  5. claims/adapters/base.py +75 -0
  6. claims/adapters/cells.py +94 -0
  7. claims/adapters/columnar.py +188 -0
  8. claims/adapters/prose.py +261 -0
  9. claims/adapters/reads.py +47 -0
  10. claims/adapters/rows.py +116 -0
  11. claims/adapters/sheet.py +149 -0
  12. claims/adapters/sqlite.py +73 -0
  13. claims/adapters/table.py +135 -0
  14. claims/adapters/tree.py +163 -0
  15. claims/checks.py +511 -0
  16. claims/cli.py +1103 -0
  17. claims/core/__init__.py +9 -0
  18. claims/core/artifacts.py +121 -0
  19. claims/core/checks.py +132 -0
  20. claims/core/claims.py +100 -0
  21. claims/core/evidence.py +708 -0
  22. claims/core/manifest.py +349 -0
  23. claims/core/numbers.py +146 -0
  24. claims/core/outcomes.py +410 -0
  25. claims/core/report.py +503 -0
  26. claims/corpus.py +344 -0
  27. claims/exceptions.py +106 -0
  28. claims/export/__init__.py +23 -0
  29. claims/export/rocrate.py +787 -0
  30. claims/export/sarif.py +401 -0
  31. claims/manifest.py +59 -0
  32. claims/models.py +222 -0
  33. claims/numbers.py +453 -0
  34. claims/origins.py +303 -0
  35. claims/policy.py +369 -0
  36. claims/provenance.py +42 -0
  37. claims/record.py +1099 -0
  38. claims/regression.py +171 -0
  39. claims/renderers/__init__.py +10 -0
  40. claims/renderers/sarif.py +294 -0
  41. claims/reruns.py +111 -0
  42. claims/resolve.py +116 -0
  43. claims/stale.py +333 -0
  44. claims/starter.py +432 -0
  45. claims/toolchain.py +120 -0
  46. claims/verify.py +1371 -0
  47. claims/why.py +398 -0
  48. claims_cli-0.6.0.dist-info/METADATA +139 -0
  49. claims_cli-0.6.0.dist-info/RECORD +51 -0
  50. claims_cli-0.6.0.dist-info/WHEEL +4 -0
  51. claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
claims/numbers.py ADDED
@@ -0,0 +1,453 @@
1
+ """Every number a manuscript prints, and whether a declared assertion accounts for it.
2
+
3
+ `repro verify` reports on the claims a manifest declares, and the author chose which to
4
+ declare: forty verified of forty says nothing of the numbers no claim was written for. This
5
+ reads the manuscript's text and counts every number in it, so the denominator is the text's.
6
+ Executes nothing and writes nothing.
7
+
8
+ **Which text.** An artifact a claim reads its own sentence from: one a `quote` reads in a claim
9
+ that also offers an assertion of another kind, or one a prose address reads. A `quote` standing
10
+ alone in its claim is a quotation from a source, and that source is not inventoried.
11
+
12
+ **What a number is.** The text is the one a quotation resolves against, `passage_fold` over
13
+ what `citations` extracts, and a number is what `claims.adapters.prose.numerals` reads in it:
14
+ digits with an optional sign, grouping commas, a decimal point and an exponent. `2,364` is one
15
+ number and `12,13` is two; `10-12` is two; a percent sign is kept with its number and is not
16
+ part of its value. A cardinal written as one word is a number with the value the prose
17
+ adapter reads from it. Nothing is dropped: digits that are not a quantity are counted under
18
+ the rule that says what they are, `claims.models.NumberRule`.
19
+
20
+ **Bound.** One occurrence is bound where it lies inside a place the passage of a `quote`
21
+ occurs and an assertion of the same claim carries a `reported` value that is this number,
22
+ digit for digit at the printed precision, or where it lies in the value a prose address
23
+ selects. A `reported` value is what the manuscript prints, so one declared with a `conversion`
24
+ binds the printed number and not the stored one, and an `aggregate` also binds its `count`. The same digits reported by some other claim bind nothing: that occurrence is
25
+ unbound and marked `reported_elsewhere`, where the value has two digits or a decimal point; a
26
+ single digit matching some reported value is counted apart and not marked.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import bisect
32
+ import decimal
33
+ import re
34
+ from collections.abc import Iterator
35
+ from typing import NamedTuple
36
+
37
+ from citations.exceptions import SourceUnreadableError
38
+ from citations.verify import extract_uncached, passage_fold
39
+
40
+ from claims.adapters.prose import CARDINALS, located, numerals
41
+ from claims.exceptions import ArtifactMissingError, ArtifactUnreadableError, ReproError
42
+ from claims.models import (
43
+ AggregateEvidence,
44
+ ArtifactRef,
45
+ Claim,
46
+ CorrespondenceEvidence,
47
+ Digest,
48
+ Manifest,
49
+ NumberForm,
50
+ NumberRule,
51
+ NumbersReport,
52
+ PrintedNumber,
53
+ ProseLocator,
54
+ QuoteEvidence,
55
+ SetAsideCount,
56
+ Standing,
57
+ TextInventory,
58
+ ValueEvidence,
59
+ )
60
+
61
+ #: Words shown on either side of a number.
62
+ CONTEXT_WORDS = 4
63
+
64
+ #: `2,364` and `1,234,567.5`: commas that group digits in threes. Any other comma between
65
+ #: digits separates two numbers.
66
+ _GROUPED = re.compile(r"[+-]?\d{1,3}(?:,\d{3})+(?:\.\d+)?(?:e[+-]?\d+)?")
67
+ _WORD = re.compile(r"[a-z]+(?:-[a-z]+)*")
68
+ _YEAR = re.compile(r"(?:18|19|20)\d\d")
69
+ _DATE = re.compile(r"(?<![\d-])\d{4}-\d{2}-\d{2}(?![\d-])")
70
+ _DOI = re.compile(r"(?<![\d.])10\.\d{4,9}/")
71
+ #: A standard, a preprint or a book named by its series and number: `rfc 6901`, `iso 8601`.
72
+ _DOCUMENT = re.compile(
73
+ r"(?<![a-z])(?:rfc|iso(?:/iec)?|iec|ieee|ansi|ecma|fips|pep|arxiv|doi|isbn|issn|pmid|pmcid)"
74
+ r"(?:: ?|[ -])?(\d+(?:[.:/-]\d+)*)"
75
+ )
76
+ #: An em or en dash set against the digit after it. `passage_fold` writes both as a hyphen, so
77
+ #: the text is folded a second time with these replaced by a character it leaves alone, to
78
+ #: tell the dash that ends a clause from the hyphen that joins a name.
79
+ _DASH_BEFORE_DIGIT = re.compile("[\u2014\u2013](?=\\d)")
80
+ _BAR = "\u2015"
81
+ _CITATION = re.compile(r"\[[1-9]\d*(?:\s?[,;-]\s?[1-9]\d*)*\]")
82
+ _LIST_MARKER = re.compile(r"\(\d{1,2}\)")
83
+ _LABEL = re.compile(
84
+ r"(?<![a-z])(?:sections?|sec\.|§|figures?|figs?\.?|tables?|tab\.|equations?|eqs?\.?|eqn\.?"
85
+ r"|appendix|appendices|chapters?|pages?|pp?\.|algorithms?|listings?|theorems?|lemmas?"
86
+ r"|propositions?|corollary|corollaries|definitions?|footnotes?|steps?|lines?)"
87
+ r" ?\(?(\d+(?:\.\d+)*(?:-\d+(?:\.\d+)*)?)"
88
+ )
89
+ #: Letters and then a digit, immediately after a number: the digits open a longer name.
90
+ _MORE_DIGITS = re.compile(r"[a-z]+\d")
91
+ _ORDINAL = re.compile(r"(?:st|nd|rd|th)(?![a-z0-9])")
92
+
93
+ Span = tuple[int, int]
94
+
95
+
96
+ class UnknownArtifactError(ReproError):
97
+ """A text was named that the manifest does not declare."""
98
+
99
+ def __init__(self, unknown: tuple[str, ...], declared: tuple[str, ...]) -> None:
100
+ self.unknown = unknown
101
+ self.declared = declared
102
+ super().__init__(
103
+ f"the manifest declares no artifact {', '.join(repr(u) for u in unknown)}; "
104
+ f"declared: {', '.join(declared) or 'none'}"
105
+ )
106
+
107
+
108
+ class NoTextError(ReproError):
109
+ """No artifact in the manifest is read as the manuscript's text, and none was named."""
110
+
111
+ def __init__(self) -> None:
112
+ super().__init__(
113
+ "no claim reads its sentence from a text: no `quote` stands beside another "
114
+ "assertion and no value has a prose address. Name the text with --text <artifact-id>."
115
+ )
116
+
117
+
118
+ class _Token(NamedTuple):
119
+ start: int
120
+ end: int
121
+ value: str | None
122
+ named: bool = False
123
+
124
+
125
+ def _prose(claim: Claim) -> Iterator[tuple[str, ProseLocator]]:
126
+ """Each prose address a claim reads a value at, with the artifact it is in."""
127
+ for evidence in claim.evidence:
128
+ if isinstance(evidence, ValueEvidence) and isinstance(evidence.locator, ProseLocator):
129
+ yield evidence.artifact, evidence.locator
130
+ elif isinstance(evidence, CorrespondenceEvidence):
131
+ for side in evidence.sides:
132
+ if isinstance(side.locator, ProseLocator):
133
+ yield side.artifact, side.locator
134
+
135
+
136
+ def _quotes(claim: Claim) -> tuple[QuoteEvidence, ...]:
137
+ """The quotations a claim reads its own sentence with: none where it offers nothing else."""
138
+ quotes = tuple(e for e in claim.evidence if isinstance(e, QuoteEvidence))
139
+ return quotes if len(quotes) < len(claim.evidence) else ()
140
+
141
+
142
+ def texts(manifest: Manifest) -> tuple[str, ...]:
143
+ """The artifacts a claim reads its own sentence from, in the order declared."""
144
+ read = {
145
+ artifact
146
+ for claim in manifest.claims
147
+ for artifact in (
148
+ *(quote.artifact for quote in _quotes(claim)),
149
+ *(artifact for artifact, _ in _prose(claim)),
150
+ )
151
+ }
152
+ return tuple(artifact.id for artifact in manifest.artifacts if artifact.id in read)
153
+
154
+
155
+ def _read(value: str | None) -> decimal.DecimalTuple | None:
156
+ """A printed value as its sign, digits and exponent, so `0.5` and `0.50` differ."""
157
+ try:
158
+ return decimal.Decimal(value).as_tuple() if value is not None else None
159
+ except decimal.InvalidOperation:
160
+ return None
161
+
162
+
163
+ def _line_starts(raw: str, folded: str) -> list[int] | None:
164
+ """Where each line of the extracted text starts in the normalized text.
165
+
166
+ `passage_fold` joins the lines with one space, and joins a word a hyphen splits across a
167
+ line break with nothing. This walks the lines under that rule, and returns None where what it
168
+ walked is not the normalized text, so no line number is ever a guess.
169
+ """
170
+ pieces = [passage_fold(line) for line in raw.split("\n")]
171
+ starts, at = [], 0
172
+ for number, piece in enumerate(pieces):
173
+ starts.append(at)
174
+ if not piece:
175
+ continue
176
+ after = next((later for later in pieces[number + 1 :] if later), "")
177
+ # A word split across the break: a letter before the hyphen and one opening the next
178
+ # line that holds anything.
179
+ broken = piece.endswith("-") and piece[-2:-1].isalpha() and after[:1].isalpha()
180
+ if broken:
181
+ piece = piece[:-1]
182
+ if not folded.startswith(piece.rstrip(), at):
183
+ return None
184
+ at += len(piece) + (0 if broken else 1)
185
+ return starts
186
+
187
+
188
+ def _dashes(raw: str, folded: str) -> frozenset[int]:
189
+ """Where the normalized text holds a hyphen that was an em or en dash before a digit."""
190
+ marked = passage_fold(_DASH_BEFORE_DIGIT.sub(_BAR, raw))
191
+ if len(marked) != len(folded):
192
+ return frozenset()
193
+ return frozenset(
194
+ found.start() for found in re.finditer(_BAR, marked) if folded[found.start()] == "-"
195
+ )
196
+
197
+
198
+ def _tokens(folded: str, dashes: frozenset[int]) -> list[_Token]:
199
+ """Every number in the text, in order: digits as the prose adapter reads them, and each
200
+ cardinal written as one word. A dash before the digits is punctuation and not a sign: an
201
+ em or en dash, or the last of a run of hyphens."""
202
+ found = []
203
+ for first, end, named in numerals(folded):
204
+ if folded[first] == "-" and (first in dashes or folded[first - 1 : first] == "-"):
205
+ first += 1
206
+ whole = folded[first:end]
207
+ if named:
208
+ found.append(_Token(first, end, None, True))
209
+ elif "," not in whole or _GROUPED.fullmatch(whole):
210
+ found.append(_Token(first, end, whole.replace(",", "")))
211
+ else:
212
+ at = first
213
+ for piece in whole.split(","):
214
+ found.append(_Token(at, at + len(piece), piece))
215
+ at += len(piece) + 1
216
+ for word in _WORD.finditer(folded):
217
+ before, after = folded[word.start() - 1 : word.start()], folded[word.end() : word.end() + 1]
218
+ if word.group() in CARDINALS and not before.isalnum() and not after.isalnum():
219
+ found.append(_Token(word.start(), word.end(), str(CARDINALS[word.group()])))
220
+ return sorted(found, key=lambda token: token.start)
221
+
222
+
223
+ def _run(folded: str, at: int) -> str:
224
+ """The run of non-whitespace characters `at` is in."""
225
+ lo = folded.rfind(" ", 0, at) + 1
226
+ hi = folded.find(" ", at)
227
+ return folded[lo : len(folded) if hi == -1 else hi]
228
+
229
+
230
+ def _joined(folded: str, start: int, dashes: frozenset[int]) -> bool:
231
+ """Whether a hyphen or underscore joins the digits to a part before them that holds a
232
+ letter: `gpt-4`, and `9750` in `i7-9750h`."""
233
+ at = start - 1
234
+ if folded[at:start] not in ("-", "_") or at in dashes:
235
+ return False
236
+ lo = at
237
+ while lo > 0 and folded[lo - 1].isalnum():
238
+ lo -= 1
239
+ return any(character.isalpha() for character in folded[lo:at])
240
+
241
+
242
+ def _rule(
243
+ folded: str, token: _Token, marked: dict[NumberRule, list[Span]], dashes: frozenset[int]
244
+ ) -> NumberRule | None:
245
+ """The first rule that reads this token as something other than a quantity."""
246
+ start, end = token.start, token.end
247
+ text = folded[start:end]
248
+
249
+ def within(rule: NumberRule) -> bool:
250
+ return any(lo <= start and end <= hi for lo, hi in marked[rule])
251
+
252
+ run = _run(folded, start)
253
+ if "://" in run or run.startswith("www."):
254
+ return NumberRule.LINK
255
+ if within(NumberRule.DATE):
256
+ return NumberRule.DATE
257
+ if within(NumberRule.DOCUMENT) or run.startswith(("doi:", "arxiv:")) or _DOI.search(run):
258
+ return NumberRule.DOCUMENT
259
+ if token.named or _joined(folded, start, dashes) or _MORE_DIGITS.match(folded, end):
260
+ return NumberRule.NAME
261
+ if text.count(".") > 1:
262
+ return NumberRule.VERSION_OR_SECTION
263
+ for rule in (NumberRule.CITATION, NumberRule.LABEL, NumberRule.LIST_MARKER):
264
+ if within(rule):
265
+ return rule
266
+ if text.isdigit() and _ORDINAL.match(folded, end):
267
+ return NumberRule.ORDINAL
268
+ if _YEAR.fullmatch(text) and folded[end : end + 1] != "%":
269
+ return NumberRule.YEAR
270
+ if text == "one":
271
+ return NumberRule.THE_WORD_ONE
272
+ return None
273
+
274
+
275
+ def _marked(folded: str) -> dict[NumberRule, list[Span]]:
276
+ """The stretches of the text the rules that read their surroundings apply to."""
277
+ return {
278
+ NumberRule.DATE: [found.span() for found in _DATE.finditer(folded)],
279
+ NumberRule.DOCUMENT: [found.span(1) for found in _DOCUMENT.finditer(folded)],
280
+ NumberRule.CITATION: [found.span() for found in _CITATION.finditer(folded)],
281
+ NumberRule.LABEL: [found.span(1) for found in _LABEL.finditer(folded)],
282
+ NumberRule.LIST_MARKER: [found.span() for found in _LIST_MARKER.finditer(folded)],
283
+ }
284
+
285
+
286
+ def _places(passage: str, folded: str) -> list[Span]:
287
+ """Every place a passage occurs in the text, both normalized as a quotation is."""
288
+ wanted = passage_fold(passage)
289
+ found, at = [], folded.find(wanted) if wanted else -1
290
+ while at != -1:
291
+ found.append((at, at + len(wanted)))
292
+ at = folded.find(wanted, at + 1)
293
+ return found
294
+
295
+
296
+ def _context(folded: str, start: int, end: int) -> str:
297
+ """The run a number is in and `CONTEXT_WORDS` runs on either side."""
298
+ lo, hi = start, end - 1
299
+ for _ in range(CONTEXT_WORDS + 1):
300
+ lo = folded.rfind(" ", 0, lo)
301
+ if lo == -1:
302
+ break
303
+ for _ in range(CONTEXT_WORDS + 1):
304
+ hi = folded.find(" ", hi + 1)
305
+ if hi == -1:
306
+ break
307
+ return folded[lo + 1 : len(folded) if hi == -1 else hi]
308
+
309
+
310
+ def _reported(claim: Claim) -> set[decimal.DecimalTuple]:
311
+ """Each value a claim's assertions say the manuscript prints.
312
+
313
+ An `aggregate` says two: the statistic it reports, and how many members it is over, which
314
+ the manifest declares as `count` and refuses to load where the list is another length.
315
+ """
316
+ stated = [getattr(evidence, "reported", None) for evidence in claim.evidence]
317
+ stated += [str(e.count) for e in claim.evidence if isinstance(e, AggregateEvidence)]
318
+ return {read for value in stated if (read := _read(value)) is not None}
319
+
320
+
321
+ def _inventory(
322
+ manifest: Manifest, artifact: ArtifactRef, reported: set[decimal.DecimalTuple]
323
+ ) -> tuple[TextInventory, list[SetAsideCount]]:
324
+ """One text: every number in it, with where it stands against the manifest."""
325
+ artifact_id, path = artifact.id, manifest.resolve(artifact)
326
+ if not path.is_file():
327
+ raise ArtifactMissingError(artifact_id, path)
328
+ try:
329
+ raw = extract_uncached(path)
330
+ except SourceUnreadableError as e:
331
+ raise ArtifactUnreadableError(path, e.detail) from e
332
+ folded = passage_fold(raw)
333
+ starts = _line_starts(raw, folded)
334
+ dashes = _dashes(raw, folded)
335
+
336
+ # What binds: the claim, the stretch of text, and the values it binds there. None is the
337
+ # value a prose address selects, which the assertion reads whatever it holds.
338
+ binding: list[tuple[str, Span, set[decimal.DecimalTuple] | None]] = []
339
+ for claim in manifest.claims:
340
+ values = _reported(claim)
341
+ for quote in _quotes(claim):
342
+ if quote.artifact == artifact_id and values:
343
+ binding += [(claim.id, place, values) for place in _places(quote.text, folded)]
344
+ for named, locator in _prose(claim):
345
+ if named == artifact_id:
346
+ before, after = passage_fold(locator.before), passage_fold(locator.after)
347
+ binding += [(claim.id, place, None) for place in located(folded, before, after)[0]]
348
+
349
+ entries = [entry for entry in manifest.set_aside if entry.artifact == artifact_id]
350
+ aside = [_places(entry.text, folded) for entry in entries]
351
+ counted = [0] * len(entries)
352
+ marked = _marked(folded)
353
+
354
+ numbers = []
355
+ for token in _tokens(folded, dashes):
356
+ start, end = token.start, token.end
357
+ value = _read(token.value)
358
+ percent = folded[end : end + 1] == "%"
359
+ claims = tuple(
360
+ dict.fromkeys(
361
+ claim_id
362
+ for claim_id, (lo, hi), values in binding
363
+ if lo <= start and end <= hi and (values is None or value in values)
364
+ )
365
+ )
366
+ held = next(
367
+ (
368
+ index
369
+ for index, places in enumerate(aside)
370
+ if any(lo <= start and end <= hi for lo, hi in places)
371
+ ),
372
+ None,
373
+ )
374
+ rule, reason = None, ""
375
+ if claims:
376
+ standing = Standing.BOUND
377
+ elif held is not None:
378
+ standing, reason = Standing.SET_ASIDE_WITH_A_REASON, entries[held].reason
379
+ counted[held] += 1
380
+ elif rule := _rule(folded, token, marked, dashes):
381
+ standing = Standing.SET_ASIDE_BY_RULE
382
+ else:
383
+ standing = Standing.UNBOUND
384
+ # A name, a link, a document's identifier and a version hold digits and no value.
385
+ valued = rule not in (
386
+ NumberRule.NAME,
387
+ NumberRule.LINK,
388
+ NumberRule.DOCUMENT,
389
+ NumberRule.VERSION_OR_SECTION,
390
+ )
391
+ elsewhere = standing is Standing.UNBOUND and value in reported
392
+ # One digit with no decimal point matches some reported value in most manifests.
393
+ telling = value is not None and (len(value.digits) > 1 or value.exponent != 0)
394
+ numbers.append(
395
+ PrintedNumber(
396
+ text=folded[start : end + 1 if percent else end],
397
+ value=token.value if value is not None and valued else None,
398
+ offset=start,
399
+ line=bisect.bisect_right(starts, start) if starts else None,
400
+ context=_context(folded, start, end),
401
+ standing=standing,
402
+ rule=rule,
403
+ claims=claims,
404
+ reason=reason,
405
+ form=NumberForm.CARDINAL_WORD if folded[start].isalpha() else NumberForm.DECIMAL,
406
+ reported_elsewhere=elsewhere and telling,
407
+ reported_elsewhere_one_digit=elsewhere and not telling,
408
+ )
409
+ )
410
+
411
+ return (
412
+ TextInventory(artifact=artifact_id, path=artifact.path.as_posix(), numbers=tuple(numbers)),
413
+ [
414
+ SetAsideCount(artifact=artifact_id, text=entry.text, reason=entry.reason, numbers=n)
415
+ for entry, n in zip(entries, counted, strict=True)
416
+ ],
417
+ )
418
+
419
+
420
+ def numbers(manifest: Manifest, only: tuple[str, ...] = ()) -> NumbersReport:
421
+ """Count every number the manuscript's text prints and say where each stands.
422
+
423
+ `only` names the text artifacts to read in place of the ones `texts` finds. Executes
424
+ nothing and writes nothing.
425
+ """
426
+ declared = {artifact.id: artifact for artifact in manifest.artifacts}
427
+ if unknown := tuple(name for name in only if name not in declared):
428
+ raise UnknownArtifactError(unknown, tuple(declared))
429
+ read = tuple(dict.fromkeys(only)) or texts(manifest)
430
+ if not read:
431
+ raise NoTextError
432
+
433
+ reported = {value for claim in manifest.claims for value in _reported(claim)}
434
+ inventories, aside = [], []
435
+ for artifact_id in read:
436
+ inventory, counted = _inventory(manifest, declared[artifact_id], reported)
437
+ inventories.append(inventory)
438
+ aside += counted
439
+ # An entry naming a text that was not read accounts for nothing, and is listed as that.
440
+ aside += [
441
+ SetAsideCount(artifact=entry.artifact, text=entry.text, reason=entry.reason, numbers=0)
442
+ for entry in manifest.set_aside
443
+ if entry.artifact not in read
444
+ ]
445
+ return NumbersReport(
446
+ project=manifest.project,
447
+ manifest_digest=Digest.of_text(manifest.model_dump_json()).value,
448
+ texts=tuple(inventories),
449
+ set_aside=tuple(aside),
450
+ )
451
+
452
+
453
+ __all__ = ["CONTEXT_WORDS", "NoTextError", "UnknownArtifactError", "numbers", "texts"]