claims-cli 0.6.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- claims/__init__.py +132 -0
- claims/__main__.py +5 -0
- claims/adapters/__init__.py +6 -0
- claims/adapters/array.py +212 -0
- claims/adapters/base.py +75 -0
- claims/adapters/cells.py +94 -0
- claims/adapters/columnar.py +188 -0
- claims/adapters/prose.py +261 -0
- claims/adapters/reads.py +47 -0
- claims/adapters/rows.py +116 -0
- claims/adapters/sheet.py +149 -0
- claims/adapters/sqlite.py +73 -0
- claims/adapters/table.py +135 -0
- claims/adapters/tree.py +163 -0
- claims/checks.py +511 -0
- claims/cli.py +1103 -0
- claims/core/__init__.py +9 -0
- claims/core/artifacts.py +121 -0
- claims/core/checks.py +132 -0
- claims/core/claims.py +100 -0
- claims/core/evidence.py +708 -0
- claims/core/manifest.py +349 -0
- claims/core/numbers.py +146 -0
- claims/core/outcomes.py +410 -0
- claims/core/report.py +503 -0
- claims/corpus.py +344 -0
- claims/exceptions.py +106 -0
- claims/export/__init__.py +23 -0
- claims/export/rocrate.py +787 -0
- claims/export/sarif.py +401 -0
- claims/manifest.py +59 -0
- claims/models.py +222 -0
- claims/numbers.py +453 -0
- claims/origins.py +303 -0
- claims/policy.py +369 -0
- claims/provenance.py +42 -0
- claims/record.py +1099 -0
- claims/regression.py +171 -0
- claims/renderers/__init__.py +10 -0
- claims/renderers/sarif.py +294 -0
- claims/reruns.py +111 -0
- claims/resolve.py +116 -0
- claims/stale.py +333 -0
- claims/starter.py +432 -0
- claims/toolchain.py +120 -0
- claims/verify.py +1371 -0
- claims/why.py +398 -0
- claims_cli-0.6.0.dist-info/METADATA +139 -0
- claims_cli-0.6.0.dist-info/RECORD +51 -0
- claims_cli-0.6.0.dist-info/WHEEL +4 -0
- claims_cli-0.6.0.dist-info/entry_points.txt +2 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Binary formats that hold one table: Parquet, Feather, Arrow, Stata, SPSS, and R's `.rds`.
|
|
2
|
+
|
|
3
|
+
Each is read into the header and rows a delimited file gives, with every cell rendered by
|
|
4
|
+
`claims.adapters.cells`, and resolved by `claims.adapters.rows`. A table locator therefore
|
|
5
|
+
addresses `results.parquet` exactly as it addresses `results.csv`, and the same table in either
|
|
6
|
+
format refuses and resolves the same manifests.
|
|
7
|
+
|
|
8
|
+
Every reader is an optional extra, and a missing one raises `BackendUnavailableError` naming
|
|
9
|
+
it, which the engine reports as `unchecked`:
|
|
10
|
+
|
|
11
|
+
.parquet .feather .arrow pyarrow reproducible-science[parquet]
|
|
12
|
+
.dta .sav .zsav pyreadstat reproducible-science[stata-spss]
|
|
13
|
+
.rds rdata reproducible-science[rds]
|
|
14
|
+
|
|
15
|
+
**No reader here executes the file.** A pickle is a program, and an `.rds` read by R can be
|
|
16
|
+
one: R's deserializer evaluated a promise embedded in the stream when the restored object was
|
|
17
|
+
first used (CVE-2024-27322). `rdata` is a parser of R's serialization format written in Python.
|
|
18
|
+
It builds the objects the bytes describe and evaluates nothing, and no R process is started.
|
|
19
|
+
`pyreadr` would also avoid R, and is not used because it is AGPL.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import lzma
|
|
25
|
+
import pathlib
|
|
26
|
+
import zlib
|
|
27
|
+
|
|
28
|
+
from claims.adapters.cells import Cell, Container, render, shortest_single
|
|
29
|
+
from claims.exceptions import ArtifactUnreadableError, BackendUnavailableError
|
|
30
|
+
from claims.toolchain import DISTRIBUTION
|
|
31
|
+
|
|
32
|
+
try: # optional: `pip install "reproducible-science[parquet]"`
|
|
33
|
+
import pyarrow
|
|
34
|
+
import pyarrow.compute
|
|
35
|
+
import pyarrow.feather
|
|
36
|
+
import pyarrow.parquet
|
|
37
|
+
except ImportError: # pragma: no cover - exercised by the extras matrix, not the suite
|
|
38
|
+
pyarrow = None
|
|
39
|
+
|
|
40
|
+
try: # optional: `pip install "reproducible-science[stata-spss]"`
|
|
41
|
+
import pyreadstat
|
|
42
|
+
except ImportError: # pragma: no cover
|
|
43
|
+
pyreadstat = None
|
|
44
|
+
|
|
45
|
+
try: # optional: `pip install "reproducible-science[rds]"`
|
|
46
|
+
import rdata
|
|
47
|
+
except ImportError: # pragma: no cover
|
|
48
|
+
rdata = None
|
|
49
|
+
|
|
50
|
+
_ARROW_SUFFIXES = {".parquet", ".feather", ".arrow"}
|
|
51
|
+
_READSTAT_SUFFIXES = {".dta", ".sav", ".zsav"}
|
|
52
|
+
COLUMNAR_SUFFIXES = _ARROW_SUFFIXES | _READSTAT_SUFFIXES | {".rds"}
|
|
53
|
+
|
|
54
|
+
Frame = tuple[list[str], list[dict[str, Cell]]]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def read_columnar(path: pathlib.Path, wanted: set[str]) -> Frame:
|
|
58
|
+
"""The file's header, and its rows restricted to the `wanted` columns it has.
|
|
59
|
+
|
|
60
|
+
Only the columns a locator names are read. A results file can hold thousands of columns,
|
|
61
|
+
and the header alone settles whether the locator names columns the file has.
|
|
62
|
+
"""
|
|
63
|
+
suffix = path.suffix.lower()
|
|
64
|
+
if suffix in _ARROW_SUFFIXES:
|
|
65
|
+
return _arrow(path, wanted)
|
|
66
|
+
if suffix in _READSTAT_SUFFIXES:
|
|
67
|
+
return _readstat(path, wanted)
|
|
68
|
+
return _rds(path, wanted)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _rows(header: list[str], columns: dict[str, list[Cell]], n: int) -> Frame:
|
|
72
|
+
return header, [{name: cells[i] for name, cells in columns.items()} for i in range(n)]
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _addressable(header: list[str], wanted: set[str]) -> list[str]:
|
|
76
|
+
"""The wanted columns that occur once. A repeated name is refused before any row is read."""
|
|
77
|
+
return [name for name in header if name in wanted and header.count(name) == 1]
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# ------------------------------------------------------------------------------------ arrow
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
_NO_PYARROW = f'pyarrow is not installed -- pip install "{DISTRIBUTION}[parquet]"'
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _arrow(path: pathlib.Path, wanted: set[str]) -> Frame:
|
|
87
|
+
if pyarrow is None:
|
|
88
|
+
raise BackendUnavailableError("table", _NO_PYARROW)
|
|
89
|
+
try:
|
|
90
|
+
if path.suffix.lower() == ".parquet":
|
|
91
|
+
header = list(pyarrow.parquet.read_schema(path).names)
|
|
92
|
+
table = pyarrow.parquet.read_table(path, columns=_addressable(header, wanted))
|
|
93
|
+
else:
|
|
94
|
+
table = pyarrow.feather.read_table(path)
|
|
95
|
+
header = list(table.column_names)
|
|
96
|
+
except (pyarrow.ArrowException, OSError) as e:
|
|
97
|
+
raise ArtifactUnreadableError(path, f"not a readable {path.suffix} file: {e}") from e
|
|
98
|
+
columns = {name: _arrow_cells(table.column(name)) for name in _addressable(header, wanted)}
|
|
99
|
+
return _rows(header, columns, table.num_rows)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _arrow_cells(column) -> list[Cell]:
|
|
103
|
+
"""A column as text, by Arrow's own cast.
|
|
104
|
+
|
|
105
|
+
Arrow renders a float as the shortest decimal that round-trips at the column's own width,
|
|
106
|
+
so a `float32` 0.1 is `0.1` rather than the 0.10000000149011612 a conversion to a Python
|
|
107
|
+
float produces. A nested column has no text form, and each of its cells is a container.
|
|
108
|
+
"""
|
|
109
|
+
if pyarrow is None: # pragma: no cover - `_arrow` has already raised
|
|
110
|
+
raise BackendUnavailableError("table", _NO_PYARROW)
|
|
111
|
+
kind = column.type
|
|
112
|
+
if pyarrow.types.is_nested(kind) or pyarrow.types.is_union(kind):
|
|
113
|
+
return [
|
|
114
|
+
None if v is None else Container(holds=f"an Arrow {kind}") for v in column.to_pylist()
|
|
115
|
+
]
|
|
116
|
+
try:
|
|
117
|
+
text = pyarrow.compute.cast(column, pyarrow.string()).to_pylist()
|
|
118
|
+
except pyarrow.ArrowException:
|
|
119
|
+
# Binary that is not UTF-8, an extension type: there is no text to compare.
|
|
120
|
+
return [
|
|
121
|
+
None if v is None else Container(holds=f"an Arrow {kind}") for v in column.to_pylist()
|
|
122
|
+
]
|
|
123
|
+
return [render(v) for v in text]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
# --------------------------------------------------------------------------------- readstat
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _readstat(path: pathlib.Path, wanted: set[str]) -> Frame:
|
|
130
|
+
if pyreadstat is None:
|
|
131
|
+
raise BackendUnavailableError(
|
|
132
|
+
"table",
|
|
133
|
+
f'pyreadstat is not installed -- pip install "{DISTRIBUTION}[stata-spss]"',
|
|
134
|
+
)
|
|
135
|
+
read = pyreadstat.read_dta if path.suffix.lower() == ".dta" else pyreadstat.read_sav
|
|
136
|
+
try:
|
|
137
|
+
_, meta = read(str(path), metadataonly=True, output_format="dict")
|
|
138
|
+
header = list(meta.column_names)
|
|
139
|
+
usecols = _addressable(header, wanted)
|
|
140
|
+
data, meta = read(str(path), usecols=usecols, output_format="dict")
|
|
141
|
+
except (pyreadstat.ReadstatError, pyreadstat.PyreadstatError, OSError) as e:
|
|
142
|
+
raise ArtifactUnreadableError(path, f"not a readable {path.suffix} file: {e}") from e
|
|
143
|
+
# Value labels are not applied: a labelled column is matched and read by its stored code,
|
|
144
|
+
# which is what the file holds. Applying them would make a predicate depend on a label set
|
|
145
|
+
# the analyst can change without touching a single value.
|
|
146
|
+
widths = meta.readstat_variable_types
|
|
147
|
+
columns = {
|
|
148
|
+
name: [
|
|
149
|
+
shortest_single(v)
|
|
150
|
+
if widths.get(name) == "float" and isinstance(v, float)
|
|
151
|
+
else render(v)
|
|
152
|
+
for v in data[name]
|
|
153
|
+
]
|
|
154
|
+
for name in usecols
|
|
155
|
+
}
|
|
156
|
+
return _rows(header, columns, meta.number_rows or 0)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
# -------------------------------------------------------------------------------------- rds
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _rds(path: pathlib.Path, wanted: set[str]) -> Frame:
|
|
163
|
+
if rdata is None:
|
|
164
|
+
raise BackendUnavailableError(
|
|
165
|
+
"table", f'rdata is not installed -- pip install "{DISTRIBUTION}[rds]"'
|
|
166
|
+
)
|
|
167
|
+
try:
|
|
168
|
+
frame = rdata.read_rds(path)
|
|
169
|
+
except (ValueError, NotImplementedError, EOFError, OSError, zlib.error, lzma.LZMAError) as e:
|
|
170
|
+
raise ArtifactUnreadableError(path, f"not a readable .rds file: {e}") from e
|
|
171
|
+
if type(frame).__name__ != "DataFrame" or not hasattr(frame, "columns"):
|
|
172
|
+
# A model fit, a vector, a list: an .rds holds any one R object. Only a data frame is a
|
|
173
|
+
# table, and a value inside anything else has no address this locator can give.
|
|
174
|
+
raise ArtifactUnreadableError(
|
|
175
|
+
path, f"holds an R object read as {type(frame).__name__}, not a data frame"
|
|
176
|
+
)
|
|
177
|
+
header = [str(c) for c in frame.columns]
|
|
178
|
+
columns = {}
|
|
179
|
+
for name in _addressable(header, wanted):
|
|
180
|
+
series = frame[name]
|
|
181
|
+
# R's `NA` reaches pandas as NaN or `pd.NA` depending on the column type, and
|
|
182
|
+
# `isna` is the one test that recognizes both.
|
|
183
|
+
missing = series.isna().tolist()
|
|
184
|
+
columns[name] = [
|
|
185
|
+
None if absent else render(v)
|
|
186
|
+
for v, absent in zip(series.tolist(), missing, strict=True)
|
|
187
|
+
]
|
|
188
|
+
return _rows(header, columns, len(frame))
|
claims/adapters/prose.py
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
"""The value between two literal anchors in the text of a document.
|
|
2
|
+
|
|
3
|
+
Every other adapter translates a locator into the addressing its format already has. Prose has
|
|
4
|
+
none: a sentence is not a tree, a table, or an indexed array. What a document does have is the
|
|
5
|
+
text on either side of a number, and that is the address this adapter resolves -- the author
|
|
6
|
+
writes the literal that precedes the value and the literal that follows it, and the adapter
|
|
7
|
+
returns what sits between.
|
|
8
|
+
|
|
9
|
+
Nothing here searches for a number. A pattern that found "the first integer in the paragraph"
|
|
10
|
+
would locate a number wherever one appeared and call that verification, which is the failure
|
|
11
|
+
`claims.adapters.base` exists to prevent.
|
|
12
|
+
|
|
13
|
+
The text is the text a quotation resolves against: `citations.verify.extract` produces it and
|
|
14
|
+
`citations.verify.passage_fold` normalizes it, so an anchor and a `quote` assertion over the same
|
|
15
|
+
document cannot disagree about what the document says.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import decimal
|
|
21
|
+
import pathlib
|
|
22
|
+
import re
|
|
23
|
+
from collections.abc import Iterator
|
|
24
|
+
|
|
25
|
+
from claims.adapters.base import Found, Resolution, _no, _ok
|
|
26
|
+
from claims.adapters.reads import once
|
|
27
|
+
from claims.exceptions import ArtifactUnreadableError, BackendUnavailableError
|
|
28
|
+
from claims.models import NumberForm, ProseLocator
|
|
29
|
+
|
|
30
|
+
_UNITS = (
|
|
31
|
+
"zero one two three four five six seven eight nine ten eleven twelve thirteen fourteen "
|
|
32
|
+
"fifteen sixteen seventeen eighteen nineteen"
|
|
33
|
+
).split()
|
|
34
|
+
|
|
35
|
+
_TENS = "twenty thirty forty fifty sixty seventy eighty ninety".split()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _cardinals() -> dict[str, int]:
|
|
39
|
+
"""English cardinals below one hundred, as one word each.
|
|
40
|
+
|
|
41
|
+
The bound is not arbitrary: the value a prose locator selects is a run of non-whitespace
|
|
42
|
+
characters, and every cardinal above ninety-nine is written as several words. Extending
|
|
43
|
+
the table would not make `one hundred and forty-five` addressable.
|
|
44
|
+
|
|
45
|
+
The table names a conversion the manifest has to ask for. It is also read under
|
|
46
|
+
`form: decimal`, where a hit means the value is refused with `number_as_word` rather than
|
|
47
|
+
reported as a missing number -- an author who spelled the count out should be told that,
|
|
48
|
+
not sent looking for a broken anchor.
|
|
49
|
+
"""
|
|
50
|
+
table = {word: value for value, word in enumerate(_UNITS)}
|
|
51
|
+
for index, tens in enumerate(_TENS):
|
|
52
|
+
base = 20 + index * 10
|
|
53
|
+
table[tens] = base
|
|
54
|
+
for unit in range(1, 10):
|
|
55
|
+
table[f"{tens}-{_UNITS[unit]}"] = base + unit
|
|
56
|
+
return table
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
CARDINALS = _cardinals()
|
|
60
|
+
|
|
61
|
+
#: A number as it is printed, for deciding where an anchor may not fall. The digits, grouping
|
|
62
|
+
#: underscores, decimal point and exponent are the unsigned part of what `decimal.Decimal`
|
|
63
|
+
#: parses, which is what reads the value in `_is_decimal` here and in `claims.verify`. The comma
|
|
64
|
+
#: is not: `Decimal("1,234")` is refused. It is here because a document prints `1,234`, and an
|
|
65
|
+
#: anchor that cut it at the comma would have `1` read as the value.
|
|
66
|
+
#:
|
|
67
|
+
#: A match counts only where no letter stands immediately before it. Digits attached to a
|
|
68
|
+
#: letter are part of a name -- the version `v0.5`, the label `H10` -- and an anchor may fall
|
|
69
|
+
#: among them. Punctuation does not make a name: `x=0.318`, `(0.318` and `p<0.05` hold numbers,
|
|
70
|
+
#: and so does a number with letters after it, `5kg`.
|
|
71
|
+
#:
|
|
72
|
+
#: A sign immediately before the digits belongs to the number where it opens the run or follows
|
|
73
|
+
#: a character that is neither a digit nor a letter: `x=-0.42`, `(-0.42)`, `| -0.42|`. After a
|
|
74
|
+
#: digit or a letter it is a range, a subtraction or a hyphen, `10-12` and `pre-2020`, and the
|
|
75
|
+
#: digits after it are a number of their own. `passage_fold` has already written U+2212 and
|
|
76
|
+
#: the dashes as `-` by the time this is read; the character is listed for a caller that has not.
|
|
77
|
+
_SIGNS = "+-\u2212"
|
|
78
|
+
_NUMBER = re.compile(r"\d+(?:_\d+)*(?:[.,]\d+(?:_\d+)*)*(?:[eE][+-]?\d+)?")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def numerals(text: str, lo: int = 0, hi: int | None = None) -> Iterator[tuple[int, int, bool]]:
|
|
82
|
+
"""Each run of digits in `text[lo:hi]` that is printed as a number: where it starts, its
|
|
83
|
+
sign included, where it ends, and whether a letter stands immediately before it.
|
|
84
|
+
|
|
85
|
+
The last is the rule above: with a letter before them the digits are part of a name, and
|
|
86
|
+
no sign is read for a name."""
|
|
87
|
+
for number in _NUMBER.finditer(text, lo, len(text) if hi is None else hi):
|
|
88
|
+
first = number.start()
|
|
89
|
+
if first > 0 and text[first - 1].isalpha():
|
|
90
|
+
yield first, number.end(), True
|
|
91
|
+
continue
|
|
92
|
+
if first > 0 and text[first - 1] in _SIGNS:
|
|
93
|
+
sign = first - 1
|
|
94
|
+
if sign == 0 or not text[sign - 1].isalnum():
|
|
95
|
+
first = sign
|
|
96
|
+
yield first, number.end(), False
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _split(text: str, at: int) -> str:
|
|
100
|
+
"""The number `at` falls strictly inside, or the empty string where it splits none."""
|
|
101
|
+
lo = at
|
|
102
|
+
while lo > 0 and not text[lo - 1].isspace():
|
|
103
|
+
lo -= 1
|
|
104
|
+
hi = at
|
|
105
|
+
while hi < len(text) and not text[hi].isspace():
|
|
106
|
+
hi += 1
|
|
107
|
+
for first, end, named in numerals(text, lo, hi):
|
|
108
|
+
if not named and first < at < end:
|
|
109
|
+
return text[first:end]
|
|
110
|
+
return ""
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _is_decimal(text: str) -> bool:
|
|
114
|
+
try:
|
|
115
|
+
decimal.Decimal(text)
|
|
116
|
+
except (decimal.InvalidOperation, ValueError):
|
|
117
|
+
return False
|
|
118
|
+
return True
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _resolve_prose(locator: ProseLocator, path: pathlib.Path) -> Found:
|
|
122
|
+
# Imported here for the reason `QuoteBackend` imports it here: an installation without a
|
|
123
|
+
# working extractor must report a check that did not run, not fail to import the package.
|
|
124
|
+
try:
|
|
125
|
+
from citations.exceptions import SourceUnreadableError
|
|
126
|
+
from citations.verify import extract_uncached, passage_fold
|
|
127
|
+
except ImportError as e: # pragma: no cover - citations is a declared dependency
|
|
128
|
+
raise BackendUnavailableError("prose", f"citations is not installed: {e}") from e
|
|
129
|
+
|
|
130
|
+
# `extract` is cached on the path, so a second verification in one process reads the text
|
|
131
|
+
# the first one extracted, whatever the file now holds. `extract_uncached` is the named
|
|
132
|
+
# read; this reached it through `extract.__wrapped__` until that attribute stopped existing
|
|
133
|
+
# and the `getattr` fallback quietly returned the cached function instead. The engine hashes every artifact
|
|
134
|
+
# immediately before this runs, so a stale read would produce a decision whose recorded
|
|
135
|
+
# artifact digest does not describe the text it was computed from. `fold` is cached on the
|
|
136
|
+
# string it is given, which is content, and stays as it is. `once` keeps the read for the
|
|
137
|
+
# verification in progress and no longer, so the next verification reads the file again.
|
|
138
|
+
try:
|
|
139
|
+
text = once(("prose", path), lambda: passage_fold(extract_uncached(path)))
|
|
140
|
+
except SourceUnreadableError as e:
|
|
141
|
+
raise ArtifactUnreadableError(path, e.detail) from e
|
|
142
|
+
if not text:
|
|
143
|
+
raise ArtifactUnreadableError(path, "no text extracted")
|
|
144
|
+
|
|
145
|
+
# `fold` trims and collapses whitespace, which is what an anchor wants: the boundary
|
|
146
|
+
# spaces are supplied by `_spans`, and an anchor written across a line break in the
|
|
147
|
+
# manifest matches a document that wrapped it somewhere else.
|
|
148
|
+
before, after = passage_fold(locator.before), passage_fold(locator.after)
|
|
149
|
+
if not before:
|
|
150
|
+
return _no(
|
|
151
|
+
Resolution.SELECTOR_INVALID,
|
|
152
|
+
"`before` is empty after normalization, so it anchors nothing",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
hits, split = _spans(text, before, after)
|
|
156
|
+
address = (
|
|
157
|
+
f'"{locator.before}" ... "{locator.after}"' if locator.after else f'"{locator.before}"'
|
|
158
|
+
)
|
|
159
|
+
if not hits:
|
|
160
|
+
refused = (
|
|
161
|
+
f"; an anchor never splits a number, and these would split {', '.join(split)}"
|
|
162
|
+
if split
|
|
163
|
+
else ""
|
|
164
|
+
)
|
|
165
|
+
return _no(
|
|
166
|
+
Resolution.ABSENT,
|
|
167
|
+
f"no value sits between {address} in {path.name}; anchors are literal text, "
|
|
168
|
+
f"matched after the same normalization a quotation is matched under{refused}",
|
|
169
|
+
)
|
|
170
|
+
values = sorted(set(hits))
|
|
171
|
+
if len(values) > 1:
|
|
172
|
+
# Repetition is normal in a document and is not ambiguity: a count stated in an
|
|
173
|
+
# abstract and again in a section is one value. Two *different* values between the
|
|
174
|
+
# same anchors is a document contradicting itself, which is a finding rather than a
|
|
175
|
+
# thing to resolve by taking the first.
|
|
176
|
+
return _no(
|
|
177
|
+
Resolution.PASSAGE_AMBIGUOUS,
|
|
178
|
+
f"{address} selects {len(values)} different values in {path.name}: {', '.join(values)}",
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
value = values[0]
|
|
182
|
+
seen = f"{len(hits)} occurrence(s)" if len(hits) > 1 else ""
|
|
183
|
+
trace = (address, seen) if seen else (address,)
|
|
184
|
+
|
|
185
|
+
if locator.form is NumberForm.CARDINAL_WORD:
|
|
186
|
+
if (number := CARDINALS.get(value)) is not None:
|
|
187
|
+
return _ok(str(number), "cardinal_word", *trace, f'the word "{value}"')
|
|
188
|
+
# A numeral where a word was declared still parses, and refusing it would break a
|
|
189
|
+
# check because someone corrected the prose. Anything else falls through to the
|
|
190
|
+
# backend, which reports it as not numeric.
|
|
191
|
+
return _ok(value, "text", *trace)
|
|
192
|
+
|
|
193
|
+
if not _is_decimal(value) and value in CARDINALS:
|
|
194
|
+
return _no(
|
|
195
|
+
Resolution.NUMBER_AS_WORD,
|
|
196
|
+
f'{address} holds "{value}" in {path.name}. Reading a word as a number is a '
|
|
197
|
+
f"semantic decision the engine does not make on its own; declare "
|
|
198
|
+
f"`form: cardinal_word` on the locator to ask for it",
|
|
199
|
+
)
|
|
200
|
+
return _ok(value, "decimal" if _is_decimal(value) else "text", *trace)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _spans(text: str, before: str, after: str) -> tuple[list[str], list[str]]:
|
|
204
|
+
"""The value after each occurrence of `before`, and the numbers an anchor would have split."""
|
|
205
|
+
found, split = located(text, before, after)
|
|
206
|
+
return [text[start:end] for start, end in found], split
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def located(text: str, before: str, after: str) -> tuple[list[tuple[int, int]], list[str]]:
|
|
210
|
+
"""Where the value after each occurrence of `before` starts and ends, and the numbers an
|
|
211
|
+
anchor would have split.
|
|
212
|
+
|
|
213
|
+
The value is a run of non-whitespace characters. With no `after` it is the whole run up to
|
|
214
|
+
the next whitespace. With one, the whole run is taken where `after` begins past it, and
|
|
215
|
+
otherwise the run is cut at the first place inside it where `after` begins: a cell printed
|
|
216
|
+
against its divider, `0.318|`, or a number closing a sentence, `3.14.`, is delimited by
|
|
217
|
+
the anchor that touches it and by nothing the engine guessed.
|
|
218
|
+
|
|
219
|
+
Neither anchor may fall inside a number. `after: "8"` does not read `0.318` as `0.31`, the
|
|
220
|
+
first full stop in `3.14.` is passed over for the second, and `before: "0.3"` does not
|
|
221
|
+
read `18`. An occurrence refused on those grounds yields no value, and the number it
|
|
222
|
+
would have split is returned so the report can say why. Digits attached to a preceding
|
|
223
|
+
letter are a name, so `before: "v0."` reads `5` from `v0.5`. A sign is part of its number
|
|
224
|
+
unless a digit or a letter precedes it, so `before: "x=-"` reads nothing from `x=-0.42`
|
|
225
|
+
and `before: "10-"` reads `12` from `10-12`.
|
|
226
|
+
|
|
227
|
+
Whitespace at an anchor boundary is ignored, on the same grounds `fold` collapses it
|
|
228
|
+
everywhere else: `before: "holds"` and `before: "holds "` address the same value, and a
|
|
229
|
+
manifest should not turn on which one the author typed.
|
|
230
|
+
"""
|
|
231
|
+
found: list[tuple[int, int]] = []
|
|
232
|
+
split: list[str] = []
|
|
233
|
+
n, start = len(text), 0
|
|
234
|
+
while (at := text.find(before, start)) != -1:
|
|
235
|
+
start = at + 1
|
|
236
|
+
cursor = at + len(before)
|
|
237
|
+
if number := _split(text, cursor):
|
|
238
|
+
split.append(number)
|
|
239
|
+
continue
|
|
240
|
+
while cursor < n and text[cursor].isspace():
|
|
241
|
+
cursor += 1
|
|
242
|
+
end = cursor
|
|
243
|
+
while end < n and not text[end].isspace():
|
|
244
|
+
end += 1
|
|
245
|
+
if end == cursor:
|
|
246
|
+
continue
|
|
247
|
+
tail = end
|
|
248
|
+
while tail < n and text[tail].isspace():
|
|
249
|
+
tail += 1
|
|
250
|
+
if not after or text.startswith(after, tail):
|
|
251
|
+
found.append((cursor, end))
|
|
252
|
+
continue
|
|
253
|
+
for stop in range(cursor + 1, end):
|
|
254
|
+
if not text.startswith(after, stop):
|
|
255
|
+
continue
|
|
256
|
+
if number := _split(text, stop):
|
|
257
|
+
split.append(number)
|
|
258
|
+
continue
|
|
259
|
+
found.append((cursor, stop))
|
|
260
|
+
break
|
|
261
|
+
return found, list(dict.fromkeys(split))
|
claims/adapters/reads.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""Reading each artifact once in a verification.
|
|
2
|
+
|
|
3
|
+
An adapter resolves one locator against one file, and a manifest of a thousand claims over one
|
|
4
|
+
table asked for that table to be read and parsed a thousand times. Within `reading_once()` a
|
|
5
|
+
load is kept under its key and returned to every later locator that asks for it.
|
|
6
|
+
|
|
7
|
+
The scope is one verification and nothing longer. The engine hashes every artifact at the start
|
|
8
|
+
of a run, and a decision records that digest as describing what was read; a file read once in
|
|
9
|
+
the run is read as close to that hash as it can be. Outside the scope nothing is kept, so a
|
|
10
|
+
second verification in the same process reads the file again, whatever the first one saw. A
|
|
11
|
+
cache keyed on the path alone and living as long as the process returned the first run's text
|
|
12
|
+
to the second.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import contextlib
|
|
18
|
+
from collections.abc import Callable, Hashable, Iterator
|
|
19
|
+
from contextvars import ContextVar
|
|
20
|
+
from typing import TypeVar, cast
|
|
21
|
+
|
|
22
|
+
T = TypeVar("T")
|
|
23
|
+
|
|
24
|
+
_HELD: ContextVar[dict[Hashable, object] | None] = ContextVar("repro_reads", default=None)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@contextlib.contextmanager
|
|
28
|
+
def reading_once() -> Iterator[None]:
|
|
29
|
+
"""Keep each load made inside the block until the block ends."""
|
|
30
|
+
token = _HELD.set({})
|
|
31
|
+
try:
|
|
32
|
+
yield
|
|
33
|
+
finally:
|
|
34
|
+
_HELD.reset(token)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def once(key: Hashable, load: Callable[[], T]) -> T:
|
|
38
|
+
"""`load()`, or what it returned earlier in this run under the same key.
|
|
39
|
+
|
|
40
|
+
A load that raises is not kept: every locator over an unreadable file reports it.
|
|
41
|
+
"""
|
|
42
|
+
held = _HELD.get()
|
|
43
|
+
if held is None:
|
|
44
|
+
return load()
|
|
45
|
+
if key not in held:
|
|
46
|
+
held[key] = load()
|
|
47
|
+
return cast(T, held[key])
|
claims/adapters/rows.py
ADDED
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
"""Selecting one cell of a table, whatever format the table was read from.
|
|
2
|
+
|
|
3
|
+
Every table adapter reduces its file to a header and a list of rows, each row a mapping from
|
|
4
|
+
column name to `cells.Cell`, and resolves a locator here. One implementation means a CSV, a
|
|
5
|
+
workbook sheet and a Parquet file holding the same table refuse the same manifests for the same
|
|
6
|
+
reasons, which three copies of this logic would drift away from.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from claims.adapters.base import Found, Resolution, _no, _ok
|
|
12
|
+
from claims.adapters.cells import Cell, Container
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _header_faults(header: list[str], column: str, where: dict, name: str) -> Found | None:
|
|
16
|
+
"""What is wrong with a header before any row is read, or None.
|
|
17
|
+
|
|
18
|
+
Shared by every table, delimited or typed, so a workbook and a CSV holding the same table
|
|
19
|
+
refuse the same manifests with the same reasons.
|
|
20
|
+
"""
|
|
21
|
+
repeated = sorted({c for c in header if header.count(c) > 1})
|
|
22
|
+
if repeated:
|
|
23
|
+
# `csv.DictReader` keeps the last field of a repeated header, so one of the columns is
|
|
24
|
+
# unreachable and a predicate naming it reports a present row as absent. Neither is a
|
|
25
|
+
# fact about the data.
|
|
26
|
+
return _no(
|
|
27
|
+
Resolution.SELECTOR_INVALID,
|
|
28
|
+
f"{name} repeats the column name {', '.join(repr(c) for c in repeated)}; "
|
|
29
|
+
f"a repeated header makes one of them unaddressable",
|
|
30
|
+
)
|
|
31
|
+
if column not in header:
|
|
32
|
+
return _no(
|
|
33
|
+
Resolution.COLUMN_ABSENT,
|
|
34
|
+
f"{name} has no column {column!r}; columns are {', '.join(header[:8])}",
|
|
35
|
+
)
|
|
36
|
+
unknown = [k for k in where if k not in header]
|
|
37
|
+
if unknown:
|
|
38
|
+
# Left to the row scan this matches nothing and reads as "no such row", blaming the
|
|
39
|
+
# table for a manifest that named a column the table never had.
|
|
40
|
+
return _no(
|
|
41
|
+
Resolution.SELECTOR_INVALID,
|
|
42
|
+
f"selector names {', '.join(repr(k) for k in unknown)}, which "
|
|
43
|
+
f"{name} has no column for; columns are {', '.join(header[:8])}",
|
|
44
|
+
)
|
|
45
|
+
return None
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _cell_found(cell: Cell, column: str, address: str) -> Found:
|
|
49
|
+
"""A selected cell as a resolution. A typed table distinguishes a missing value from text.
|
|
50
|
+
|
|
51
|
+
`address` is how the row was picked, `where model='vit'` or `at row 3`.
|
|
52
|
+
"""
|
|
53
|
+
if cell is None:
|
|
54
|
+
# The typed readers' null: a Parquet null, an R `NA`, an empty workbook cell, a formula
|
|
55
|
+
# whose result was never computed. Like a NULL in SQLite it asserts nothing, so it is
|
|
56
|
+
# absent rather than the empty string a comparison would call non-numeric.
|
|
57
|
+
return _no(Resolution.ABSENT, f"{column} holds no value {address}")
|
|
58
|
+
if isinstance(cell, Container):
|
|
59
|
+
return _no(Resolution.NOT_SCALAR, f"{column} holds {cell.holds} {address}")
|
|
60
|
+
return _ok(cell, "str", f"{column} {address}")
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def index_rows(rows: list[dict[str, Cell]], columns: tuple[str, ...]) -> dict[tuple, list[int]]:
|
|
64
|
+
"""Row numbers by the cells each row holds in `columns`, for a table of text cells.
|
|
65
|
+
|
|
66
|
+
One pass over the table answers every locator that selects on the same columns, where a
|
|
67
|
+
scan per locator read every row once for each of them.
|
|
68
|
+
"""
|
|
69
|
+
index: dict[tuple, list[int]] = {}
|
|
70
|
+
for at, row in enumerate(rows):
|
|
71
|
+
index.setdefault(tuple(row.get(column) for column in columns), []).append(at)
|
|
72
|
+
return index
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def resolve_rows(
|
|
76
|
+
header: list[str],
|
|
77
|
+
rows: list[dict[str, Cell]],
|
|
78
|
+
column: str,
|
|
79
|
+
where: dict,
|
|
80
|
+
name: str,
|
|
81
|
+
index: dict[tuple, list[int]] | None = None,
|
|
82
|
+
) -> Found:
|
|
83
|
+
"""The one row whose key cells equal `where`, and its cell in `column`.
|
|
84
|
+
|
|
85
|
+
`where` holds text already, in the convention of the table it is matched against:
|
|
86
|
+
`predicate_text` for delimited text, `cells.render` for a typed table. `index` is
|
|
87
|
+
`index_rows` over the sorted names of `where`, and selects the rows a scan would.
|
|
88
|
+
"""
|
|
89
|
+
if failure := _header_faults(header, column, where, name):
|
|
90
|
+
return failure
|
|
91
|
+
if index is None:
|
|
92
|
+
matched = [
|
|
93
|
+
i for i, row in enumerate(rows) if all(row.get(k) == v for k, v in where.items())
|
|
94
|
+
]
|
|
95
|
+
else:
|
|
96
|
+
matched = index.get(tuple(where[k] for k in sorted(where)), [])
|
|
97
|
+
described = ", ".join(f"{k}={v!r}" for k, v in where.items())
|
|
98
|
+
if not matched:
|
|
99
|
+
return _no(Resolution.ABSENT, f"no row in {name} where {described}")
|
|
100
|
+
if len(matched) > 1:
|
|
101
|
+
return _no(Resolution.AMBIGUOUS, f"{len(matched)} rows in {name} where {described}")
|
|
102
|
+
return _cell_found(rows[matched[0]].get(column), column, f"where {described}")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def resolve_row_at(
|
|
106
|
+
header: list[str], rows: list[dict[str, Cell]], column: str, row: int, name: str
|
|
107
|
+
) -> Found:
|
|
108
|
+
"""The cell in `column` at data row `row`, counting from zero below the header."""
|
|
109
|
+
if failure := _header_faults(header, column, {}, name):
|
|
110
|
+
return failure
|
|
111
|
+
if row >= len(rows):
|
|
112
|
+
return _no(
|
|
113
|
+
Resolution.ABSENT,
|
|
114
|
+
f"{name} has {len(rows)} data rows; row {row} is past the end",
|
|
115
|
+
)
|
|
116
|
+
return _cell_found(rows[row].get(column), column, f"at row {row}")
|