claims-cli 0.6.0__py3-none-any.whl

claims/adapters/sheet.py

@@ -0,0 +1,149 @@
+"""Excel workbooks, addressed by sheet, column and a row predicate.
+
+`.xlsx` and `.xlsm` are read with `openpyxl` and `.xls` with `xlrd`, both optional
+(`pip install "reproducible-science[sheets]"`). Without them a check is `unchecked` and says
+which extra to install; it never fails and never passes.
+
+A cell is read as the value a formula last computed, which is what the sheet shows, and never
+as the formula. A workbook saved by a program that does not compute formulas holds no computed
+value for them, and such a cell is reported as holding no value: a check that substituted the
+formula text, or zero, would be comparing the manuscript against something the file does not
+say.
+"""
+
+from __future__ import annotations
+
+import pathlib
+import warnings
+import zipfile
+
+from claims.adapters.base import Found, Resolution, _no
+from claims.adapters.cells import Cell, render
+from claims.adapters.rows import resolve_rows
+from claims.exceptions import ArtifactUnreadableError, BackendUnavailableError
+from claims.models import SheetLocator
+from claims.toolchain import DISTRIBUTION
+
+try:  # optional: `pip install "reproducible-science[sheets]"`
+    import openpyxl
+except ImportError:  # pragma: no cover - exercised by the extras matrix, not the suite
+    openpyxl = None
+
+try:  # optional: `pip install "reproducible-science[sheets]"`
+    import xlrd
+    import xlrd.biffh
+except ImportError:  # pragma: no cover
+    xlrd = None
+
+WORKBOOK_SUFFIXES = {".xlsx", ".xlsm", ".xls"}
+
+_INSTALL = f'pip install "{DISTRIBUTION}[sheets]"'
+
+
+def _xlsx(path: pathlib.Path, sheet: str | None) -> tuple[list[str], list[list[object]]]:
+    """Sheet names, and the rows of the one selected; `sheet` is resolved by `_choose`."""
+    if openpyxl is None:
+        raise BackendUnavailableError("sheet", f"openpyxl is not installed -- {_INSTALL}")
+    # openpyxl warns about workbook features it does not model -- a data validation extension,
+    # a conditional format. None is a cell's value, and the warning would land in the middle of
+    # a verification report.
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+        try:
+            book = openpyxl.load_workbook(path, read_only=True, data_only=True)
+        except (zipfile.BadZipFile, KeyError, OSError, ValueError) as e:
+            raise ArtifactUnreadableError(path, f"not a readable workbook: {e}") from e
+        try:
+            names = list(book.sheetnames)
+            chosen = _choose(names, sheet)
+            if chosen is None:
+                return names, []
+            ws = book[chosen]
+            # A workbook written by something other than Excel can record a used range of one
+            # cell, and a read-only sheet trusts it: every row after the first is silently not
+            # returned. The citations extractor met this on publishers' supplements.
+            ws.reset_dimensions()
+            return names, [list(row) for row in ws.iter_rows(values_only=True)]
+        finally:
+            book.close()
+
+
+def _xls(path: pathlib.Path, sheet: str | None) -> tuple[list[str], list[list[object]]]:
+    if xlrd is None:
+        raise BackendUnavailableError("sheet", f"xlrd is not installed -- {_INSTALL}")
+    try:
+        book = xlrd.open_workbook(str(path))
+    except (xlrd.biffh.XLRDError, OSError, EOFError, AssertionError) as e:
+        raise ArtifactUnreadableError(path, f"not a readable workbook: {e}") from e
+    names = book.sheet_names()
+    chosen = _choose(names, sheet)
+    if chosen is None:
+        return names, []
+    ws = book.sheet_by_name(chosen)
+
+    def value(cell) -> object:
+        # The old format has no integer type, and an empty cell is the empty string. A date is
+        # a float tagged as one; it is not a number a manuscript reports, and reading it as
+        # one would compare a day count against a printed value.
+        if cell.ctype in (xlrd.XL_CELL_EMPTY, xlrd.XL_CELL_BLANK):
+            return None
+        if cell.ctype == xlrd.XL_CELL_DATE:
+            return xlrd.xldate_as_datetime(cell.value, book.datemode)
+        if cell.ctype == xlrd.XL_CELL_BOOLEAN:
+            return bool(cell.value)
+        if cell.ctype == xlrd.XL_CELL_ERROR:
+            return xlrd.error_text_from_code.get(cell.value, "#ERROR")
+        return cell.value
+
+    return names, [[value(c) for c in ws.row(r)] for r in range(ws.nrows)]
+
+
+def _choose(names: list[str], sheet: str | None) -> str | None:
+    """The sheet a locator selects, or None when it selects none; `_resolve_sheet` says why."""
+    if sheet is None:
+        return names[0] if len(names) == 1 else None
+    return sheet if sheet in names else None
+
+
+def _frame(raw: list[list[object]]) -> tuple[list[str], list[dict[str, Cell]]]:
+    """The first row as the header and every later row keyed by it, as delimited text is read.
+
+    A column whose header cell is empty is left out: no locator can name it, and a sheet's used
+    range routinely runs past its last named column. Kept as columns named "", two of them
+    would repeat a header name and the repeated-header rule would refuse the whole sheet.
+    """
+    if not raw:
+        return [], []
+    names = [h if isinstance(h, str) else "" for h in map(render, raw[0])]
+    named = [(i, n) for i, n in enumerate(names) if n]
+    header = [n for _, n in named]
+    rows: list[dict[str, Cell]] = [
+        {n: render(values[i]) if i < len(values) else None for i, n in named} for values in raw[1:]
+    ]
+    return header, rows
+
+
+def _resolve_sheet(locator: SheetLocator, path: pathlib.Path) -> Found:
+    suffix = path.suffix.lower()
+    if suffix not in WORKBOOK_SUFFIXES:
+        return _no(
+            Resolution.FORMAT_UNSUPPORTED,
+            f"a sheet locator addresses an Excel workbook; {path.name} is "
+            f"{path.suffix or 'extensionless'}",
+        )
+    read = _xls if suffix == ".xls" else _xlsx
+    names, raw = read(path, locator.sheet)
+    if locator.sheet is None and len(names) != 1:
+        return _no(
+            Resolution.SELECTOR_INVALID,
+            f"{path.name} holds {len(names)} sheets; name one of {', '.join(names[:8])}",
+        )
+    if locator.sheet is not None and locator.sheet not in names:
+        return _no(
+            Resolution.SELECTOR_INVALID,
+            f"{path.name} has no sheet {locator.sheet!r}; sheets are {', '.join(names[:8])}",
+        )
+    header, rows = _frame(raw)
+    name = f"{path.name}, sheet {locator.sheet or names[0]!r}"
+    wanted = {k: render(v) for k, v in locator.where.items()}
+    return resolve_rows(header, rows, locator.column, wanted, name)

claims/adapters/sqlite.py

@@ -0,0 +1,73 @@
+"""SQLite, addressed by table, column and a key predicate."""
+
+from __future__ import annotations
+
+import pathlib
+import sqlite3
+
+from claims.adapters.base import Found, Resolution, _no, _ok
+from claims.exceptions import ArtifactUnreadableError
+from claims.models import (
+    SqliteLocator,
+)
+
+# ---------------------------------------------------------------------------------- sqlite
+
+_SQLITE_SUFFIXES = {".db", ".sqlite", ".sqlite3"}
+
+
+def _resolve_sqlite(locator: SqliteLocator, path: pathlib.Path) -> Found:
+    if path.suffix.lower() not in _SQLITE_SUFFIXES:
+        return _no(
+            Resolution.FORMAT_UNSUPPORTED,
+            f"a sqlite locator addresses a database file; {path.name} is {path.suffix}",
+        )
+    try:
+        connection = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    except sqlite3.Error as e:
+        raise ArtifactUnreadableError(path, str(e)) from e
+    try:
+        tables = {
+            r[0]
+            for r in connection.execute(
+                "SELECT name FROM sqlite_master WHERE type IN ('table','view')"
+            )
+        }
+        if locator.table not in tables:
+            return _no(
+                Resolution.SELECTOR_INVALID,
+                f"{path.name} has no table {locator.table!r}; tables are "
+                f"{', '.join(sorted(tables)[:8])}",
+            )
+        # Identifiers cannot be bound as parameters, so they are checked against the schema
+        # and then quoted. Values are always bound.
+        columns = {r[1] for r in connection.execute(f'PRAGMA table_info("{locator.table}")')}
+        unknown = [c for c in (locator.column, *locator.where) if c not in columns]
+        if unknown:
+            return _no(
+                Resolution.SELECTOR_INVALID,
+                f"{locator.table} has no column {', '.join(repr(c) for c in unknown)}; "
+                f"columns are {', '.join(sorted(columns)[:8])}",
+            )
+        clause = " AND ".join(f'"{k}" IS ?' for k in locator.where)
+        query = f'SELECT "{locator.column}" FROM "{locator.table}"' + (
+            f" WHERE {clause}" if clause else ""
+        )
+        try:
+            found = connection.execute(query, tuple(locator.where.values())).fetchall()
+        except sqlite3.Error as e:
+            raise ArtifactUnreadableError(path, str(e)) from e
+    finally:
+        connection.close()
+
+    described = ", ".join(f"{k}={v!r}" for k, v in locator.where.items())
+    if not found:
+        return _no(Resolution.ABSENT, f"no row in {locator.table} where {described}")
+    if len(found) > 1:
+        return _no(Resolution.AMBIGUOUS, f"{len(found)} rows in {locator.table} where {described}")
+    value = found[0][0]
+    if value is None:
+        return _no(Resolution.ABSENT, f"{locator.column} is NULL where {described}")
+    if isinstance(value, (bytes, memoryview)):
+        return _no(Resolution.NOT_SCALAR, f"{locator.column} holds a blob")
+    return _ok(str(value), type(value).__name__, f"{locator.table}.{locator.column}")

claims/adapters/table.py

@@ -0,0 +1,135 @@
+"""Tables, addressed by column plus a row predicate.
+
+CSV, TSV and PSV are read here. Parquet, Feather, Stata, SPSS and R data frames are read by
+`claims.adapters.columnar` into the same header and rows, and resolved by the same code, so a
+table and its export to another format refuse and resolve the same manifests.
+"""
+
+from __future__ import annotations
+
+import csv
+import io
+import pathlib
+
+from claims.adapters.base import Found, Resolution, _no
+from claims.adapters.cells import Cell, render
+from claims.adapters.columnar import COLUMNAR_SUFFIXES, read_columnar
+from claims.adapters.reads import once
+from claims.adapters.rows import index_rows, resolve_row_at, resolve_rows
+from claims.adapters.sheet import WORKBOOK_SUFFIXES
+from claims.exceptions import ArtifactUnreadableError
+from claims.models import (
+    PredicateValue,
+    TableLocator,
+    TablePositionLocator,
+)
+
+# ----------------------------------------------------------------------------------- tables
+
+#: Delimiters implied by a suffix, checked before sniffing.
+_DELIMITERS = {".csv": ",", ".tsv": "\t", ".psv": "|"}
+_TABLE_SUFFIXES = set(_DELIMITERS) | {".txt", ""}
+
+
+def sniff_delimiter(path: pathlib.Path, sample: str) -> str:
+    """The delimiter for a table, from its suffix or from the header line.
+
+    Suffix first, because a `.tsv` whose header happens to contain commas is still tab
+    separated and sniffing it would split every row in the wrong place.
+    """
+    if known := _DELIMITERS.get(path.suffix.lower()):
+        return known
+    header = sample.splitlines()[0] if sample.splitlines() else ""
+    counts = {d: header.count(d) for d in (",", "\t", ";", "|")}
+    best = max(counts, key=lambda delimiter: counts[delimiter])
+    return best if counts[best] else ","
+
+
+def read_table(path: pathlib.Path, delimiter: str = "") -> tuple[list[str], list[dict]]:
+    """Header and rows. Raises `ArtifactUnreadableError` when the file is not a table."""
+    try:
+        text = path.read_text(encoding="utf-8-sig")
+    except (OSError, UnicodeDecodeError) as e:
+        raise ArtifactUnreadableError(path, str(e)) from e
+    if not text.strip():
+        raise ArtifactUnreadableError(path, "file is empty")
+    reader = csv.DictReader(io.StringIO(text), delimiter=delimiter or sniff_delimiter(path, text))
+    rows = list(reader)
+    if reader.fieldnames is None:
+        raise ArtifactUnreadableError(path, "no header row")
+    return list(reader.fieldnames), rows
+
+
+def predicate_text(value: PredicateValue) -> str:
+    """A predicate value as the text a delimited cell would hold.
+
+    Delimited files have no types: every cell is text. Comparing as text and never coercing
+    keeps `"001"` distinct from `1`, which for an identifier column is the difference between
+    two different rows.
+    """
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    return str(value)
+
+
+def _delimited(path: pathlib.Path, delimiter: str) -> tuple[list[str], list[dict[str, Cell]]]:
+    header, rows = read_table(path, delimiter)
+    # Every delimited cell is text. A short row's missing fields come back as None, which here
+    # would read as a null; they were always the empty string, and stay so. The `None` key
+    # holds a long row's surplus fields, which no header names.
+    text = [{k: (v or "").strip() for k, v in row.items() if k is not None} for row in rows]
+    return header, text
+
+
+def _load(path: pathlib.Path, delimiter: str, wanted: set[str]) -> tuple[tuple, Found | None]:
+    """A table's header and rows from whichever reader its suffix names, or why not.
+
+    `wanted` is the columns a locator names, which is all a columnar reader reads.
+    """
+    suffix = path.suffix.lower()
+    if suffix in COLUMNAR_SUFFIXES:
+        if delimiter:
+            return (), _no(
+                Resolution.SELECTOR_INVALID,
+                f"{path.name} is {suffix}, which has no delimiter to override",
+            )
+        return read_columnar(path, wanted), None
+    if suffix in WORKBOOK_SUFFIXES:
+        return (), _no(
+            Resolution.FORMAT_UNSUPPORTED,
+            f"{path.name} is a workbook, which holds several tables; address it with "
+            f"`kind: sheet`, which names the sheet",
+        )
+    if suffix not in _TABLE_SUFFIXES:
+        return (), _no(
+            Resolution.FORMAT_UNSUPPORTED,
+            f"a table locator addresses delimited text or a columnar file; "
+            f"{path.name} is {path.suffix}",
+        )
+    return once(("delimited", path, delimiter), lambda: _delimited(path, delimiter)), None
+
+
+def _resolve_table(locator: TableLocator, path: pathlib.Path) -> Found:
+    loaded, failure = _load(path, locator.delimiter, {locator.column, *locator.where})
+    if failure is not None:
+        return failure
+    header, rows = loaded
+    # Delimited text is matched as text, as it always was; a typed table renders the predicate
+    # in the convention its cells were rendered in, so `seed: 2` selects a stored 2.0.
+    columnar = path.suffix.lower() in COLUMNAR_SUFFIXES
+    key = render if columnar else predicate_text
+    wanted = {k: key(v) for k, v in locator.where.items()}
+    if columnar:
+        # A columnar read holds only the columns one locator named, so it is not shared.
+        return resolve_rows(header, rows, locator.column, wanted, path.name)
+    columns = tuple(sorted(wanted))
+    index = once(("rows", path, locator.delimiter, columns), lambda: index_rows(rows, columns))
+    return resolve_rows(header, rows, locator.column, wanted, path.name, index)
+
+
+def _resolve_table_position(locator: TablePositionLocator, path: pathlib.Path) -> Found:
+    loaded, failure = _load(path, locator.delimiter, {locator.column})
+    if failure is not None:
+        return failure
+    header, rows = loaded
+    return resolve_row_at(header, rows, locator.column, locator.row, path.name)

claims/adapters/tree.py

@@ -0,0 +1,163 @@
+"""JSON and restricted YAML, addressed by RFC 6901 pointer."""
+
+from __future__ import annotations
+
+import decimal
+import json
+import pathlib
+
+import yaml
+
+from claims.adapters.base import Found, Resolution, _no, _ok
+from claims.adapters.reads import once
+from claims.exceptions import ArtifactUnreadableError
+from claims.models import (
+    TreeLocator,
+)
+
+# ------------------------------------------------------------------------------------ trees
+
+_TREE_SUFFIXES = {".json": "json", ".yaml": "yaml", ".yml": "yaml"}
+_MISSING = object()
+
+
+class Written(str):
+    """A number with a fraction or an exponent, as the characters the file wrote for it.
+
+    Both loaders hand such a number over as a binary float, and the float was then printed
+    back: `0.910` came out as `0.91`, `1e-7` as `1e-07`, a value of twenty significant digits
+    as its nearest seventeen, `9007199254740993.0` as `9007199254740992.0`, and `1e400` as
+    `inf`. A comparison at printed precision in exact decimals was made against digits the
+    file never held. The text is kept in place of the float and is what `decimal.Decimal`
+    reads, so the value compared is the decimal the file wrote, trailing zeros included.
+
+    An integer needs none of this: both loaders read one exactly, at any length.
+    """
+
+    __slots__ = ()
+
+
+def resolve_pointer(document: object, pointer: str) -> object:
+    """RFC 6901 JSON Pointer resolution. Returns `_MISSING` when the pointer does not resolve.
+
+    `~1` is a literal `/` and `~0` a literal `~`, unescaped in that order, so a key containing
+    a slash is addressable and a key containing a period is unremarkable.
+    """
+    if pointer == "":
+        return document
+    if not pointer.startswith("/"):
+        raise ValueError(f"JSON Pointer must start with '/': {pointer!r}")
+    node = document
+    for raw in pointer[1:].split("/"):
+        token = raw.replace("~1", "/").replace("~0", "~")
+        if isinstance(node, dict):
+            if token not in node:
+                return _MISSING
+            node = node[token]
+        elif isinstance(node, list):
+            # RFC 6901 array indices are digits with no leading zeros, so "01" addresses
+            # nothing and is not silently read as 1.
+            if not token.isdigit() or (len(token) > 1 and token[0] == "0"):
+                return _MISSING
+            index = int(token)
+            if index >= len(node):
+                return _MISSING
+            node = node[index]
+        else:
+            return _MISSING
+    return node
+
+
+def _no_duplicate_json_keys(pairs: list[tuple[str, object]]) -> dict:
+    """`json.loads` keeps the last of a duplicated key, exactly as PyYAML does.
+
+    The YAML path has rejected this from the start; JSON did not, so one artifact could hold
+    two values for one quantity and resolve to whichever came last. That is the finding this
+    repository's own regression corpus records against someone else's paper.
+    """
+    seen: dict[str, object] = {}
+    for key, value in pairs:
+        if key in seen:
+            raise ValueError(f"duplicate key {key!r}")
+        seen[key] = value
+    return seen
+
+
+class _StrictYaml(yaml.SafeLoader):
+    """SafeLoader that refuses duplicate mapping keys.
+
+    PyYAML keeps the last of a duplicated key, so a file with two `accuracy:` entries resolves
+    to one of them with nothing said. An artifact that cannot be read one way only is not one
+    a pointer can address.
+    """
+
+
+def _no_duplicate_keys(loader, node, deep=False):
+    mapping = {}
+    for key_node, value_node in node.value:
+        key = loader.construct_object(key_node, deep=deep)
+        if key in mapping:
+            raise yaml.constructor.ConstructorError(
+                None, None, f"duplicate key {key!r}", key_node.start_mark
+            )
+        mapping[key] = loader.construct_object(value_node, deep=deep)
+    return mapping
+
+
+_StrictYaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _no_duplicate_keys)
+
+
+def _float_as_written(loader, node):
+    """A YAML float as its text, where the text is a decimal literal.
+
+    YAML has float forms `decimal.Decimal` does not read: `.inf`, `.nan` and base-sixty
+    `1:30.5`. Those stay the floats the loader makes of them, as they were.
+    """
+    text = loader.construct_scalar(node)
+    try:
+        decimal.Decimal(text)
+    except decimal.InvalidOperation:
+        return loader.construct_yaml_float(node)
+    return Written(text)
+
+
+_StrictYaml.add_constructor("tag:yaml.org,2002:float", _float_as_written)
+
+
+def _load_tree(path: pathlib.Path) -> object:
+    fmt = _TREE_SUFFIXES.get(path.suffix.lower())
+    try:
+        text = path.read_text(encoding="utf-8-sig")
+    except (OSError, UnicodeDecodeError) as e:
+        raise ArtifactUnreadableError(path, str(e)) from e
+    if fmt == "json":
+        try:
+            return json.loads(text, object_pairs_hook=_no_duplicate_json_keys, parse_float=Written)
+        except json.JSONDecodeError as e:
+            raise ArtifactUnreadableError(path, f"not valid JSON: {e}") from e
+        except ValueError as e:  # raised by the hook below
+            raise ArtifactUnreadableError(path, str(e)) from e
+    try:
+        return yaml.load(text, Loader=_StrictYaml)
+    except yaml.YAMLError as e:
+        raise ArtifactUnreadableError(path, f"not valid YAML: {e}") from e
+
+
+def _resolve_tree(locator: TreeLocator, path: pathlib.Path) -> Found:
+    if path.suffix.lower() not in _TREE_SUFFIXES:
+        return _no(
+            Resolution.FORMAT_UNSUPPORTED,
+            f"a tree locator addresses JSON or YAML; {path.name} is "
+            f"{path.suffix or 'extensionless'}",
+        )
+    node = resolve_pointer(once(("tree", path), lambda: _load_tree(path)), locator.pointer)
+    if node is _MISSING:
+        return _no(Resolution.ABSENT, f"{locator.pointer} does not resolve in {path.name}")
+    if isinstance(node, (dict, list)):
+        return _no(
+            Resolution.NOT_SCALAR, f"{locator.pointer} holds a {type(node).__name__}, not a value"
+        )
+    if node is None:
+        return _no(Resolution.ABSENT, f"{locator.pointer} holds null")
+    native = "float" if isinstance(node, Written) else type(node).__name__
+    return _ok(str(node), native, locator.pointer)