sheet-lint 0.1.0__py3-none-any.whl

sheet_lint/__init__.py

@@ -0,0 +1,38 @@
+"""sheet-lint - find the quiet defects in a CSV or Excel workbook.
+
+The command-line tool is the product; this module is the same audit exposed as
+a library so it can be run from a script or a notebook::
+
+    from sheet_lint import lint_file
+
+    result = lint_file("book.xlsx")
+    for finding in result.findings:
+        print(finding.severity, finding.code, finding.summary)
+        for location in finding.locations:
+            print("   ", location.render())
+
+Standard library only.  No dependencies, no network, no state.
+"""
+
+from __future__ import annotations
+
+from .engine import LintResult, SheetSummary, lint_file, lint_sheets
+from .errors import SheetLintError
+from .model import Finding, Location, Severity
+from .report import render_json, render_text
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "__version__",
+    "lint_file",
+    "lint_sheets",
+    "LintResult",
+    "SheetSummary",
+    "Finding",
+    "Location",
+    "Severity",
+    "SheetLintError",
+    "render_text",
+    "render_json",
+]

sheet_lint/__main__.py

@@ -0,0 +1,10 @@
+"""Entry point for ``python3 -m sheet_lint``."""
+
+from __future__ import annotations
+
+import sys
+
+from .cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

sheet_lint/checks/__init__.py

@@ -0,0 +1,163 @@
+"""The registry of checks.
+
+Each check is a small function of a :class:`~sheet_lint.context.SheetContext`
+that returns findings.  Keeping the registry in one place means the CLI can
+list exactly what it runs, and the README can be checked against it.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, List, Sequence, Tuple
+
+from ..context import SheetContext
+from ..model import Finding
+from . import duplicates, formulas, headers, keys, rows, types
+
+__all__ = ["Check", "CHECKS", "checks_for", "run_checks", "XLSX_ONLY_CODES"]
+
+Formats = Tuple[str, ...]
+CheckFunction = Callable[[SheetContext], List[Finding]]
+
+
+@dataclass(frozen=True)
+class Check:
+    """One named check and the file formats it can run on."""
+
+    code: str
+    title: str
+    formats: Formats
+    run: CheckFunction
+
+    def applies_to(self, kind: str) -> bool:
+        """True when this check can run against a sheet of the given kind."""
+        return kind in self.formats
+
+
+BOTH: Formats = ("csv", "xlsx")
+XLSX: Formats = ("xlsx",)
+
+CHECKS: Tuple[Check, ...] = (
+    Check(
+        "duplicate_key", "Duplicate identifier values", BOTH,
+        duplicates.check_duplicate_key,
+    ),
+    Check(
+        "duplicate_rows", "Whole rows repeated", BOTH,
+        duplicates.check_exact_duplicate_rows,
+    ),
+    Check(
+        "near_duplicate_records", "Near-duplicate records", BOTH,
+        duplicates.check_near_duplicates,
+    ),
+    Check(
+        "missing_key_column", "No identifier column", BOTH,
+        keys.check_missing_key_column,
+    ),
+    Check(
+        "blank_key_values", "Records with no identifier", BOTH,
+        keys.check_blank_key_values,
+    ),
+    Check(
+        "padded_key_values", "Identifier values with stray whitespace", BOTH,
+        keys.check_padded_key_values,
+    ),
+    Check(
+        "inconsistent_key_format", "Identifier column with mixed conventions",
+        BOTH, keys.check_inconsistent_key_format,
+    ),
+    Check(
+        "key_sequence_gap", "Gaps in the identifier sequence", BOTH,
+        keys.check_key_sequence_gaps,
+    ),
+    Check(
+        "numbers_stored_as_text", "Numbers held as text", BOTH,
+        types.check_numbers_stored_as_text,
+    ),
+    Check(
+        "mixed_types_in_column", "Mixed types in one column", BOTH,
+        types.check_mixed_types,
+    ),
+    Check(
+        "rows_after_gap", "Blank rows with data underneath", BOTH,
+        rows.check_rows_after_gap,
+    ),
+    Check(
+        "total_row_not_last", "A total row is not the last row", BOTH,
+        rows.check_total_row_not_last,
+    ),
+    Check(
+        "hidden_rows", "Hidden rows or columns", XLSX,
+        rows.check_hidden_rows,
+    ),
+    Check(
+        "blank_header", "Columns with data and no header", BOTH,
+        headers.check_blank_headers,
+    ),
+    Check(
+        "duplicate_header", "Two columns with the same header", BOTH,
+        headers.check_duplicate_headers,
+    ),
+    Check(
+        "header_row_missing", "No header row", BOTH,
+        headers.check_header_row_missing,
+    ),
+    Check(
+        "header_content_mismatch", "Header does not match the column", BOTH,
+        headers.check_header_content_mismatch,
+    ),
+    Check(
+        "possible_title_row", "A title row above the header row", BOTH,
+        headers.check_possible_title_row,
+    ),
+    Check(
+        "range_stops_short", "A formula range stops before the data ends", XLSX,
+        formulas.check_range_stops_short,
+    ),
+    Check(
+        "hardcoded_in_formula_column",
+        "A typed value inside a calculated column", XLSX,
+        formulas.check_hardcoded_in_formula_column,
+    ),
+    Check(
+        "literal_constant_in_formulas",
+        "A constant typed into several formulas", XLSX,
+        formulas.check_literal_constant_in_formulas,
+    ),
+)
+
+#: Codes that only ever fire on a workbook.  Listed in the report so a CSV
+#: result does not look more complete than it is.
+XLSX_ONLY_CODES: Tuple[str, ...] = tuple(
+    check.code for check in CHECKS if check.formats == XLSX
+)
+
+
+def checks_for(kind: str) -> List[Check]:
+    """Every check that can run against a sheet of the given kind."""
+    return [check for check in CHECKS if check.applies_to(kind)]
+
+
+def run_checks(ctx: SheetContext) -> List[Finding]:
+    """Run every applicable check against one sheet.
+
+    A check that raises is not allowed to take the report down with it: it is
+    recorded as a note and the rest of the audit continues.
+    """
+    findings: List[Finding] = []
+    for check in checks_for(ctx.sheet.kind):
+        try:
+            findings.extend(check.run(ctx))
+        except Exception as exc:  # pragma: no cover - defensive
+            ctx.add_note(
+                "check %s could not complete on sheet %r: %s: %s"
+                % (check.code, ctx.name, type(exc).__name__, exc)
+            )
+    return findings
+
+
+def check_rows() -> Sequence[Tuple[str, str, str]]:
+    """``(code, formats, title)`` for every check, for ``--list-checks``."""
+    return [
+        (check.code, ", ".join(check.formats), check.title) for check in CHECKS
+    ]

sheet_lint/checks/duplicates.py

@@ -0,0 +1,303 @@
+"""Duplicate detection.
+
+Duplicates are counted on the identifier column, not on rows.  "48 rows, 43
+invoices" is a fact a client can act on; "48 rows" is not.  When a sheet has no
+identifier column at all, the fallback is whole-row duplicates, and the report
+says which rule was used.
+"""
+
+from __future__ import annotations
+
+import difflib
+import re
+from typing import Dict, List, Optional, Tuple
+
+from .. import infer
+from ..context import (
+    MAX_LOCATIONS,
+    MAX_PAIR_COMPARISONS,
+    NEAR_DUPLICATE_RATIO,
+    SheetContext,
+)
+from ..model import Finding, Location, Severity
+from ..textutil import plural, verb
+
+__all__ = [
+    "check_duplicate_key",
+    "check_exact_duplicate_rows",
+    "check_near_duplicates",
+]
+
+_TOKEN = re.compile(r"[a-z0-9]+")
+
+#: Rows must share this much of their vocabulary before the slow comparison
+#: runs, and their lengths must be within this fraction of each other.
+_JACCARD_FLOOR = 0.40
+_LENGTH_TOLERANCE = 0.25
+
+#: A row with fewer than this many distinct tokens is too thin to compare.
+_MIN_TOKENS = 2
+
+
+def _key_cells(ctx: SheetContext) -> List[Tuple[int, str]]:
+    """Return ``(row, text)`` for every populated identifier cell."""
+    assert ctx.key is not None
+    cells: List[Tuple[int, str]] = []
+    for row in ctx.sheet.data_row_numbers():
+        cell = ctx.sheet.cell(row, ctx.key.index)
+        if cell is None or cell.is_empty:
+            continue
+        cells.append((row, cell.text))
+    return cells
+
+
+def check_duplicate_key(ctx: SheetContext) -> List[Finding]:
+    """Report rows that repeat an identifier already used by another row."""
+    key = ctx.key
+    if key is None:
+        return []
+    cells = _key_cells(ctx)
+    if len(cells) < 2:
+        return []
+
+    groups: Dict[str, List[Tuple[int, str]]] = {}
+    for row, text in cells:
+        groups.setdefault(infer.identity_key(text), []).append((row, text))
+
+    repeated = [entries for entries in groups.values() if len(entries) > 1]
+    if not repeated:
+        return []
+    repeated.sort(key=lambda entries: entries[0][0])
+    extra_rows = sum(len(entries) - 1 for entries in repeated)
+
+    locations: List[Location] = []
+    for entries in repeated:
+        first_row, first_raw = entries[0]
+        for row, raw in entries[1:]:
+            detail = "key %r repeats the value at %s" % (
+                raw,
+                ctx.cell_ref(first_row, key.index),
+            )
+            if infer.fold(raw) != infer.fold(first_raw):
+                detail += " (written differently: %r)" % (first_raw,)
+            locations.append(
+                Location(
+                    sheet=ctx.name,
+                    ref=ctx.cell_ref(row, key.index),
+                    detail=detail,
+                    row=row,
+                )
+            )
+            if len(locations) >= MAX_LOCATIONS:
+                break
+        if len(locations) >= MAX_LOCATIONS:
+            break
+
+    return [
+        Finding(
+            code="duplicate_key",
+            title="Duplicate values in the identifier column",
+            severity=Severity.HIGH,
+            sheet=ctx.name,
+            summary=(
+                "%d populated %s values collapse to %d distinct identifiers; "
+                "%s %s a key that is already in use"
+                % (
+                    len(cells),
+                    key.header or key.letter,
+                    len(groups),
+                    plural(extra_rows, "row"),
+                    verb(extra_rows, "repeats", "repeat"),
+                )
+            ),
+            locations=locations,
+            why=(
+                "Every total, join and export that reads this column counts the "
+                "repeated key twice, so one record quietly becomes two."
+            ),
+            count=extra_rows,
+        )
+    ]
+
+
+def check_exact_duplicate_rows(ctx: SheetContext) -> List[Finding]:
+    """Report byte-identical rows, used only when there is no identifier column."""
+    if ctx.key is not None:
+        return []
+    sheet = ctx.sheet
+    groups: Dict[str, List[int]] = {}
+    for row in sheet.data_row_numbers():
+        if sheet.row_is_blank(row):
+            continue
+        signature = "\x1f".join(
+            infer.fold(sheet.cell(row, col).text)
+            for col in range(1, sheet.n_cols + 1)
+            if sheet.cell(row, col) is not None and not sheet.cell(row, col).is_empty
+        )
+        if not signature.strip("\x1f"):
+            continue
+        groups.setdefault(signature, []).append(row)
+
+    repeated = [rows for rows in groups.values() if len(rows) > 1]
+    if not repeated:
+        return []
+    repeated.sort(key=lambda rows: rows[0])
+    extra_rows = sum(len(rows) - 1 for rows in repeated)
+    locations: List[Location] = []
+    for rows in repeated:
+        first = rows[0]
+        for row in rows[1:]:
+            locations.append(
+                Location(
+                    sheet=ctx.name,
+                    ref=ctx.row_ref(row),
+                    detail="identical to row %d in every column" % first,
+                    row=row,
+                )
+            )
+            if len(locations) >= MAX_LOCATIONS:
+                break
+        if len(locations) >= MAX_LOCATIONS:
+            break
+
+    return [
+        Finding(
+            code="duplicate_rows",
+            title="Whole rows repeated",
+            severity=Severity.HIGH,
+            sheet=ctx.name,
+            summary=(
+                "%s %s another row in all %d populated columns, once case and "
+                "spacing are ignored"
+                % (
+                    plural(extra_rows, "row"),
+                    verb(extra_rows, "matches", "match"),
+                    sheet.n_cols,
+                )
+            ),
+            locations=locations,
+            why=(
+                "With no identifier column there is nothing to count records on, "
+                "so repeated rows are counted twice by anything that totals this "
+                "sheet."
+            ),
+            count=extra_rows,
+        )
+    ]
+
+
+def _record_text(ctx: SheetContext, row: int, skip_col: Optional[int]) -> str:
+    """Fold one row into a comparable string, ignoring the identifier column."""
+    parts: List[str] = []
+    for col in range(1, ctx.sheet.n_cols + 1):
+        if col == skip_col:
+            continue
+        cell = ctx.sheet.cell(row, col)
+        if cell is None or cell.is_empty:
+            continue
+        parts.append(infer.fold(cell.text))
+    return " ".join(part for part in parts if part)
+
+
+def check_near_duplicates(ctx: SheetContext) -> List[Finding]:
+    """Report rows that are nearly identical outside the identifier column.
+
+    Comparison is quadratic in the worst case, so rows are first blocked by
+    vocabulary overlap and length; the pair budget is finite and, if it runs
+    out, the report says so instead of quietly returning a short answer.
+    """
+    sheet = ctx.sheet
+    skip_col = ctx.key.index if ctx.key is not None else None
+    records: List[Tuple[int, str, str, frozenset]] = []
+    for row in sheet.data_row_numbers():
+        if sheet.row_is_blank(row):
+            continue
+        text = _record_text(ctx, row, skip_col)
+        tokens = frozenset(_TOKEN.findall(text))
+        if len(tokens) < _MIN_TOKENS:
+            continue
+        key_text = ""
+        if ctx.key is not None:
+            cell = sheet.cell(row, ctx.key.index)
+            key_text = infer.identity_key(cell.text) if cell is not None else ""
+        records.append((row, text, key_text, tokens))
+
+    if len(records) < 2:
+        return []
+
+    pairs: List[Tuple[int, int, float]] = []
+    comparisons = 0
+    truncated = False
+    for i in range(len(records) - 1):
+        row_a, text_a, key_a, tokens_a = records[i]
+        for j in range(i + 1, len(records)):
+            row_b, text_b, key_b, tokens_b = records[j]
+            if comparisons >= MAX_PAIR_COMPARISONS:
+                truncated = True
+                break
+            comparisons += 1
+            if key_a and key_a == key_b:
+                continue  # already reported as a duplicate key
+            if ctx.key is None and text_a == text_b:
+                continue  # a whole-row duplicate, which duplicate_rows reports
+            longest = max(len(text_a), len(text_b)) or 1
+            if abs(len(text_a) - len(text_b)) / float(longest) > _LENGTH_TOLERANCE:
+                continue
+            union = tokens_a | tokens_b
+            if not union:
+                continue
+            overlap = len(tokens_a & tokens_b) / float(len(union))
+            if overlap < _JACCARD_FLOOR:
+                continue
+            matcher = difflib.SequenceMatcher(None, text_a, text_b)
+            if matcher.real_quick_ratio() < NEAR_DUPLICATE_RATIO:
+                continue
+            ratio = matcher.ratio()
+            if ratio >= NEAR_DUPLICATE_RATIO:
+                pairs.append((row_a, row_b, ratio))
+        if truncated:
+            break
+
+    if truncated:
+        ctx.add_note(
+            "near-duplicate scan stopped after %d row comparisons; %d rows were "
+            "compared and the tail of the sheet was not cross-checked"
+            % (MAX_PAIR_COMPARISONS, len(records))
+        )
+    if not pairs:
+        return []
+
+    pairs.sort(key=lambda pair: (-pair[2], pair[0], pair[1]))
+    locations = [
+        Location(
+            sheet=ctx.name,
+            ref=ctx.row_ref(row_a),
+            detail="%d%% identical to row %d outside the identifier column"
+            % (round(ratio * 100), row_b),
+            row=row_a,
+        )
+        for row_a, row_b, ratio in pairs[:MAX_LOCATIONS]
+    ]
+    return [
+        Finding(
+            code="near_duplicate_records",
+            title="Near-duplicate records",
+            severity=Severity.MEDIUM,
+            sheet=ctx.name,
+            summary=(
+                "%s %s at %d%% or better outside the identifier column (case, "
+                "spacing and punctuation ignored)"
+                % (
+                    plural(len(pairs), "row pair"),
+                    verb(len(pairs), "matches", "match"),
+                    round(NEAR_DUPLICATE_RATIO * 100),
+                )
+            ),
+            locations=locations,
+            why=(
+                "Two spellings of one record are two records to every total and "
+                "every join, so one customer's spend is split across two rows."
+            ),
+            count=len(pairs),
+        )
+    ]