rowcanary 1.0.0__py3-none-any.whl

rowcanary/__init__.py

@@ -0,0 +1,25 @@
+"""Detect silent changes in imported and exported data using control samples."""
+
+__version__ = "1.0.0"
+
+from .core import PROBES, ProbeFileError, Report, check_file, write_probes
+from .core import ID_COLUMN, VALUE_COLUMN, Issue, Probe
+from .config import Config, load_config
+from .probes import select_probes
+from .reporting import html_report
+
+__all__ = [
+    "PROBES",
+    "ID_COLUMN",
+    "VALUE_COLUMN",
+    "ProbeFileError",
+    "Probe",
+    "Issue",
+    "Report",
+    "Config",
+    "load_config",
+    "select_probes",
+    "html_report",
+    "check_file",
+    "write_probes",
+]

rowcanary/__main__.py

@@ -0,0 +1,4 @@
+from .cli import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

rowcanary/cli.py

@@ -0,0 +1,201 @@
+"""Human-friendly command-line interface for RowCanary."""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import sys
+import tempfile
+from pathlib import Path
+from dataclasses import fields, replace
+from typing import TextIO
+
+from . import __version__
+from .core import ProbeFileError, check_file, write_probes, ID_COLUMN, VALUE_COLUMN
+from .config import Config, load_config
+from .formats import FORMATS
+from .reporting import export_report
+
+
+def _print_human(message: str, *, file: TextIO | None = None) -> None:
+    """Keep untrusted text on one line and represent unsupported characters."""
+    stream = file if file is not None else sys.stdout
+    display = "".join(
+        character if character.isprintable() else character.encode("unicode_escape").decode("ascii")
+        for character in message
+    )
+    encoding = getattr(stream, "encoding", None) or "utf-8"
+    print(display.encode(encoding, errors="backslashreplace").decode(encoding), file=stream)
+
+
+def render_report(report, *, output_json: bool = False) -> None:
+    if output_json:
+        print(json.dumps(report.as_dict(), ensure_ascii=True, indent=2))
+        return
+    result = "PASS" if report.ok else "FAIL"
+    _print_human(f"{result}  {report.passed}/{report.total} test values preserved")
+    if report.issues:
+        for issue in report.issues:
+            _print_human(f"  [{issue.code}] {issue.ident}: {issue.reason}")
+            if issue.expected is not None:
+                _print_human(f"    expected: {issue.expected!r}")
+            if issue.actual is not None:
+                _print_human(f"    actual:   {issue.actual!r}")
+    else:
+        _print_human("  No changes detected in the test values.")
+    for observation in report.observations:
+        _print_human(f"  [{observation.severity}:{observation.code}] {observation.reason}")
+    _print_human("  Scope: only the provided control samples, not an entire real dataset.")
+
+
+def demo() -> int:
+    with tempfile.TemporaryDirectory(prefix="rowcanary-") as directory:
+        sample = Path(directory) / "probe.csv"
+        altered = Path(directory) / "altered.csv"
+        write_probes(sample)
+        with sample.open(newline="", encoding="utf-8") as stream:
+            rows = list(csv.DictReader(stream))
+        for row in rows:
+            if row[ID_COLUMN] == "zeroes":
+                row[VALUE_COLUMN] = "12345"
+            elif row[ID_COLUMN] == "scientific":
+                row[VALUE_COLUMN] = "5000000000000"
+            elif row[ID_COLUMN] == "formula":
+                row[VALUE_COLUMN] = "2"
+        with altered.open("w", newline="", encoding="utf-8") as stream:
+            writer = csv.DictWriter(stream, fieldnames=[ID_COLUMN, VALUE_COLUMN])
+            writer.writeheader()
+            writer.writerows(rows)
+        print("DEMO: three sample values were intentionally modified.")
+        render_report(check_file(altered))
+        print("Use 'rowcanary make probe.csv' to test a real import/export process.")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="rowcanary", description="Test whether an import/export process silently changes data."
+    )
+    parser.add_argument("--version", action="version", version=f"rowcanary {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+    maker = sub.add_parser("make", help="Generate a deterministic file of edge-case test values")
+    maker.add_argument("output", type=Path, nargs="?", default=Path("rowcanary-probe.csv"))
+    maker.add_argument(
+        "--delimiter", default=argparse.SUPPRESS, help="CSV field separator (default: comma)"
+    )
+    checker = sub.add_parser("check", help="Audit an exported control file")
+    checker.add_argument("input", type=Path)
+    batch = sub.add_parser("batch", help="Audit several exports and retain every result")
+    batch.add_argument("inputs", type=Path, nargs="+")
+    for command in (maker, checker, batch):
+        command.add_argument("--config", type=Path, help="Reusable JSON configuration file")
+        command.add_argument(
+            "--format", dest="file_format", choices=("auto", *FORMATS), default=argparse.SUPPRESS
+        )
+        command.add_argument(
+            "--encoding",
+            default=argparse.SUPPRESS,
+            help="Text encoding (BOM encodings detected when checking)",
+        )
+        command.add_argument(
+            "--probe-set",
+            default=argparse.SUPPRESS,
+            help="default, extended, none, or comma-separated categories",
+        )
+        command.add_argument(
+            "--custom-values",
+            type=Path,
+            default=argparse.SUPPRESS,
+            help="JSON array of custom {id, value} controls",
+        )
+    for command in (checker, batch):
+        command.add_argument("--id-column", default=argparse.SUPPRESS)
+        command.add_argument("--value-column", default=argparse.SUPPRESS)
+        command.add_argument(
+            "--delimiter", default=argparse.SUPPRESS, help="CSV field separator (default: comma)"
+        )
+        command.add_argument(
+            "--json", action="store_true", help="Print a machine-readable JSON report"
+        )
+        command.add_argument(
+            "--strict-order",
+            action=argparse.BooleanOptionalAction,
+            default=argparse.SUPPRESS,
+            help="Treat reordered rows as an integrity difference",
+        )
+        command.add_argument(
+            "--report", type=Path, help="Create a new standalone .html or .json report"
+        )
+    sub.add_parser("demo", help="Run a self-contained example with intentionally broken values")
+    server = sub.add_parser("serve", help="Open the local-only browser interface")
+    server.add_argument("--port", type=int, default=8765)
+    server.add_argument("--open", action="store_true", help="Open the local URL in your browser")
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "serve":
+            from .web import serve
+
+            return serve(port=args.port, open_browser=args.open)
+        if args.command in ("make", "check", "batch"):
+            config = load_config(args.config) if args.config else Config()
+            overrides = {
+                f.name: getattr(args, f.name) for f in fields(Config) if hasattr(args, f.name)
+            }
+            config = replace(config, **overrides)
+        if args.command == "make":
+            options = config.make_options()
+            probes = options["probes"]
+            write_probes(args.output, **options)
+            _print_human(f"Created {args.output} with {len(probes)} control rows.")
+            _print_human("Import and export it, then run: rowcanary check <exported.csv>")
+            if config.delimiter != ",":
+                _print_human("Use the same --delimiter when checking the exported CSV.")
+            _print_human(
+                "WARNING: Values beginning with '=', '+', or '-' may be evaluated "
+                "by spreadsheet software; use only in isolated test data."
+            )
+            return 0
+        if args.command in ("check", "batch"):
+            options = config.check_options()
+            if args.command == "check":
+                report = check_file(args.input, **options)
+                if args.report:
+                    export_report(args.report, [report])
+                render_report(report, output_json=args.json)
+                return 0 if report.ok else 1
+            results = []
+            status = 0
+            for path in args.inputs:
+                try:
+                    report = check_file(path, **options)
+                    results.append(report.as_dict())
+                    status = max(status, 0 if report.ok else 1)
+                    if not args.json:
+                        _print_human(str(path))
+                        render_report(report)
+                except (ProbeFileError, OSError) as exc:
+                    results.append({"input": str(path), "error": str(exc)})
+                    status = 2
+                    if not args.json:
+                        _print_human(f"{path}: {exc}", file=sys.stderr)
+            if args.report:
+                export_report(args.report, results, batch=True)
+            if args.json:
+                print(
+                    json.dumps(
+                        {"schema_version": "1.0", "results": results}, ensure_ascii=True, indent=2
+                    )
+                )
+            return status
+        return demo()
+    except (ProbeFileError, OSError) as exc:
+        if args.command in ("check", "batch") and args.json:
+            print(json.dumps({"error": str(exc)}, ensure_ascii=True, indent=2))
+        else:
+            _print_human(f"rowcanary: {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

rowcanary/config.py

@@ -0,0 +1,110 @@
+"""Validated, immutable options shared by Python and command-line integrations."""
+
+from __future__ import annotations
+
+import codecs
+import json
+from dataclasses import dataclass, fields
+from pathlib import Path
+
+from .core import ID_COLUMN, VALUE_COLUMN, ProbeFileError, _json_object, _validate_delimiter
+from .formats import FORMATS
+from .probes import EXTENDED, select_probes
+
+MAX_CONFIG_BYTES = 64 * 1024
+
+
+@dataclass(frozen=True)
+class Config:
+    """Reusable options; explicit CLI flags take precedence over loaded values."""
+
+    file_format: str = "auto"
+    encoding: str | None = None
+    delimiter: str = ","
+    id_column: str = ID_COLUMN
+    value_column: str = VALUE_COLUMN
+    probe_set: str = "default"
+    custom_values: Path | None = None
+    strict_order: bool = False
+    schema_version: str = "1.0"
+
+    def __post_init__(self):
+        for name in ("file_format", "delimiter", "id_column", "value_column", "probe_set"):
+            value = getattr(self, name)
+            if not isinstance(value, str) or not value or len(value) > 255:
+                raise ProbeFileError(
+                    f"Configuration {name} must be non-empty text (up to 255 chars)"
+                )
+            try:
+                value.encode("utf-8")
+            except UnicodeError as exc:
+                raise ProbeFileError(f"Configuration {name} must contain valid Unicode") from exc
+        if self.schema_version != "1.0":
+            raise ProbeFileError("Unsupported configuration schema_version")
+        if self.file_format not in ("auto", *FORMATS):
+            raise ProbeFileError("Unknown configuration file_format")
+        _validate_delimiter(self.delimiter)
+        if self.id_column == self.value_column:
+            raise ProbeFileError("ID and value columns must be different")
+        if type(self.strict_order) is not bool:
+            raise ProbeFileError("Configuration strict_order must be a boolean")
+        if self.encoding is not None:
+            if not isinstance(self.encoding, str) or not self.encoding or len(self.encoding) > 64:
+                raise ProbeFileError("Configuration encoding must be text or null")
+            try:
+                codecs.lookup(self.encoding)
+            except (LookupError, ValueError) as exc:
+                raise ProbeFileError("Unknown configuration encoding") from exc
+        if self.probe_set not in ("default", "extended", "none") and not set(
+            self.probe_set.split(",")
+        ) <= {p.category for p in EXTENDED}:
+            raise ProbeFileError("Unknown configuration probe_set or category")
+        if self.custom_values is not None:
+            if not isinstance(self.custom_values, (str, Path)) or not str(self.custom_values):
+                raise ProbeFileError("Configuration custom_values must be a path or null")
+            if "\x00" in str(self.custom_values):
+                raise ProbeFileError("Configuration custom_values cannot contain NUL")
+            object.__setattr__(self, "custom_values", Path(self.custom_values))
+
+    def make_options(self) -> dict:
+        """Keyword arguments for write_probes; generation uses fixed control columns."""
+        return dict(
+            file_format=self.file_format,
+            encoding=self.encoding or "utf-8",
+            delimiter=self.delimiter,
+            probes=select_probes(self.probe_set, self.custom_values),
+        )
+
+    def check_options(self) -> dict:
+        """Keyword arguments for check_file, with BOM-aware decoding by default."""
+        options = self.make_options()
+        options.update(
+            encoding=self.encoding or "utf-8-sig",
+            id_column=self.id_column,
+            value_column=self.value_column,
+            strict_order=self.strict_order,
+        )
+        return options
+
+
+def load_config(path: Path) -> Config:
+    """Read at most 64 KiB of UTF-8 JSON; resolve custom paths beside the config."""
+    path = Path(path)
+    try:
+        with path.open("rb") as stream:
+            data = stream.read(MAX_CONFIG_BYTES + 1)
+        if len(data) > MAX_CONFIG_BYTES:
+            raise ProbeFileError("Configuration exceeds the 64 KiB safety limit")
+        document = json.loads(data.decode("utf-8-sig"), object_pairs_hook=_json_object)
+    except (ValueError, UnicodeError, RecursionError) as exc:
+        raise ProbeFileError(f"Invalid configuration JSON: {exc}") from exc
+    if not isinstance(document, dict) or document.keys() - {f.name for f in fields(Config)}:
+        raise ProbeFileError("Configuration must be an object with known option names")
+    config = Config(**document)
+    if config.custom_values is not None:
+        from dataclasses import replace
+
+        config = replace(
+            config, custom_values=(path.resolve().parent / config.custom_values).resolve()
+        )
+    return config

rowcanary/config.schema.json

@@ -0,0 +1,17 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "RowCanary configuration 1.0",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": {"const": "1.0"},
+    "file_format": {"enum": ["auto", "csv", "tsv", "json", "jsonl", "xlsx"]},
+    "encoding": {"type": ["string", "null"], "minLength": 1, "maxLength": 64},
+    "delimiter": {"type": "string", "minLength": 1, "maxLength": 1, "pattern": "^[^\\r\\n\\u0000\"]$"},
+    "id_column": {"type": "string", "minLength": 1, "maxLength": 255},
+    "value_column": {"type": "string", "minLength": 1, "maxLength": 255},
+    "probe_set": {"type": "string", "minLength": 1, "maxLength": 255},
+    "custom_values": {"type": ["string", "null"], "minLength": 1, "pattern": "^[^\\u0000]+$"},
+    "strict_order": {"type": "boolean"}
+  }
+}