sensorlint 0.1.0__py3-none-any.whl

sensorlint/__init__.py

@@ -0,0 +1,162 @@
+"""sensorlint: assertions for sensor-data pipelines that abort instead of warning.
+
+Sensor pipelines do not usually fail by crashing. They fail by producing
+believable numbers. A truncated payload becomes a short file. Raw converter
+counts become a physical measurement off by a constant factor. A swapped pair
+of channels becomes a trend on the wrong location. A stage that processed zero
+records becomes a green build.
+
+Every check here raises. There is no warning mode, because a warning in a
+batch job is a log line nobody reads.
+
+Quickstart::
+
+    import numpy as np
+    from sensorlint import assert_expected_length, assert_not_clipped, safe_gunzip
+
+    payload = safe_gunzip(raw_bytes)          # refuses to return partial data
+    samples = np.frombuffer(payload, dtype=np.int16)
+    assert_expected_length(samples, 20480)
+    assert_not_clipped(samples, adc_max=32767, leading_samples=2048)
+
+Every ``assert_*`` has a ``check_*`` twin that returns a
+:class:`~sensorlint.result.CheckResult` instead of raising, so results can be
+aggregated across a fleet with :mod:`sensorlint.report`.
+"""
+
+from __future__ import annotations
+
+from sensorlint.checks import (
+    DecodeProbe,
+    RawCountsVerdict,
+    RunLengthReport,
+    StageCounter,
+    TranspositionVerdict,
+    assert_channels_not_transposed,
+    assert_coverage,
+    assert_decoded_fully,
+    assert_expected_length,
+    assert_nonzero_kept,
+    assert_not_clipped,
+    assert_not_stale,
+    assert_sample_rate,
+    assert_scale_declared,
+    autodetect_rails,
+    check_channel_transposition,
+    check_coverage,
+    check_decoded_fully,
+    check_expected_length,
+    check_nonzero_kept,
+    check_not_clipped,
+    check_not_stale,
+    check_sample_rate,
+    check_scale_declared,
+    coverage_histogram,
+    detect_channel_transposition,
+    estimate_expected_ratio_db,
+    estimate_sample_rate,
+    longest_frozen_run,
+    longest_interpolated_run,
+    looks_like_raw_counts,
+    measure_length,
+    nonzero_kept,
+    pipeline_stage,
+    probe_compressed,
+    safe_gunzip,
+    sniff_container,
+    transposition_run_lengths,
+)
+from sensorlint.errors import (
+    ClippingError,
+    CoverageError,
+    DecodeError,
+    EmptyStageError,
+    LengthError,
+    SampleRateError,
+    ScaleError,
+    SensorLintError,
+    StalenessError,
+    TranspositionError,
+)
+from sensorlint.report import (
+    ReportSummary,
+    format_report,
+    format_table,
+    summarize,
+    worst_offenders,
+)
+from sensorlint.result import CheckResult, Severity
+
+__version__ = "0.1.0"
+
+__all__ = [
+    # result and reporting
+    "CheckResult",
+    "ReportSummary",
+    "Severity",
+    "format_report",
+    "format_table",
+    "summarize",
+    "worst_offenders",
+    # errors
+    "ClippingError",
+    "CoverageError",
+    "DecodeError",
+    "EmptyStageError",
+    "LengthError",
+    "SampleRateError",
+    "ScaleError",
+    "SensorLintError",
+    "StalenessError",
+    "TranspositionError",
+    # supporting types
+    "DecodeProbe",
+    "RawCountsVerdict",
+    "RunLengthReport",
+    "StageCounter",
+    "TranspositionVerdict",
+    # 1. decode
+    "assert_decoded_fully",
+    "check_decoded_fully",
+    "probe_compressed",
+    "safe_gunzip",
+    "sniff_container",
+    # 2. length
+    "assert_expected_length",
+    "check_expected_length",
+    "measure_length",
+    # 3. clipping
+    "assert_not_clipped",
+    "autodetect_rails",
+    "check_not_clipped",
+    # 4. scale
+    "assert_scale_declared",
+    "check_scale_declared",
+    "looks_like_raw_counts",
+    # 5. staleness
+    "assert_not_stale",
+    "check_not_stale",
+    "longest_frozen_run",
+    "longest_interpolated_run",
+    # 6. sample rate
+    "assert_sample_rate",
+    "check_sample_rate",
+    "estimate_sample_rate",
+    # 7. transposition
+    "assert_channels_not_transposed",
+    "check_channel_transposition",
+    "detect_channel_transposition",
+    "estimate_expected_ratio_db",
+    "transposition_run_lengths",
+    # 8. coverage
+    "assert_coverage",
+    "check_coverage",
+    "coverage_histogram",
+    # 9. nonzero kept
+    "assert_nonzero_kept",
+    "check_nonzero_kept",
+    "nonzero_kept",
+    "pipeline_stage",
+    # version
+    "__version__",
+]

sensorlint/__main__.py

@@ -0,0 +1,377 @@
+"""Command-line entry point: ``python -m sensorlint check <file>``.
+
+Deliberately small. The CLI exists so a check can be dropped into a shell
+pipeline or a CI step without writing Python, and so that the exit code
+carries the verdict. It runs whichever checks the flags you passed make
+applicable, prints a report, and exits non-zero if anything failed.
+
+If a flag is not supplied, the corresponding check is skipped and the report
+says so. It does not guess thresholds on your behalf.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any, List, Optional, Sequence, Tuple
+
+import numpy as np
+
+from sensorlint import __version__
+from sensorlint.checks.clipping import check_not_clipped
+from sensorlint.checks.length import check_expected_length
+from sensorlint.checks.sample_rate import check_sample_rate
+from sensorlint.checks.scale import check_scale_declared
+from sensorlint.checks.staleness import check_not_stale
+from sensorlint.report import format_report, summarize
+from sensorlint.result import CheckResult, Severity
+
+__all__ = ["build_parser", "load_series", "main", "run_checks"]
+
+_EXIT_OK = 0
+_EXIT_FAILED = 1
+_EXIT_USAGE = 2
+
+
+def load_series(
+    path: Path,
+    *,
+    column: int = 0,
+    timestamp_column: Optional[int] = None,
+    delimiter: str = ",",
+    skip_header: int = 0,
+) -> Tuple[Any, Optional[Any]]:
+    """Load ``(values, timestamps)`` from a ``.npy`` or delimited text file.
+
+    Args:
+        path: File to read. ``.npy`` is loaded with :func:`numpy.load`;
+            anything else is parsed as delimited text.
+        column: Which column holds the sample values, for 2-D input.
+        timestamp_column: Which column holds timestamps, if any.
+        delimiter: Field separator for text files.
+        skip_header: Header rows to skip in text files.
+
+    Returns:
+        ``(values, timestamps)`` where ``timestamps`` is ``None`` unless
+        ``timestamp_column`` was given.
+
+    Raises:
+        ValueError: if the file cannot be interpreted as a numeric series,
+            or the requested columns do not exist.
+    """
+    if path.suffix.lower() == ".npy":
+        data = np.load(path, allow_pickle=False)
+    else:
+        data = np.genfromtxt(
+            path, delimiter=delimiter, skip_header=skip_header, dtype=np.float64
+        )
+
+    array = np.asarray(data)
+    if array.ndim == 0:
+        raise ValueError(f"{path} holds a scalar, not a series")
+
+    if array.ndim == 1:
+        if timestamp_column is not None:
+            raise ValueError(
+                f"{path} has one column; --timestamp-column needs a 2-D file"
+            )
+        if column != 0:
+            raise ValueError(f"{path} has one column; --column {column} is out of range")
+        return array, None
+
+    if array.ndim > 2:
+        raise ValueError(f"{path} has {array.ndim} dimensions; expected 1 or 2")
+
+    n_cols = array.shape[1]
+    if not 0 <= column < n_cols:
+        raise ValueError(f"--column {column} is out of range for {n_cols} column(s)")
+    values = array[:, column]
+
+    timestamps = None
+    if timestamp_column is not None:
+        if not 0 <= timestamp_column < n_cols:
+            raise ValueError(
+                f"--timestamp-column {timestamp_column} is out of range for "
+                f"{n_cols} column(s)"
+            )
+        timestamps = array[:, timestamp_column]
+
+    return values, timestamps
+
+
+def run_checks(args: argparse.Namespace) -> List[CheckResult]:
+    """Run every check the supplied flags make applicable.
+
+    Returns:
+        The results, in the order the checks ran.
+    """
+    path = Path(args.path)
+    values, timestamps = load_series(
+        path,
+        column=args.column,
+        timestamp_column=args.timestamp_column,
+        delimiter=args.delimiter,
+        skip_header=args.skip_header,
+    )
+    target = path.name
+    results: List[CheckResult] = []
+
+    if args.expected_length is not None:
+        results.append(
+            check_expected_length(
+                values,
+                args.expected_length,
+                args.length_tolerance,
+                target=target,
+            )
+        )
+
+    if args.adc_max is not None or args.adc_min is not None or args.autodetect_rails:
+        results.append(
+            check_not_clipped(
+                values,
+                adc_max=args.adc_max,
+                adc_min=args.adc_min,
+                max_fraction=args.max_clipped_fraction,
+                leading_samples=args.leading_samples,
+                leading_max_fraction=args.leading_max_fraction,
+                target=target,
+            )
+        )
+
+    if not args.no_scale_check:
+        results.append(
+            check_scale_declared(
+                values, args.scale, args.unit, target=target
+            )
+        )
+
+    if not args.no_stale_check:
+        results.append(
+            check_not_stale(
+                values,
+                max_frozen_run=args.max_frozen_run,
+                max_interpolated_run=args.max_interpolated_run,
+                target=target,
+            )
+        )
+
+    if args.fs is not None:
+        if timestamps is None:
+            # No timestamp column: synthesise the times the claimed rate
+            # implies. That makes the rate check trivially true, so it is
+            # skipped rather than faked.
+            results.append(
+                CheckResult(
+                    check="sample_rate",
+                    passed=True,
+                    severity=Severity.WARNING,
+                    message=(
+                        f"skipped: --fs {args.fs} given but no --timestamp-column, "
+                        "so there are no observed intervals to compare against"
+                    ),
+                    details={"skipped": True, "claimed_fs": args.fs},
+                    target=target,
+                )
+            )
+        else:
+            results.append(
+                check_sample_rate(
+                    timestamps,
+                    args.fs,
+                    tolerance_pct=args.fs_tolerance_pct,
+                    allow_gaps=args.allow_gaps,
+                    unit=args.time_unit,
+                    target=target,
+                )
+            )
+
+    return results
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Construct the argument parser."""
+    parser = argparse.ArgumentParser(
+        prog="sensorlint",
+        description=(
+            "Assertions for sensor-data pipelines. Every check aborts rather "
+            "than warns."
+        ),
+    )
+    parser.add_argument("--version", action="version", version=f"sensorlint {__version__}")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    check = sub.add_parser(
+        "check",
+        help="run the applicable checks against a .npy or delimited text file",
+        description=(
+            "Runs the checks your flags make applicable and exits non-zero if any "
+            "fail. Checks with no corresponding flag are skipped rather than run "
+            "with a guessed threshold."
+        ),
+    )
+    check.add_argument("path", help="path to a .npy or delimited text file")
+
+    load = check.add_argument_group("loading")
+    load.add_argument(
+        "--column", type=int, default=0, help="column holding sample values (default 0)"
+    )
+    load.add_argument(
+        "--timestamp-column",
+        type=int,
+        default=None,
+        help="column holding timestamps; required for --fs to do anything",
+    )
+    load.add_argument("--delimiter", default=",", help="text field separator (default ,)")
+    load.add_argument(
+        "--skip-header", type=int, default=0, help="header rows to skip in text files"
+    )
+    load.add_argument(
+        "--time-unit",
+        default="s",
+        choices=["s", "ms", "us", "ns"],
+        help="unit of the timestamp column (default s)",
+    )
+
+    length = check.add_argument_group("length")
+    length.add_argument(
+        "--expected-length", type=int, default=None, help="required sample count"
+    )
+    length.add_argument(
+        "--length-tolerance",
+        type=int,
+        default=0,
+        help="allowed absolute deviation in samples (default 0)",
+    )
+
+    clip = check.add_argument_group("clipping")
+    clip.add_argument("--adc-max", type=float, default=None, help="positive rail value")
+    clip.add_argument("--adc-min", type=float, default=None, help="negative rail value")
+    clip.add_argument(
+        "--autodetect-rails",
+        action="store_true",
+        help="run the clipping check with rails inferred from repeated extremes",
+    )
+    clip.add_argument(
+        "--max-clipped-fraction",
+        type=float,
+        default=0.001,
+        help="whole-array railed fraction allowed (default 0.001)",
+    )
+    clip.add_argument(
+        "--leading-samples",
+        type=int,
+        default=None,
+        help="also test this many leading samples separately",
+    )
+    clip.add_argument(
+        "--leading-max-fraction",
+        type=float,
+        default=None,
+        help="railed fraction allowed in the leading window",
+    )
+
+    scale = check.add_argument_group("scale")
+    scale.add_argument("--scale", type=float, default=None, help="counts-to-physical factor")
+    scale.add_argument("--unit", default=None, help="physical unit, e.g. g or mm/s")
+    scale.add_argument(
+        "--no-scale-check", action="store_true", help="skip the scale-declared check"
+    )
+
+    stale = check.add_argument_group("staleness")
+    stale.add_argument(
+        "--max-frozen-run", type=int, default=10, help="longest repeated-value run allowed"
+    )
+    stale.add_argument(
+        "--max-interpolated-run",
+        type=int,
+        default=12,
+        help="longest constant-slope run allowed",
+    )
+    stale.add_argument(
+        "--no-stale-check", action="store_true", help="skip the staleness check"
+    )
+
+    rate = check.add_argument_group("sample rate")
+    rate.add_argument("--fs", type=float, default=None, help="claimed sampling rate in Hz")
+    rate.add_argument(
+        "--fs-tolerance-pct",
+        type=float,
+        default=1.0,
+        help="allowed rate error, percent (default 1.0)",
+    )
+    rate.add_argument(
+        "--allow-gaps", action="store_true", help="tolerate gaps in the timestamp series"
+    )
+
+    out = check.add_argument_group("output")
+    out.add_argument("--json", action="store_true", help="emit JSON instead of a table")
+    out.add_argument(
+        "--quiet", action="store_true", help="print nothing; rely on the exit code"
+    )
+
+    return parser
+
+
+def main(argv: Optional[Sequence[str]] = None) -> int:
+    """Run the CLI.
+
+    Returns:
+        ``0`` when every applicable check passed, ``1`` when any failed,
+        ``2`` on a usage or loading error.
+    """
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    try:
+        results = run_checks(args)
+    except (OSError, ValueError, TypeError) as exc:
+        print(f"sensorlint: {exc}", file=sys.stderr)
+        return _EXIT_USAGE
+
+    summary = summarize(results)
+
+    if args.json:
+        payload = {
+            "summary": summary.to_dict(),
+            "results": [r.to_dict() for r in results],
+        }
+        print(json.dumps(payload, indent=2, default=_json_default))
+    elif not args.quiet:
+        print(format_report(results, title=f"sensorlint {Path(args.path).name}"))
+        # A skipped check counts as a pass in the table, which would let it
+        # hide inside a green run. Say so explicitly instead.
+        skipped = [r for r in results if r.details.get("skipped")]
+        if skipped:
+            print("")
+            print("skipped:")
+            for item in skipped:
+                print(f"  {item.check}: {item.message.removeprefix('skipped: ')}")
+
+    if not results and not args.quiet and not args.json:
+        print(
+            "\nno checks ran: pass at least one of --expected-length, --adc-max, "
+            "--autodetect-rails or --fs",
+            file=sys.stderr,
+        )
+
+    return _EXIT_OK if summary.clean else _EXIT_FAILED
+
+
+def _json_default(value: Any) -> Any:
+    """Coerce numpy scalars so ``json.dumps`` can serialise check details."""
+    if isinstance(value, (np.integer,)):
+        return int(value)
+    if isinstance(value, (np.floating,)):
+        return float(value)
+    if isinstance(value, (np.bool_,)):
+        return bool(value)
+    if isinstance(value, np.ndarray):
+        return value.tolist()
+    return str(value)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())