sciproof 0.1.0__py3-none-any.whl

sciproof/__init__.py

@@ -0,0 +1,7 @@
+"""SciProof: record computational experiments and create verifiable evidence for them.
+
+SciProof checks *integrity* and *execution reproducibility* of recorded runs. It does not,
+and cannot, establish that a scientific result is correct. See docs/CONCEPTS.md.
+"""
+
+__version__ = "0.1.0"

sciproof/__main__.py

@@ -0,0 +1,6 @@
+import sys
+
+from sciproof.cli import main
+
+if __name__ == "__main__":
+    sys.exit(main())

sciproof/capture.py

@@ -0,0 +1,263 @@
+"""Capture an experiment: run it once and write an evidence manifest."""
+
+from __future__ import annotations
+
+import math
+import os
+import shutil
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from sciproof import __version__
+from sciproof.environment import interpreter_info, platform_info, selected_env_vars
+from sciproof.errors import CaptureError, SciProofError, UnsafePathError
+from sciproof.gitinfo import git_info
+from sciproof.hashing import sha256_file
+from sciproof.manifest import (
+    MANIFEST_SUFFIX,
+    NAME_RE,
+    SCHEMA,
+    SCHEMA_VERSION,
+    snapshot_dir_name,
+    write_manifest,
+)
+from sciproof.numeric import NumericFormatError, load_numeric
+from sciproof.paths import to_manifest_path
+from sciproof.runner import (
+    CommandInterrupted,
+    CommandNotRunnable,
+    CommandTimeout,
+    run_command,
+    summarize_stream,
+)
+
+DEFAULT_MAX_LOG_BYTES = 64 * 1024
+DEFAULT_RTOL = 1e-9
+DEFAULT_ATOL = 0.0
+
+
+@dataclass
+class CaptureResult:
+    manifest_path: Path
+    manifest: dict[str, Any]
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def exit_code(self) -> int:
+        return int(self.manifest["execution"]["exit_code"])
+
+
+def _hash_inputs(base: Path, inputs: list[str]) -> list[dict[str, Any]]:
+    records = []
+    for relative in inputs:
+        path = base / relative
+        if path.is_dir():
+            raise CaptureError(
+                f"input {relative!r} is a directory; only files can be declared "
+                "(directory support is tracked in issue #5)"
+            )
+        if not path.is_file():
+            raise CaptureError(f"input file not found: {relative}")
+        digest, size = sha256_file(path)
+        records.append({"path": relative, "sha256": digest, "size": size})
+    return records
+
+
+def _unique(items: list[str], what: str) -> None:
+    if len(set(items)) != len(items):
+        raise CaptureError(f"{what} contains the same path more than once")
+
+
+def capture(
+    *,
+    name: str,
+    argv: list[str],
+    base: Path,
+    inputs: list[str] | None = None,
+    outputs: list[str] | None = None,
+    numeric_outputs: list[str] | None = None,
+    rtol: float = DEFAULT_RTOL,
+    atol: float = DEFAULT_ATOL,
+    timeout: float | None = None,
+    max_log_bytes: int = DEFAULT_MAX_LOG_BYTES,
+    force: bool = False,
+) -> CaptureResult:
+    """Run ``argv`` once in ``base`` and write ``<name>.sciproof.json`` there.
+
+    Paths in ``inputs``/``outputs``/``numeric_outputs`` are user paths relative to ``base``
+    (or absolute paths inside it). The manifest is only written if everything succeeded;
+    a command that exits with a nonzero status is still recorded.
+    """
+    if not NAME_RE.fullmatch(name):
+        raise CaptureError(
+            f"invalid name {name!r}: use letters, digits, '.', '_' or '-' (max 100 characters)"
+        )
+    if not argv or not argv[0]:
+        raise CaptureError("no command given")
+    if not (math.isfinite(rtol) and math.isfinite(atol)) or rtol < 0 or atol < 0:
+        raise CaptureError("rtol and atol must be finite numbers that are not negative")
+    if timeout is not None and not (math.isfinite(timeout) and timeout > 0):
+        raise CaptureError("timeout must be a positive number of seconds")
+    if max_log_bytes < 0:
+        raise CaptureError("max-log-bytes must not be negative")
+    base = base.resolve()
+
+    try:
+        input_paths = [to_manifest_path(base, p) for p in (inputs or [])]
+        exact_paths = [to_manifest_path(base, p) for p in (outputs or [])]
+        numeric_paths = [to_manifest_path(base, p) for p in (numeric_outputs or [])]
+    except UnsafePathError as exc:
+        raise CaptureError(str(exc)) from exc
+    _unique(input_paths, "--input")
+    _unique(exact_paths + numeric_paths, "--output/--output-numeric")
+    overlap = set(input_paths) & set(exact_paths + numeric_paths)
+    if overlap:
+        raise CaptureError(f"a file cannot be both input and output: {sorted(overlap)[0]}")
+    if not (input_paths or exact_paths or numeric_paths):
+        raise CaptureError(
+            "declare at least one --input, --output or --output-numeric file; "
+            "SciProof only verifies files that were declared"
+        )
+
+    manifest_path = base / f"{name}{MANIFEST_SUFFIX}"
+    if manifest_path.exists() and not force:
+        raise CaptureError(f"{manifest_path.name} already exists (use --force to overwrite it)")
+
+    warnings: list[str] = []
+    input_records = _hash_inputs(base, input_paths)
+    before = {}
+    for relative in exact_paths + numeric_paths:
+        target = base / relative
+        if target.is_dir():
+            raise CaptureError(
+                f"output {relative!r} is a directory; only files can be declared "
+                "(directory support is tracked in issue #5)"
+            )
+        before[relative] = sha256_file(target)[0] if target.is_file() else None
+
+    git = git_info(base)
+    if git and git["dirty"]:
+        warnings.append(
+            "the Git working tree has uncommitted changes, so the commit alone does not "
+            "identify the code that was run"
+        )
+    interpreter, interpreter_source = interpreter_info(argv[0], base)
+
+    try:
+        result = run_command(argv, base, timeout)
+    except CommandInterrupted as exc:
+        raise CaptureError("interrupted; no evidence was written") from exc
+    except CommandTimeout as exc:
+        raise CaptureError(f"{exc}; no evidence was written") from exc
+    except CommandNotRunnable as exc:
+        raise CaptureError(str(exc)) from exc
+
+    # The command must treat declared inputs as read-only, otherwise the evidence is meaningless.
+    for record in input_records:
+        digest, _ = (
+            sha256_file(base / record["path"]) if (base / record["path"]).is_file() else (None, 0)
+        )
+        if digest != record["sha256"]:
+            raise CaptureError(
+                f"the command modified or removed the declared input {record['path']!r}; "
+                "inputs must be read-only. No evidence was written."
+            )
+
+    # Snapshots are written to a staging directory and only replace the old ones once the new
+    # manifest has been written, so a failed capture never damages earlier evidence and
+    # --force never leaves snapshots that the new manifest does not reference.
+    snapshot_dir = base / snapshot_dir_name(name)
+    staging = base / f"{snapshot_dir_name(name)}.tmp"
+    shutil.rmtree(staging, ignore_errors=True)
+    try:
+        output_records: list[dict[str, Any]] = []
+        for relative in exact_paths + numeric_paths:
+            numeric = relative in numeric_paths
+            target = base / relative
+            record: dict[str, Any] = {"path": relative}
+            if target.is_file():
+                digest, size = sha256_file(target)
+                record.update(present=True, sha256=digest, size=size)
+                if before[relative] == digest:
+                    warnings.append(
+                        f"output {relative!r} existed before the run and was not changed by it"
+                    )
+            elif result.exit_code == 0:
+                raise CaptureError(
+                    f"the command succeeded but the declared output {relative!r} does not exist. "
+                    "No evidence was written."
+                )
+            else:
+                record.update(present=False, sha256=None, size=0)
+                if numeric:
+                    raise CaptureError(
+                        f"the command failed and the numeric output {relative!r} does not exist"
+                    )
+
+            if numeric:
+                try:
+                    load_numeric(target)
+                except NumericFormatError as exc:
+                    raise CaptureError(str(exc)) from exc
+                staging.mkdir(exist_ok=True)
+                snapshot_name = f"{len(output_records)}-{target.name}"
+                shutil.copyfile(target, staging / snapshot_name)
+                snapshot_digest, _ = sha256_file(staging / snapshot_name)
+                record["compare"] = {
+                    "mode": "numeric",
+                    "rtol": rtol,
+                    "atol": atol,
+                    "snapshot": {
+                        "path": f"{snapshot_dir_name(name)}/{snapshot_name}",
+                        "sha256": snapshot_digest,
+                    },
+                }
+            else:
+                record["compare"] = {"mode": "exact"}
+            output_records.append(record)
+
+        manifest: dict[str, Any] = {
+            "schema": SCHEMA,
+            "schema_version": SCHEMA_VERSION,
+            "name": name,
+            "created_at": datetime.now(UTC).replace(microsecond=0).isoformat(),
+            "tool": {"name": "sciproof", "version": __version__},
+            "command": {"argv": list(argv), "cwd": "."},
+            "environment": {
+                "source": interpreter_source,
+                "python": {
+                    "version": interpreter["version"],
+                    "implementation": interpreter["implementation"],
+                    "byteorder": interpreter.get("byteorder", ""),
+                },
+                "platform": platform_info(),
+                "env_vars": selected_env_vars(),
+            },
+            "dependencies": [
+                {"name": pkg_name, "version": pkg_version}
+                for pkg_name, pkg_version in interpreter["packages"]
+            ],
+            "git": git,
+            "inputs": input_records,
+            "execution": {
+                "exit_code": result.exit_code,
+                "duration_seconds": round(result.duration_seconds, 3),
+                "stdout": summarize_stream(result.stdout, max_log_bytes),
+                "stderr": summarize_stream(result.stderr, max_log_bytes),
+            },
+            "outputs": output_records,
+        }
+        try:
+            write_manifest(manifest_path, manifest)
+        except OSError as exc:
+            raise CaptureError(f"cannot write {manifest_path}: {exc}") from exc
+        except SciProofError as exc:  # a bug in SciProof itself; never write an invalid manifest
+            raise CaptureError(f"internal error: generated an invalid manifest: {exc}") from exc
+        shutil.rmtree(snapshot_dir, ignore_errors=True)
+        if staging.exists():
+            os.replace(staging, snapshot_dir)
+    finally:
+        shutil.rmtree(staging, ignore_errors=True)
+    return CaptureResult(manifest_path=manifest_path, manifest=manifest, warnings=warnings)

sciproof/cli.py

@@ -0,0 +1,249 @@
+"""Command line interface.
+
+Exit codes
+----------
+0  success / verification PASS
+1  verification FAIL, or the captured command failed (evidence is still written)
+2  verification INCONCLUSIVE
+3  usage error, invalid manifest, or capture could not be completed
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from pathlib import Path
+
+from sciproof import __version__
+from sciproof.capture import DEFAULT_ATOL, DEFAULT_MAX_LOG_BYTES, DEFAULT_RTOL, capture
+from sciproof.errors import ConfirmationRequired, SciProofError
+from sciproof.inspect_manifest import inspect_file
+from sciproof.textutil import safe_text
+from sciproof.verify import (
+    DEFAULT_RERUN_TIMEOUT_SECONDS,
+    Status,
+    format_report,
+    format_report_json,
+    verify,
+)
+
+EXIT_OK = 0
+EXIT_FAIL = 1
+EXIT_INCONCLUSIVE = 2
+EXIT_ERROR = 3
+
+_STATUS_EXIT = {
+    Status.PASS: EXIT_OK,
+    Status.FAIL: EXIT_FAIL,
+    Status.INCONCLUSIVE: EXIT_INCONCLUSIVE,
+}
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="sciproof",
+        description="Record computational experiments and create verifiable evidence for them.",
+    )
+    parser.add_argument("--version", action="version", version=f"sciproof {__version__}")
+    sub = parser.add_subparsers(dest="command_name", required=True)
+
+    cap = sub.add_parser(
+        "capture",
+        help="run a command once and record evidence",
+        description=(
+            "Run COMMAND once in the current directory and write NAME.sciproof.json. "
+            "Everything after '--' is the command (it is not run through a shell)."
+        ),
+    )
+    cap.add_argument("--name", required=True, help="experiment name (also names the manifest)")
+    cap.add_argument(
+        "--input",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="declare an input file (hashed before and after the run); repeatable",
+    )
+    cap.add_argument(
+        "--output",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="declare an output file compared by exact hash; repeatable",
+    )
+    cap.add_argument(
+        "--output-numeric",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="declare a .json/.csv output compared with a numeric tolerance; repeatable",
+    )
+    cap.add_argument(
+        "--rtol",
+        type=float,
+        default=DEFAULT_RTOL,
+        help=f"relative tolerance for numeric outputs (default {DEFAULT_RTOL})",
+    )
+    cap.add_argument(
+        "--atol",
+        type=float,
+        default=DEFAULT_ATOL,
+        help=f"absolute tolerance for numeric outputs (default {DEFAULT_ATOL})",
+    )
+    cap.add_argument(
+        "--timeout",
+        type=float,
+        default=None,
+        metavar="SECONDS",
+        help="stop the command after this many seconds (no evidence is written)",
+    )
+    cap.add_argument(
+        "--max-log-bytes",
+        type=int,
+        default=DEFAULT_MAX_LOG_BYTES,
+        help="keep at most this many bytes of stdout and of stderr in the manifest",
+    )
+    cap.add_argument("--force", action="store_true", help="overwrite an existing manifest")
+    cap.add_argument(
+        "--allow-nonzero",
+        action="store_true",
+        help="exit with status 0 even if the command failed (evidence is written either way)",
+    )
+    cap.add_argument(
+        "cmd", nargs=argparse.REMAINDER, metavar="-- COMMAND [ARGS...]", help="the command to run"
+    )
+
+    ver = sub.add_parser(
+        "verify",
+        help="check a manifest (integrity by default, re-execution only with --rerun)",
+    )
+    ver.add_argument("manifest", type=Path)
+    ver.add_argument(
+        "--rerun",
+        action="store_true",
+        help="EXECUTE the recorded command again and compare results "
+        "(runs arbitrary code with your permissions)",
+    )
+    ver.add_argument(
+        "--yes", action="store_true", help="do not ask for confirmation before --rerun"
+    )
+    ver.add_argument(
+        "--strict-env",
+        action="store_true",
+        help="with --rerun: report INCONCLUSIVE if the environment differs",
+    )
+    ver.add_argument(
+        "--timeout",
+        type=float,
+        default=DEFAULT_RERUN_TIMEOUT_SECONDS,
+        metavar="SECONDS",
+        help="stop the rerun after this many seconds (default %(default)s)",
+    )
+
+    ver.add_argument(
+        "--json",
+        action="store_true",
+        help="print the report as one JSON object (exit codes are unchanged)",
+    )
+
+    ins = sub.add_parser("inspect", help="show a readable summary; never executes anything")
+    ins.add_argument("manifest", type=Path)
+    return parser
+
+
+def _confirm(prompt: str) -> bool:
+    if not sys.stdin.isatty():
+        raise ConfirmationRequired(
+            "refusing to execute the recorded command: no terminal to ask for confirmation; "
+            "re-run with --yes after reading the command (see 'sciproof inspect')"
+        )
+    print(prompt, file=sys.stderr)  # stderr keeps stdout clean for --json
+    answer = input("\nExecute this command? Type 'yes' to continue: ")
+    return answer.strip().lower() == "yes"
+
+
+def _run_capture(args: argparse.Namespace) -> int:
+    command = list(args.cmd)
+    if command and command[0] == "--":
+        command = command[1:]
+    if not command:
+        raise SciProofError(
+            "no command given; usage: sciproof capture --name NAME [options] -- COMMAND"
+        )
+    result = capture(
+        name=args.name,
+        argv=command,
+        base=Path.cwd(),
+        inputs=args.input,
+        outputs=args.output,
+        numeric_outputs=args.output_numeric,
+        rtol=args.rtol,
+        atol=args.atol,
+        timeout=args.timeout,
+        max_log_bytes=args.max_log_bytes,
+        force=args.force,
+    )
+    for warning in result.warnings:
+        print(f"warning: {warning}", file=sys.stderr)
+    manifest = result.manifest
+    print(f"Evidence written to {result.manifest_path.name}")
+    print(f"  command exit code: {result.exit_code}")
+    print(f"  inputs recorded:   {len(manifest['inputs'])}")
+    print(f"  outputs recorded:  {len(manifest['outputs'])}")
+    print(
+        "Note: stdout/stderr are stored in the manifest; check them for secrets before sharing it."
+    )
+    if result.exit_code != 0:
+        print(
+            f"The command failed (exit code {result.exit_code}); the failure was recorded.",
+            file=sys.stderr,
+        )
+        return EXIT_OK if args.allow_nonzero else EXIT_FAIL
+    return EXIT_OK
+
+
+def _run_verify(args: argparse.Namespace) -> int:
+    report = verify(
+        args.manifest,
+        rerun=args.rerun,
+        assume_yes=args.yes,
+        strict_env=args.strict_env,
+        timeout=args.timeout,
+        confirm=_confirm if args.rerun and not args.yes else None,
+    )
+    print(format_report_json(report) if args.json else format_report(report))
+    return _STATUS_EXIT[report.status]
+
+
+def _make_output_robust() -> None:
+    """Never crash because the terminal cannot encode a character from a manifest or a log.
+
+    Captured stdout/stderr and file names can contain any character, while a console may use
+    a narrow encoding (for example cp1252 on Windows). Unencodable characters are printed as
+    escapes (``\u2713``) instead of raising ``UnicodeEncodeError``.
+    """
+    for stream in (sys.stdout, sys.stderr):
+        reconfigure = getattr(stream, "reconfigure", None)
+        if reconfigure is not None:
+            reconfigure(errors="backslashreplace")
+
+
+def main(argv: list[str] | None = None) -> int:
+    _make_output_robust()
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.command_name == "capture":
+            return _run_capture(args)
+        if args.command_name == "verify":
+            return _run_verify(args)
+        print(inspect_file(args.manifest))
+        return EXIT_OK
+    except SciProofError as exc:
+        print(f"error: {safe_text(str(exc), 2000)}", file=sys.stderr)
+        return EXIT_ERROR
+    except KeyboardInterrupt:
+        print("interrupted", file=sys.stderr)
+        return EXIT_ERROR
+    except EOFError:
+        print("error: no confirmation received", file=sys.stderr)
+        return EXIT_ERROR

sciproof/environment.py

@@ -0,0 +1,123 @@
+"""Collect environment metadata without leaking secrets.
+
+Only an explicit allowlist of environment variables is ever recorded. Absolute paths
+(interpreter location, home directory, user name) are deliberately left out.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import re
+import shutil
+import subprocess  # noqa: S404  (used to ask the recorded interpreter for its version)
+import sys
+from pathlib import Path
+from typing import Any
+
+# Variables that influence numerical or locale behaviour and are not secret.
+ENV_ALLOWLIST = (
+    "PYTHONHASHSEED",
+    "TZ",
+    "LANG",
+    "LC_ALL",
+    "OMP_NUM_THREADS",
+    "MKL_NUM_THREADS",
+    "OPENBLAS_NUM_THREADS",
+    "NUMEXPR_NUM_THREADS",
+    "CUDA_VISIBLE_DEVICES",
+)
+
+PROBE_TIMEOUT_SECONDS = 30
+
+# Executed by the interpreter that runs the experiment. It prints one JSON document.
+_PROBE_SCRIPT = r"""
+import importlib.metadata as md, json, platform, sys
+seen = {}
+for dist in md.distributions():
+    name = (dist.metadata["Name"] or "").strip()
+    if name:
+        seen[name.lower().replace("_", "-")] = [name, dist.version]
+print(json.dumps({
+    "version": platform.python_version(),
+    "implementation": platform.python_implementation(),
+    "byteorder": sys.byteorder,
+    "packages": sorted(seen.values(), key=lambda item: item[0].lower()),
+}))
+"""
+
+
+def platform_info() -> dict[str, str]:
+    return {
+        "system": platform.system(),
+        "release": platform.release(),
+        "machine": platform.machine(),
+        "platform": platform.platform(),
+    }
+
+
+def selected_env_vars(environ: dict[str, str] | None = None) -> dict[str, str]:
+    environ = os.environ if environ is None else environ
+    return {name: environ[name] for name in ENV_ALLOWLIST if name in environ}
+
+
+def _own_interpreter_info() -> dict[str, Any]:
+    import importlib.metadata as md
+
+    seen: dict[str, list[str]] = {}
+    for dist in md.distributions():
+        name = (dist.metadata["Name"] or "").strip()
+        if name:
+            seen[name.lower().replace("_", "-")] = [name, dist.version]
+    return {
+        "version": platform.python_version(),
+        "implementation": platform.python_implementation(),
+        "byteorder": sys.byteorder,
+        "packages": sorted(seen.values(), key=lambda item: item[0].lower()),
+    }
+
+
+_INTERPRETER_NAME = re.compile(r"^(py|pythonw?(\d+(\.\d+)*[dt]*)?|pypy(\d+(\.\d+)*)?)$")
+
+
+def _looks_like_python(executable: str) -> bool:
+    """True only for names of real interpreters (``python``, ``python3.13``, ``py``, ``pypy3``).
+
+    The probe runs ``<executable> -c <snippet>``; running it on ``python_train.sh`` or
+    ``python-config`` would execute the user's own program with unexpected arguments.
+    """
+    name = Path(executable).name.lower()
+    if name.endswith(".exe"):
+        name = name[:-4]
+    return bool(_INTERPRETER_NAME.fullmatch(name))
+
+
+def interpreter_info(argv0: str, cwd: Path) -> tuple[dict[str, Any], str]:
+    """Describe the Python interpreter that runs the experiment.
+
+    If ``argv0`` is a Python interpreter, it is asked for its own version and installed
+    packages (this runs a fixed, built-in snippet and not the experiment). Otherwise the
+    interpreter that runs SciProof is described and the source is reported as such.
+
+    Returns ``(info, source)`` where source is ``"command-interpreter"`` or
+    ``"sciproof-process"``.
+    """
+    resolved = shutil.which(argv0, path=None) if _looks_like_python(argv0) else None
+    if resolved is None and _looks_like_python(argv0) and Path(argv0).exists():
+        resolved = argv0
+    if resolved is not None:
+        try:
+            completed = subprocess.run(  # noqa: S603
+                [resolved, "-c", _PROBE_SCRIPT],
+                cwd=cwd,
+                capture_output=True,
+                text=True,
+                timeout=PROBE_TIMEOUT_SECONDS,
+                check=True,
+            )
+            info = json.loads(completed.stdout)
+            return info, "command-interpreter"
+        except (OSError, subprocess.SubprocessError, ValueError):
+            pass
+    return _own_interpreter_info(), "sciproof-process"

sciproof/errors.py

@@ -0,0 +1,25 @@
+"""Exception types used by SciProof."""
+
+
+class SciProofError(Exception):
+    """Base class for all expected SciProof errors (shown to the user without a traceback)."""
+
+
+class ManifestError(SciProofError):
+    """The manifest is missing, malformed, unsupported or contains unsafe values."""
+
+
+class UnsafePathError(SciProofError):
+    """A path is absolute, escapes the base directory, or otherwise cannot be trusted."""
+
+
+class CaptureError(SciProofError):
+    """The experiment could not be captured."""
+
+
+class UsageError(SciProofError):
+    """An option has an invalid value."""
+
+
+class ConfirmationRequired(SciProofError):
+    """Re-execution was requested but not confirmed by the user."""