runproof-engine 0.1.0__py3-none-any.whl

runproof_engine/diff.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from .utils import safe_value
+
+
+@dataclass
+class Difference:
+    area: str
+    path: str
+    kind: str
+    before: Any
+    after: Any
+    explanation: str
+    confidence: str = "evidence"
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "area": self.area,
+            "path": self.path,
+            "kind": self.kind,
+            "before": safe_value(self.before),
+            "after": safe_value(self.after),
+            "explanation": self.explanation,
+            "confidence": self.confidence,
+        }
+
+
+@dataclass
+class RunDiff:
+    left_run_id: str
+    right_run_id: str
+    differences: list[Difference] = field(default_factory=list)
+
+    @property
+    def identical(self) -> bool:
+        return not self.differences
+
+    @property
+    def first_divergent_step(self) -> str | None:
+        for difference in self.differences:
+            if difference.area == "steps":
+                return difference.path.split(".")[0]
+        return None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "left_run_id": self.left_run_id,
+            "right_run_id": self.right_run_id,
+            "identical": self.identical,
+            "first_divergent_step": self.first_divergent_step,
+            "differences": [difference.to_dict() for difference in self.differences],
+        }
+
+    def render(self) -> str:
+        if self.identical:
+            return f"Runs {self.left_run_id} and {self.right_run_id} are observably identical."
+        lines = [
+            f"Run diff: {self.left_run_id} -> {self.right_run_id}",
+            f"Differences: {len(self.differences)}",
+        ]
+        if self.first_divergent_step:
+            lines.append(f"First divergent step: {self.first_divergent_step}")
+        for difference in self.differences:
+            lines.append(
+                f"- [{difference.area}] {difference.path}: {difference.explanation} "
+                f"(confidence={difference.confidence})"
+            )
+        return "\n".join(lines)
+
+
+def compare_manifests(left: dict[str, Any], right: dict[str, Any]) -> RunDiff:
+    left_run = left.get("run", {})
+    right_run = right.get("run", {})
+    result = RunDiff(
+        left_run_id=str(left_run.get("run_id", "unknown")),
+        right_run_id=str(right_run.get("run_id", "unknown")),
+    )
+    _compare_inputs(left.get("inputs", []), right.get("inputs", []), result)
+    _compare_steps(left.get("steps", []), right.get("steps", []), result)
+    _compare_outputs(left.get("outputs", []), right.get("outputs", []), result)
+    _compare_checks(left.get("checks", []), right.get("checks", []), result)
+    _compare_environment(left.get("environment"), right.get("environment"), result)
+    if left_run.get("status") != right_run.get("status"):
+        result.differences.append(Difference(
+            area="run",
+            path="status",
+            kind="changed",
+            before=left_run.get("status"),
+            after=right_run.get("status"),
+            explanation="run status changed",
+        ))
+    return result
+
+
+def _index(records: list[dict[str, Any]], key: str = "name") -> dict[str, dict[str, Any]]:
+    return {str(item.get(key, index)): item for index, item in enumerate(records)}
+
+
+def _compare_inputs(left: list[dict[str, Any]], right: list[dict[str, Any]], result: RunDiff) -> None:
+    left_map, right_map = _index(left), _index(right)
+    for name in sorted(set(left_map) | set(right_map)):
+        if name not in left_map:
+            result.differences.append(Difference("inputs", name, "added", None, right_map[name], "input was added"))
+            continue
+        if name not in right_map:
+            result.differences.append(Difference("inputs", name, "removed", left_map[name], None, "input was removed"))
+            continue
+        before, after = left_map[name], right_map[name]
+        if before.get("sha256") != after.get("sha256"):
+            result.differences.append(Difference(
+                "inputs", f"{name}.sha256", "changed", before.get("sha256"), after.get("sha256"),
+                "input content changed according to its SHA-256 fingerprint",
+            ))
+        for field_name in ("size_bytes", "rows", "columns", "schema"):
+            if before.get(field_name) != after.get(field_name) and (field_name in before or field_name in after):
+                result.differences.append(Difference(
+                    "inputs", f"{name}.{field_name}", "changed", before.get(field_name), after.get(field_name),
+                    f"input metadata field '{field_name}' changed",
+                ))
+
+
+def _compare_steps(left: list[dict[str, Any]], right: list[dict[str, Any]], result: RunDiff) -> None:
+    left_map, right_map = _index(left), _index(right)
+    for name in sorted(set(left_map) | set(right_map)):
+        if name not in left_map:
+            result.differences.append(Difference("steps", name, "added", None, right_map[name], "step was added"))
+            continue
+        if name not in right_map:
+            result.differences.append(Difference("steps", name, "removed", left_map[name], None, "step was removed"))
+            continue
+        before, after = left_map[name], right_map[name]
+        if before.get("status") != after.get("status"):
+            result.differences.append(Difference("steps", f"{name}.status", "changed", before.get("status"), after.get("status"), "step status changed"))
+        before_output = (before.get("output") or {}).get("fingerprint")
+        after_output = (after.get("output") or {}).get("fingerprint")
+        if before_output != after_output:
+            result.differences.append(Difference(
+                "steps", f"{name}.output.fingerprint", "changed", before_output, after_output,
+                "step output changed; downstream artifacts may be affected",
+            ))
+        before_function = (before.get("function") or {}).get("source_sha256")
+        after_function = (after.get("function") or {}).get("source_sha256")
+        if before_function != after_function:
+            result.differences.append(Difference(
+                "steps", f"{name}.function.source_sha256", "changed", before_function, after_function,
+                "step source fingerprint changed",
+            ))
+
+
+def _compare_outputs(left: list[dict[str, Any]], right: list[dict[str, Any]], result: RunDiff) -> None:
+    left_map, right_map = _index(left), _index(right)
+    for name in sorted(set(left_map) | set(right_map)):
+        if name not in left_map:
+            result.differences.append(Difference("outputs", name, "added", None, right_map[name], "output was added"))
+        elif name not in right_map:
+            result.differences.append(Difference("outputs", name, "removed", left_map[name], None, "output was removed"))
+        elif left_map[name].get("sha256") != right_map[name].get("sha256"):
+            result.differences.append(Difference(
+                "outputs", f"{name}.sha256", "changed", left_map[name].get("sha256"), right_map[name].get("sha256"),
+                "artifact content changed",
+            ))
+
+
+def _compare_checks(left: list[dict[str, Any]], right: list[dict[str, Any]], result: RunDiff) -> None:
+    left_map, right_map = _index(left), _index(right)
+    for name in sorted(set(left_map) | set(right_map)):
+        if name not in left_map or name not in right_map:
+            result.differences.append(Difference("checks", name, "changed", left_map.get(name), right_map.get(name), "check set changed"))
+        elif left_map[name].get("passed") != right_map[name].get("passed"):
+            result.differences.append(Difference(
+                "checks", f"{name}.passed", "changed", left_map[name].get("passed"), right_map[name].get("passed"),
+                "check outcome changed",
+            ))
+
+
+def _compare_environment(left: dict[str, Any] | None, right: dict[str, Any] | None, result: RunDiff) -> None:
+    if left is None or right is None:
+        if left != right:
+            result.differences.append(Difference("environment", "snapshot", "changed", bool(left), bool(right), "environment snapshot availability changed"))
+        return
+    for field_name in ("python_version", "platform", "machine"):
+        if left.get(field_name) != right.get(field_name):
+            result.differences.append(Difference(
+                "environment", field_name, "changed", left.get(field_name), right.get(field_name),
+                f"environment field '{field_name}' changed",
+            ))
+    left_packages = {item.get("name"): item.get("version") for item in left.get("packages", [])}
+    right_packages = {item.get("name"): item.get("version") for item in right.get("packages", [])}
+    for name in sorted(set(left_packages) | set(right_packages)):
+        if left_packages.get(name) != right_packages.get(name):
+            result.differences.append(Difference(
+                "environment", f"packages.{name}", "changed", left_packages.get(name), right_packages.get(name),
+                "installed package version changed",
+            ))

runproof_engine/policy.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any
+
+
+class PolicyDenied(PermissionError):
+    """Raised when a requested action is not allowed by the run policy."""
+
+
+@dataclass
+class Policy:
+    allowed_actions: set[str] = field(default_factory=lambda: {"read", "write_artifact", "execute_python"})
+    approval_required: set[str] = field(default_factory=lambda: {"install_package", "network", "upload", "publish", "delete", "shell"})
+    denied_actions: set[str] = field(default_factory=set)
+
+    def authorize(self, action: str, *, approved: bool = False, target: str | None = None) -> dict[str, Any]:
+        action = str(action)
+        if action in self.denied_actions:
+            raise PolicyDenied(f"action denied by policy: {action}")
+        if action not in self.allowed_actions and action not in self.approval_required:
+            raise PolicyDenied(f"action is not declared in policy: {action}")
+        if action in self.approval_required and not approved:
+            raise PolicyDenied(f"approval required for action: {action}")
+        return {"action": action, "target": target, "approved": approved}
+
+
+def safe_default_policy() -> Policy:
+    return Policy()

runproof_engine/replay.py

@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .diff import RunDiff, compare_manifests
+from .utils import file_metadata, read_json, sha256_file
+
+
+@dataclass
+class ReplayReport:
+    status: str
+    run_id: str
+    mode: str
+    reasons: list[str] = field(default_factory=list)
+    compared_run_id: str | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "status": self.status,
+            "run_id": self.run_id,
+            "mode": self.mode,
+            "reasons": self.reasons,
+            "compared_run_id": self.compared_run_id,
+        }
+
+    def __str__(self) -> str:
+        suffix = f"; reasons={self.reasons}" if self.reasons else ""
+        return f"ReplayReport(status={self.status!r}, run_id={self.run_id!r}, mode={self.mode!r}{suffix})"
+
+
+class LoadedRun:
+    def __init__(self, artifact_dir: str | os.PathLike[str]) -> None:
+        self.artifact_dir = Path(artifact_dir).expanduser().resolve()
+        self.manifest_path = self.artifact_dir / "manifest.json"
+        if not self.manifest_path.is_file():
+            raise FileNotFoundError(f"RunProof manifest not found: {self.manifest_path}")
+        self.manifest = read_json(self.manifest_path)
+
+    @property
+    def run_id(self) -> str:
+        return str(self.manifest.get("run", {}).get("run_id", self.artifact_dir.name))
+
+    @property
+    def name(self) -> str:
+        return str(self.manifest.get("run", {}).get("name", self.artifact_dir.parent.name))
+
+    @property
+    def status(self) -> str:
+        return str(self.manifest.get("run", {}).get("status", "unknown"))
+
+    def verify_integrity(self) -> ReplayReport:
+        reasons: list[str] = []
+        for record in self.manifest.get("inputs", []):
+            source = Path(record.get("path", ""))
+            if source.is_file():
+                if file_metadata(source).get("sha256") != record.get("sha256"):
+                    reasons.append(f"input changed: {record.get('name', source.name)}")
+                continue
+            captured = record.get("captured_copy")
+            captured_path = (self.artifact_dir / captured).resolve() if captured else None
+            if captured_path and captured_path.is_file() and file_metadata(captured_path).get("sha256") == record.get("sha256"):
+                continue
+            reasons.append(f"input missing: {source}")
+        for record in self.manifest.get("outputs", []):
+            output = (self.artifact_dir / record.get("path", "")).resolve()
+            if not output.is_file():
+                reasons.append(f"output missing: {output}")
+            elif file_metadata(output).get("sha256") != record.get("sha256"):
+                reasons.append(f"artifact changed: {record.get('name', output.name)}")
+        integrity_file = self.artifact_dir / "integrity.json"
+        if integrity_file.is_file():
+            try:
+                integrity = read_json(integrity_file)
+                for record in integrity.get("files", []):
+                    artifact = (self.artifact_dir / record["path"]).resolve()
+                    if not artifact.is_file():
+                        reasons.append(f"integrity file missing: {record['path']}")
+                    elif sha256_file(artifact) != record.get("sha256"):
+                        reasons.append(f"integrity mismatch: {record['path']}")
+            except (KeyError, TypeError, ValueError):
+                reasons.append("integrity manifest is invalid")
+        status = "verified" if not reasons else "non_reproducible"
+        return ReplayReport(status=status, run_id=self.run_id, mode="integrity", reasons=reasons)
+
+    def replay(
+        self,
+        *,
+        mode: str = "strict",
+        runner: Callable[[LoadedRun], str | os.PathLike[str] | LoadedRun] | None = None,
+    ) -> ReplayReport:
+        """Validate replay prerequisites and optionally compare a user-supplied re-execution.
+
+        RunProof does not guess how to reconstruct arbitrary Python state. A runner is
+        required to execute the original workflow again; without it, strict mode only
+        verifies captured evidence and reports that no new execution was performed.
+        """
+        if mode not in {"strict", "fresh"}:
+            raise ValueError("mode must be 'strict' or 'fresh'")
+        if runner is None:
+            integrity = self.verify_integrity()
+            if integrity.status != "verified":
+                integrity.mode = mode
+                return integrity
+            return ReplayReport(
+                status="replay_ready",
+                run_id=self.run_id,
+                mode=mode,
+                reasons=["integrity verified; pass runner=... to execute and compare the workflow"],
+            )
+        new_artifact = runner(self)
+        other = new_artifact if isinstance(new_artifact, LoadedRun) else load_run(new_artifact)
+        comparison = self.diff(other)
+        return ReplayReport(
+            status="verified" if comparison.identical else "non_reproducible",
+            run_id=self.run_id,
+            mode=mode,
+            reasons=[] if comparison.identical else [difference.explanation for difference in comparison.differences],
+            compared_run_id=other.run_id,
+        )
+
+    def diff(self, other: LoadedRun | str | os.PathLike[str]) -> RunDiff:
+        other_run = other if isinstance(other, LoadedRun) else load_run(other)
+        return compare_manifests(self.manifest, other_run.manifest)
+
+
+def load_run(artifact_dir: str | os.PathLike[str]) -> LoadedRun:
+    return LoadedRun(artifact_dir)

runproof_engine/utils.py

@@ -0,0 +1,197 @@
+from __future__ import annotations
+
+import csv
+import dataclasses
+import hashlib
+import inspect
+import json
+import os
+import platform
+import sys
+from datetime import datetime, timezone
+from importlib import metadata
+from pathlib import Path
+from typing import Any
+
+SECRET_MARKERS = (
+    "secret",
+    "password",
+    "passwd",
+    "token",
+    "api_key",
+    "apikey",
+    "private_key",
+    "authorization",
+    "credential",
+)
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def sha256_bytes(value: bytes) -> str:
+    return hashlib.sha256(value).hexdigest()
+
+
+def sha256_file(path: str | os.PathLike[str], chunk_size: int = 1024 * 1024) -> str:
+    digest = hashlib.sha256()
+    with Path(path).open("rb") as handle:
+        while chunk := handle.read(chunk_size):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def canonical_json(value: Any) -> str:
+    return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":"), default=str)
+
+
+def fingerprint(value: Any) -> str:
+    return sha256_bytes(canonical_json(safe_value(value, max_items=200)).encode("utf-8"))
+
+
+def _redacted_key(key: Any) -> bool:
+    normalized = str(key).lower().replace("-", "_")
+    return any(marker in normalized for marker in SECRET_MARKERS)
+
+
+def safe_value(value: Any, *, max_items: int = 100, max_text: int = 500) -> Any:
+    """Return a bounded, JSON-safe representation without exposing likely secrets."""
+    if value is None or isinstance(value, (bool, int, float)):
+        return value
+    if isinstance(value, (str, Path)):
+        text = str(value)
+        return text if len(text) <= max_text else text[:max_text] + "…"
+    if dataclasses.is_dataclass(value):
+        return safe_value(dataclasses.asdict(value), max_items=max_items, max_text=max_text)
+    if isinstance(value, dict):
+        items = list(value.items())[:max_items]
+        return {
+            str(key): "[REDACTED]" if _redacted_key(key) else safe_value(item, max_items=max_items, max_text=max_text)
+            for key, item in items
+        }
+    if isinstance(value, (list, tuple, set)):
+        values = list(value)
+        bounded = [safe_value(item, max_items=max_items, max_text=max_text) for item in values[:max_items]]
+        if len(values) > max_items:
+            bounded.append(f"… {len(values) - max_items} more items")
+        return bounded
+    if hasattr(value, "to_dict") and callable(value.to_dict):
+        try:
+            return safe_value(value.to_dict(), max_items=max_items, max_text=max_text)
+        except (AttributeError, KeyError, TypeError, ValueError, RuntimeError):
+            pass
+    if hasattr(value, "__dict__"):
+        try:
+            return {
+                "type": f"{type(value).__module__}.{type(value).__qualname__}",
+                "attributes": safe_value(vars(value), max_items=max_items, max_text=max_text),
+            }
+        except (AttributeError, TypeError, ValueError, RuntimeError):
+            pass
+    return {
+        "type": f"{type(value).__module__}.{type(value).__qualname__}",
+        "repr": str(value)[:max_text],
+    }
+
+
+def summarize(value: Any) -> dict[str, Any]:
+    """Describe a runtime value without serializing an unbounded payload."""
+    summary: dict[str, Any] = {
+        "type": f"{type(value).__module__}.{type(value).__qualname__}",
+        "fingerprint": fingerprint(value),
+    }
+    if value is None or isinstance(value, (bool, int, float, str)):
+        summary["value"] = safe_value(value)
+    elif isinstance(value, (list, tuple, set, dict)):
+        summary["length"] = len(value)
+        if isinstance(value, dict):
+            summary["keys"] = [str(key) for key in list(value.keys())[:100]]
+    else:
+        for attribute in ("shape", "columns", "dtypes", "size"):
+            if hasattr(value, attribute):
+                try:
+                    summary[attribute] = safe_value(getattr(value, attribute))
+                except (AttributeError, KeyError, TypeError, ValueError, RuntimeError):
+                    pass
+    return summary
+
+
+def function_descriptor(function: Any) -> dict[str, Any]:
+    descriptor: dict[str, Any] = {
+        "module": getattr(function, "__module__", None),
+        "qualname": getattr(function, "__qualname__", repr(function)),
+    }
+    try:
+        source = inspect.getsource(function)
+        descriptor["source_sha256"] = sha256_bytes(source.encode("utf-8"))
+    except (OSError, TypeError):
+        descriptor["source_sha256"] = None
+    return descriptor
+
+
+def environment_snapshot() -> dict[str, Any]:
+    packages: list[dict[str, str]] = []
+    try:
+        for distribution in metadata.distributions():
+            name = distribution.metadata.get("Name")
+            version = distribution.version
+            if name and version:
+                packages.append({"name": name.lower(), "version": version})
+        packages.sort(key=lambda item: item["name"])
+    except (ImportError, OSError, RuntimeError, ValueError):
+        packages = []
+    return {
+        "python_version": sys.version,
+        "python_executable": sys.executable,
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+        "packages": packages,
+    }
+
+
+def _tabular_metadata(file_path: Path) -> dict[str, Any]:
+    suffix = file_path.suffix.lower()
+    if suffix == ".csv":
+        try:
+            with file_path.open("r", encoding="utf-8-sig", newline="") as handle:
+                reader = csv.reader(handle)
+                columns = next(reader, [])
+                rows = sum(1 for _ in reader)
+            return {"rows": rows, "columns": columns, "schema": {column: "unknown" for column in columns}}
+        except (UnicodeDecodeError, csv.Error):
+            return {}
+    if suffix == ".jsonl":
+        try:
+            with file_path.open("r", encoding="utf-8") as handle:
+                rows = sum(1 for line in handle if line.strip())
+            return {"rows": rows}
+        except UnicodeDecodeError:
+            return {}
+    return {}
+
+
+def file_metadata(path: str | os.PathLike[str], *, copy_path: str | None = None) -> dict[str, Any]:
+    file_path = Path(path).expanduser().resolve()
+    if not file_path.is_file():
+        raise FileNotFoundError(file_path)
+    metadata_record: dict[str, Any] = {
+        "path": str(file_path),
+        "name": file_path.name,
+        "size_bytes": file_path.stat().st_size,
+        "sha256": sha256_file(file_path),
+    }
+    metadata_record.update(_tabular_metadata(file_path))
+    if copy_path:
+        metadata_record["captured_copy"] = copy_path
+    return metadata_record
+
+
+def write_json(path: str | os.PathLike[str], value: Any) -> None:
+    target = Path(path)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(json.dumps(value, indent=2, ensure_ascii=False, default=str) + "\n", encoding="utf-8")
+
+
+def read_json(path: str | os.PathLike[str]) -> Any:
+    return json.loads(Path(path).read_text(encoding="utf-8"))