sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/runner.py

@@ -0,0 +1,544 @@
+"""Pack-owned frozen-pool candidate execution and label-free capture."""
+
+from __future__ import annotations
+
+import json
+import os
+import platform
+import subprocess
+import sys
+import threading
+import time
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Mapping, Sequence
+
+from .contracts import (
+    CandidateRequest,
+    CandidateSpec,
+    ContractError,
+    FrozenPoolCase,
+    build_candidate_request,
+    canonical_json,
+    ranked_candidate_ids,
+    validate_candidate_response,
+)
+from .evidence import (
+    COVERAGE_SCHEMA_VERSION,
+    EVENT_SCHEMA_VERSION,
+    RUN_SCHEMA_VERSION,
+    RunRef,
+    sha256_bytes,
+    write_blob,
+    write_json,
+    write_jsonl,
+)
+from .packs import PackDescriptor, check_candidate, get_pack
+from .records import CaptureIdentity, EvidenceCapability
+
+MAX_STREAM_BYTES = 1_048_576
+ENV_POLICY_ID = "minimal-candidate-runtime.v1"
+_ENV_ALLOWLIST = frozenset(
+    {
+        "PATH",
+        "PATHEXT",
+        "SYSTEMROOT",
+        "WINDIR",
+        "COMSPEC",
+        "TEMP",
+        "TMP",
+        "TMPDIR",
+        "HOME",
+        "USERPROFILE",
+        "APPDATA",
+        "LOCALAPPDATA",
+        "PROGRAMDATA",
+        "LANG",
+        "LC_ALL",
+    }
+)
+
+
+def _wall_time() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def _candidate_environment() -> dict[str, str]:
+    """Pass only ordinary process-bootstrap variables, never sync credentials.
+
+    Candidates requiring their own service credentials must load them through
+    their own wrapper/configuration. Environment values are not recorded in the
+    bundle. This is credential separation, not an OS sandbox.
+    """
+
+    env = {key: value for key, value in os.environ.items() if key.upper() in _ENV_ALLOWLIST}
+    env["PYTHONUTF8"] = "1"
+    return env
+
+
+@dataclass
+class _BoundedStream:
+    retained: bytearray = field(default_factory=bytearray)
+    observed_bytes: int = 0
+    eof: bool = False
+    error: str | None = None
+
+    def drain(self, stream: Any) -> None:
+        try:
+            while True:
+                chunk = stream.read(65536)
+                if not chunk:
+                    self.eof = True
+                    break
+                self.observed_bytes += len(chunk)
+                remaining = MAX_STREAM_BYTES - len(self.retained)
+                if remaining > 0:
+                    self.retained.extend(chunk[:remaining])
+        except OSError as exc:
+            self.error = f"{type(exc).__name__}: {exc}"
+
+    @property
+    def capped(self) -> bool:
+        return self.observed_bytes > MAX_STREAM_BYTES
+
+
+@dataclass
+class _InputWrite:
+    complete: bool = False
+    error: str | None = None
+
+    def send(self, stream: Any, payload: bytes) -> None:
+        try:
+            stream.write(payload)
+            stream.flush()
+            self.complete = True
+        except OSError as exc:
+            self.error = f"{type(exc).__name__}: {exc}"
+        finally:
+            try:
+                stream.close()
+            except OSError:
+                pass
+
+
+def _parse_response(stdout: bytes, request: CandidateRequest) -> tuple[tuple[str, ...] | None, str | None, str | None]:
+    try:
+        text = stdout.decode("utf-8", errors="strict")
+
+        def unique_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+            result: dict[str, Any] = {}
+            for key, value in pairs:
+                if key in result:
+                    raise ValueError(f"duplicate JSON field {key!r}")
+                result[key] = value
+            return result
+
+        def invalid_constant(value: str) -> None:
+            raise ValueError(f"non-finite JSON number {value}")
+
+        value = json.loads(text, object_pairs_hook=unique_fields, parse_constant=invalid_constant)
+    except (UnicodeDecodeError, json.JSONDecodeError, ValueError, RecursionError) as exc:
+        code = "non_finite_score" if "non-finite JSON number" in str(exc) else "malformed_json"
+        return None, code, f"{type(exc).__name__}: {exc}"
+    try:
+        response = validate_candidate_response(value, request)
+        return ranked_candidate_ids(response, request), None, None
+    except ContractError as exc:
+        reason = str(exc)
+        if reason.startswith("response query_id does not match"):
+            code = "wrong_query_id"
+        elif reason.startswith("response pool_digest does not match"):
+            code = "wrong_pool_digest"
+        elif reason.startswith("duplicate score for candidate_id"):
+            code = "duplicate_candidate_ids"
+        elif "must be a finite JSON number" in reason:
+            code = "non_finite_score"
+        elif reason.startswith("response candidate IDs mismatch"):
+            ids = {item.candidate_id for item in request.candidates}
+            returned = {item.get("candidate_id") for item in value.get("scores", []) if isinstance(item, dict)}
+            missing, extra = ids - returned, returned - ids
+            code = "missing_and_extra_candidate_ids" if missing and extra else "missing_candidate_ids" if missing else "extra_candidate_ids"
+        else:
+            code = "invalid_response"
+        return None, code, reason
+
+
+def _case_snapshot(case: FrozenPoolCase) -> dict[str, Any]:
+    """Explicit label-free case snapshot. Never recurse through references."""
+
+    return {
+        "schema_version": "sphynx.rag.rerank.pool.v1",
+        "dataset_id": case.dataset_id,
+        "dataset_version": case.dataset_version,
+        "case_id": case.case_id,
+        "query_id": case.query_id,
+        "query": case.query,
+        "split": case.split,
+        "split_group_id": case.split_group_id,
+        "cluster_id": case.cluster_id,
+        "suite_type": case.suite_type,
+        "versions": case.versions.to_dict(),
+        "candidates": [item._digest_dict() for item in case.candidates],
+        "pool_digest": case.pool_digest,
+    }
+
+
+def _capture_one(
+    run_path: Path,
+    run_id: str,
+    candidate: CandidateSpec,
+    case: FrozenPoolCase,
+    pack_id: str,
+    trial_index: int,
+    timeout_seconds: float,
+) -> tuple[str, str, str]:
+    capture_id = "cap-" + uuid.uuid4().hex
+    identity = CaptureIdentity(run_id, capture_id, case.case_id, trial_index, 1, case.pool_digest)
+    capture_dir = run_path / "captures" / capture_id
+    capture_dir.mkdir(parents=True, exist_ok=False)
+    request = build_candidate_request(case, pack_id)
+    request_bytes = (request.to_json() + "\n").encode("utf-8")
+    snapshot_bytes = (canonical_json(_case_snapshot(case)) + "\n").encode("utf-8")
+
+    events: list[dict[str, Any]] = []
+
+    def event(event_type: str, *, status: str, details: dict[str, Any] | None = None) -> None:
+        events.append(
+            {
+                "schema_version": EVENT_SCHEMA_VERSION,
+                "event_id": "evt-" + uuid.uuid4().hex,
+                "sequence": len(events) + 1,
+                "capture_id": capture_id,
+                "actor": "runner",
+                "source": "runner_observed",
+                "event_type": event_type,
+                "status": status,
+                "wall_time_utc": _wall_time(),
+                "monotonic_ns": time.monotonic_ns(),
+                "clock_domain": "python.monotonic_ns.process_local",
+                "details": details or {},
+            }
+        )
+
+    started_wall = _wall_time()
+    started_ns = time.monotonic_ns()
+    event("case_started", status="started", details={"request_sha256": sha256_bytes(request_bytes)})
+    stdout = _BoundedStream()
+    stderr = _BoundedStream()
+    input_write = _InputWrite()
+    process: subprocess.Popen[bytes] | None = None
+    exit_code: int | None = None
+    timed_out = False
+    spawn_error: str | None = None
+    process_error: str | None = None
+    process_ended_ns: int | None = None
+    stream_threads: list[threading.Thread] = []
+    writer_thread: threading.Thread | None = None
+    try:
+        process = subprocess.Popen(
+            candidate.argv,
+            shell=False,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            env=_candidate_environment(),
+            bufsize=0,
+        )
+        event("process_started", status="started", details={"pid": process.pid})
+        assert process.stdin is not None and process.stdout is not None and process.stderr is not None
+        for collector, stream in ((stdout, process.stdout), (stderr, process.stderr)):
+            thread = threading.Thread(target=collector.drain, args=(stream,), daemon=True)
+            thread.start()
+            stream_threads.append(thread)
+        writer_thread = threading.Thread(target=input_write.send, args=(process.stdin, request_bytes), daemon=True)
+        writer_thread.start()
+        try:
+            remaining = max(0.0, timeout_seconds - (time.monotonic_ns() - started_ns) / 1_000_000_000)
+            exit_code = process.wait(timeout=remaining)
+        except subprocess.TimeoutExpired:
+            timed_out = True
+            try:
+                process.kill()
+            except ProcessLookupError:
+                pass
+            try:
+                exit_code = process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process_error = "candidate process did not exit after kill"
+        process_ended_ns = time.monotonic_ns()
+        event("process_exited", status="timeout" if timed_out else "exited", details={"exit_code": exit_code})
+    except OSError as exc:
+        if process is None:
+            spawn_error = f"{type(exc).__name__}: {exc}"
+            event("process_spawn_failed", status="failed", details={"error_type": type(exc).__name__})
+        else:
+            process_error = f"{type(exc).__name__}: {exc}"
+            process_ended_ns = time.monotonic_ns()
+            event("process_failed", status="failed", details={"error_type": type(exc).__name__})
+    finally:
+        if writer_thread is not None:
+            writer_thread.join(timeout=1)
+        for thread in stream_threads:
+            thread.join(timeout=1)
+        if process is not None:
+            for stream in (process.stdin, process.stdout, process.stderr):
+                if stream is not None and (stream is process.stdin or all(not thread.is_alive() for thread in stream_threads)):
+                    try:
+                        stream.close()
+                    except OSError:
+                        pass
+
+    if spawn_error is not None:
+        status, failure_code, failure_reason = "spawn_error", "spawn_error", spawn_error
+    elif timed_out:
+        status, failure_code, failure_reason = "timeout", "timeout", f"candidate exceeded {timeout_seconds:g} seconds"
+    elif process_error is not None:
+        status, failure_code, failure_reason = "process_error", "process_error", process_error
+    elif not input_write.complete:
+        status, failure_code, failure_reason = "input_write_error", "input_write_error", input_write.error or "input write did not finish"
+    elif exit_code != 0:
+        status, failure_code, failure_reason = "crash", "nonzero_exit", f"candidate exited with code {exit_code}"
+    elif stdout.capped or not stdout.eof or stdout.error:
+        status, failure_code, failure_reason = "output_truncated", "output_truncated", stdout.error or "stdout was capped or incomplete"
+    else:
+        ranked, failure_code, failure_reason = _parse_response(bytes(stdout.retained), request)
+        status = "success" if failure_code is None else "invalid_response"
+    if status != "success":
+        ranked = None
+    event("response_checked", status="valid" if status == "success" else "invalid", details={"failure_code": failure_code})
+    event("case_terminal", status=status, details={"failure_code": failure_code})
+    ended_ns = time.monotonic_ns()
+    ended_wall = _wall_time()
+
+    stdout_complete = process is not None and not timed_out and stdout.eof and not stdout.capped and stdout.error is None
+    stderr_complete = process is not None and not timed_out and stderr.eof and not stderr.capped and stderr.error is None
+    artifacts = [
+        write_blob(capture_dir, role="candidate_request", data=request_bytes, media_type="application/json", observed_bytes=len(request_bytes), complete=True),
+        write_blob(capture_dir, role="case_snapshot", data=snapshot_bytes, media_type="application/json", observed_bytes=len(snapshot_bytes), complete=True),
+        write_blob(
+            capture_dir,
+            role="candidate_stdout",
+            data=bytes(stdout.retained),
+            media_type="application/json" if status == "success" else "application/octet-stream",
+            observed_bytes=stdout.observed_bytes,
+            complete=stdout_complete,
+            observed=process is not None,
+        ),
+        write_blob(
+            capture_dir,
+            role="candidate_stderr",
+            data=bytes(stderr.retained),
+            media_type="text/plain",
+            observed_bytes=stderr.observed_bytes,
+            complete=stderr_complete,
+            observed=process is not None,
+        ),
+    ]
+
+    def stream_capability(name: str, collector: _BoundedStream, complete: bool) -> EvidenceCapability:
+        if process is None:
+            return EvidenceCapability(name, "v1", "not_captured", 0, "candidate process did not start")
+        if not complete:
+            return EvidenceCapability(name, "v1", "truncated", int(bool(collector.retained)), "process timed out, stream exceeded cap, or stream did not close cleanly")
+        return EvidenceCapability(name, "v1", "available", int(bool(collector.retained)))
+
+    capabilities = [
+        EvidenceCapability("candidate_input", "v1", "available", 1),
+        EvidenceCapability("case_snapshot", "v1", "available", 1),
+        stream_capability("candidate_stdout_raw", stdout, stdout_complete),
+        stream_capability("candidate_stderr_raw", stderr, stderr_complete),
+        EvidenceCapability(
+            "candidate_response_validated",
+            "v1",
+            "available" if status == "success" else "not_captured",
+            1 if status == "success" else 0,
+            None if status == "success" else f"terminal outcome: {failure_code}",
+        ),
+        EvidenceCapability("runner_timing", "v1", "available", 1),
+    ]
+    coverage = {
+        "schema_version": COVERAGE_SCHEMA_VERSION,
+        "capture_id": capture_id,
+        "capabilities": [
+            {
+                "capability_id": item.capability_id,
+                "capability_schema_version": item.capability_schema_version,
+                "state": item.state,
+                "observed_count": item.observed_count,
+                "reason": item.reason,
+            }
+            for item in capabilities
+        ],
+    }
+    artifacts_sha = write_jsonl(capture_dir / "artifacts.jsonl", artifacts)
+    events_sha = write_jsonl(capture_dir / "events.jsonl", events)
+    coverage_sha = write_json(capture_dir / "coverage.json", coverage)
+    capture_manifest = {
+        "schema_version": identity.schema_version,
+        "identity": {
+            "schema_version": identity.schema_version,
+            "run_id": identity.run_id,
+            "capture_id": identity.capture_id,
+            "case_id": identity.case_id,
+            "trial_index": identity.trial_index,
+            "attempt_index": identity.attempt_index,
+            "pool_digest": identity.pool_digest,
+            "trust_tier": identity.trust_tier,
+        },
+        "query_id": case.query_id,
+        "status": status,
+        "failure_code": failure_code,
+        "failure_reason": failure_reason,
+        "exit_code": exit_code,
+        "ranked_candidate_ids": list(ranked) if ranked is not None else None,
+        "timing": {
+            "clock_domain": "python.monotonic_ns.process_local",
+            "started_monotonic_ns": started_ns,
+            "ended_monotonic_ns": ended_ns,
+            "elapsed_ns": ended_ns - started_ns,
+            "candidate_latency_ns": process_ended_ns - started_ns if process_ended_ns is not None else None,
+            "started_wall_time_utc": started_wall,
+            "ended_wall_time_utc": ended_wall,
+        },
+        "events_sha256": events_sha,
+        "artifacts_sha256": artifacts_sha,
+        "coverage_sha256": coverage_sha,
+        "artifact_roles": [item["role"] for item in artifacts],
+        "source": "runner_observed",
+    }
+    capture_hash = write_json(capture_dir / "manifest.json", capture_manifest)
+    return capture_id, status, capture_hash
+
+
+def _run_with_cases(
+    candidate: CandidateSpec,
+    descriptor: PackDescriptor,
+    cases: Sequence[FrozenPoolCase],
+    output_dir: str | os.PathLike[str],
+) -> RunRef:
+    """Internal testable executor; official case selection comes from the pack."""
+
+    if not isinstance(candidate, CandidateSpec):
+        raise TypeError("candidate must be CandidateSpec")
+    if not descriptor.runnable:
+        raise ValueError(f"pack {descriptor.pack_id} is not runnable")
+    if candidate.interface_version != descriptor.interface_version:
+        raise ContractError("candidate interface does not match pack")
+    cases = tuple(cases)
+    if not cases or any(not isinstance(case, FrozenPoolCase) for case in cases):
+        raise ValueError("runnable pack must supply FrozenPoolCase values")
+    if len({case.case_id for case in cases}) != len(cases):
+        raise ValueError("pack contains duplicate case IDs")
+    if descriptor.case_count is not None and descriptor.case_count != len(cases):
+        raise ValueError("pack case_count does not match loaded cases")
+    policy = descriptor.execution_policy
+    if type(policy.trials) is not int or policy.trials < 1 or not 0 < policy.per_case_timeout_seconds < float("inf"):
+        raise ValueError("pack execution policy has invalid trials or timeout")
+
+    output_root = Path(output_dir).expanduser().resolve()
+    run_id = "run-" + uuid.uuid4().hex
+    run_path = output_root / "runs" / run_id
+    run_path.mkdir(parents=True, exist_ok=False)
+    started_wall = _wall_time()
+    capture_ids: list[str] = []
+    capture_hashes: dict[str, str] = {}
+    statuses: list[str] = []
+    for case in cases:
+        for trial_index in range(1, policy.trials + 1):
+            capture_id, status, capture_hash = _capture_one(
+                run_path,
+                run_id,
+                candidate,
+                case,
+                descriptor.pack_id,
+                trial_index,
+                policy.per_case_timeout_seconds,
+            )
+            capture_ids.append(capture_id)
+            capture_hashes[capture_id] = capture_hash
+            statuses.append(status)
+    manifest = {
+        "schema_version": RUN_SCHEMA_VERSION,
+        "run_id": run_id,
+        "pack_id": descriptor.pack_id,
+        "pack_release": descriptor.pack_release,
+        "track": descriptor.track,
+        "candidate": {
+            "name": candidate.name,
+            "version": candidate.version,
+            "interface_version": candidate.interface_version,
+            "argv_sha256": sha256_bytes(canonical_json(list(candidate.argv)).encode("utf-8")),
+        },
+        "execution_policy": {
+            "policy_version": policy.policy_version,
+            "trials": policy.trials,
+            "per_case_timeout_seconds": policy.per_case_timeout_seconds,
+            "resource_policy_id": policy.resource_policy_id,
+            "environment_policy_id": ENV_POLICY_ID,
+            "stdout_stderr_retention_limit_bytes_each": MAX_STREAM_BYTES,
+        },
+        "cases": [
+            {
+                "dataset_id": case.dataset_id,
+                "dataset_version": case.dataset_version,
+                "case_id": case.case_id,
+                "query_id": case.query_id,
+                "pool_digest": case.pool_digest,
+                "split": case.split,
+                "split_group_id": case.split_group_id,
+                "cluster_id": case.cluster_id,
+                "suite_type": case.suite_type,
+                "versions": case.versions.to_dict(),
+            }
+            for case in cases
+        ],
+        "trust_tier": "local_unverified",
+        "provenance": "runner_observed",
+        "runner": {
+            "release": "v1",
+            "python_version": platform.python_version(),
+            "platform": sys.platform,
+            "clock_domain": "python.monotonic_ns.process_local",
+        },
+        "started_wall_time_utc": started_wall,
+        "ended_wall_time_utc": _wall_time(),
+        "status": "completed" if all(status == "success" for status in statuses) else "completed_with_failures",
+        "case_count": len(cases),
+        "capture_count": len(capture_ids),
+        "success_count": sum(status == "success" for status in statuses),
+        "failure_count": sum(status != "success" for status in statuses),
+        "capture_ids": capture_ids,
+        "capture_manifest_hashes": capture_hashes,
+    }
+    write_json(run_path / "manifest.json", manifest)
+    return RunRef(run_id, run_path, descriptor.pack_id)
+
+
+def run(candidate: CandidateSpec, pack_id: str, output_dir: str | os.PathLike[str], *,
+        parameters: Mapping[str, Any] | None = None) -> RunRef:
+    """Run every installed case and trial fixed by a runnable pack.
+
+    Each case starts a fresh argv subprocess in the caller's current directory.
+    The subprocess gets only the versioned request on stdin and a minimal OS
+    environment. A failed case remains a terminal, replayable capture.
+
+    Optional parameters are resolved before filesystem/process work. The v1
+    adapter accepts only its existing test settings; contract-only definitions
+    require a future execution adapter. Reporting preferences do not render a
+    report here. The legacy run manifest and request schemas remain unchanged.
+    """
+
+    if parameters is not None or pack_id in {"rag.rerank.v2", "audio.similarity.v1"}:
+        from .evals import require_legacy_rerank
+        require_legacy_rerank(pack_id, parameters)
+    check = check_candidate(candidate, pack_id)
+    if not check.compatible:
+        raise ContractError("; ".join(check.reasons))
+    descriptor = get_pack(pack_id)
+    if not descriptor.runnable:
+        raise ValueError(f"pack {pack_id} is {descriptor.status}; installed cases are required before run")
+    from .rag import load_cases
+
+    return _run_with_cases(candidate, descriptor, load_cases(pack_id), output_dir)