techtree 0.1.0__py3-none-any.whl

techtree/verifiers/outputs.py

@@ -0,0 +1,368 @@
+"""The files one variant's evaluation must have left behind. Spec section 6.13.
+
+A real ``eval`` run leaves three files Techtree keeps: the resolved
+configuration, the raw upstream episodes, and the upstream log. The normalizer
+that the engine runs over them does not replace any of them (spec section 6.12)
+— Techtree retains raw upstream evidence *and* its normalized projection,
+because a projection nobody can check against its source is a claim rather than
+evidence.
+
+*Two of the three live in subdirectories now.* The released engine writes
+``configs/resolved/eval.json`` and ``logs/attempt_<n>/eval.log`` rather than the
+flat ``config.toml`` and ``eval.log`` of the development build
+(``docs/verifiers-pin-0.3.1.md``, deviations D1 and D5), and the resolved
+configuration is JSON because only JSON round-trips the explicit nulls the
+engine writes. The attempt number is ``1`` and can be nothing else: an attempt
+directory is minted per *launch*, and Techtree launches a run directory exactly
+once — it never resumes and never re-enters one (deviation D3).
+
+*The engine also writes ``logs/latest``, and Techtree ignores it.* It is a
+symbolic link to the current attempt directory, repointed by every later
+launch. A digest taken over a link is not a digest of the file it names, and a
+path that another launch can silently redirect is not a citation, so the
+evidence this module hashes is the attempt directory itself. Nothing here
+follows, records or hashes the link.
+
+*The engine writes no copy of the launch configuration for a Techtree run.* It
+copies the launch file verbatim to ``configs/eval.toml`` only when that file is
+TOML, and Techtree's is ``input.json`` (deviation D5). Nothing is lost: the
+compiled input is already the run's own file and already digested where it was
+written.
+
+A dry-run directory is deliberately not accepted here. Dry run writes only the
+resolved configuration (``docs/verifiers-eval.md``, finding E2), so applying
+the run's requirements to it would report a truncated run where there was never
+a run at all.
+
+``build_variant_result`` does not interpret anything itself. It hands the raw
+directory to the engine tool ``normalize_eval_output.py``, which runs under the
+engine's own interpreter with the pinned Verifiers wire models available to it
+(decisions document 0003 A3). Reading a Trace with the version of Verifiers
+that wrote it is the difference between interpreting the record and guessing at
+it, and it is why the interpretation is pinned exactly as tightly as the
+library.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Final
+
+from pydantic import ValidationError as PydanticValidationError
+
+from techtree.canonical import sha256_digest_bytes
+from techtree.engines.registry import EngineRegistry
+from techtree.engines.runner import EngineRunner
+from techtree.errors import EngineError, ValidationError
+from techtree.models.base import ArtifactRef, Digest
+from techtree.verifiers.models import (
+    NORMALIZED_EPISODES_FILENAME,
+    ChildProcessOutcome,
+    NormalizedEpisode,
+    SubjectImageResolution,
+    VariantExecutionPlan,
+    VariantExecutionResult,
+)
+
+__all__ = [
+    "DEFAULT_NORMALIZE_TIMEOUT_SECONDS",
+    "EVAL_LOG_MEDIA_TYPE",
+    "EVAL_LOG_PATH",
+    "NORMALIZED_EPISODES_FILENAME",
+    "NORMALIZED_EPISODES_MEDIA_TYPE",
+    "NORMALIZE_EVAL_OUTPUT_TOOL",
+    "RESOLVED_CONFIG_MEDIA_TYPE",
+    "RESOLVED_CONFIG_PATH",
+    "TRACES_FILENAME",
+    "TRACES_MEDIA_TYPE",
+    "VARIANT_OUTPUT_INCOMPLETE",
+    "artifact_for",
+    "build_variant_result",
+    "normalize_eval_output",
+    "read_normalized_episodes",
+    "require_output_files",
+    "required_output_paths",
+]
+
+#: Stable error code. Spec section 6.13.
+VARIANT_OUTPUT_INCOMPLETE: Final = "variant_output_incomplete"
+
+#: The configuration the engine resolved and wrote back, relative to the run's
+#: own directory. Nested and JSON since v0.3.1 (deviation D1), and the same
+#: relative path whether the run was real or a dry run, which is what lets the
+#: validation confirm the path the real run's reader will use.
+RESOLVED_CONFIG_PATH: Final = "configs/resolved/eval.json"
+
+TRACES_FILENAME: Final = "traces.jsonl"
+
+#: This launch's log. One attempt directory is minted per launch and Techtree
+#: launches once, so the run's log is always attempt one's (deviation D5). The
+#: ``logs/latest`` link beside it is deliberately not what is read: see the
+#: module docstring.
+EVAL_LOG_PATH: Final = "logs/attempt_1/eval.log"
+
+#: The engine helper that turns raw episodes into the protocol projection. It
+#: lives inside the digested bundle (decisions document 0003 A3).
+NORMALIZE_EVAL_OUTPUT_TOOL: Final = "normalize_eval_output.py"
+
+RESOLVED_CONFIG_MEDIA_TYPE: Final = "application/json"
+TRACES_MEDIA_TYPE: Final = "application/x-ndjson"
+EVAL_LOG_MEDIA_TYPE: Final = "text/plain"
+NORMALIZED_EPISODES_MEDIA_TYPE: Final = "application/x-ndjson"
+
+DEFAULT_NORMALIZE_TIMEOUT_SECONDS: Final = 300.0
+
+
+def required_output_paths(output_dir: Path) -> dict[str, Path]:
+    """Return the three files a completed evaluation writes.
+
+    ``output_dir`` is the run's own directory — the one ``--run.name`` pins,
+    one level below the directory the compiled configuration groups runs under.
+    """
+    return {
+        "config": output_dir / RESOLVED_CONFIG_PATH,
+        "traces": output_dir / TRACES_FILENAME,
+        "eval_log": output_dir / EVAL_LOG_PATH,
+    }
+
+
+def require_output_files(output_dir: Path) -> dict[str, Path]:
+    """Return the three files, or refuse.
+
+    An empty ``traces.jsonl`` counts as missing. Upstream truncates the file to
+    empty before the first rollout, so its emptiness after the child exits
+    means no episode ever completed, which is a failed run rather than a run
+    with no results.
+    """
+    paths = required_output_paths(output_dir)
+    missing = sorted(name for name, path in paths.items() if not path.is_file())
+    if missing:
+        raise ValidationError(
+            "the evaluation output is incomplete; "
+            f"missing {', '.join(missing)} under {output_dir.name}",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"output_dir": str(output_dir), "missing": list(missing)},
+        )
+    if paths["traces"].stat().st_size == 0:
+        raise ValidationError(
+            "the evaluation recorded no episodes; traces.jsonl is empty",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"output_dir": str(output_dir), "missing": ["episodes"]},
+        )
+    return paths
+
+
+def artifact_for(path: Path, media_type: str) -> ArtifactRef:
+    """Hash a file's exact bytes and build its artifact reference."""
+    try:
+        data = path.read_bytes()
+    except OSError as error:
+        raise ValidationError(
+            f"the evaluation artifact {path.name} could not be read",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path)},
+        ) from error
+    if not data:
+        raise ValidationError(
+            f"the evaluation artifact {path.name} is empty",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path)},
+        )
+    return ArtifactRef(
+        digest=sha256_digest_bytes(data),
+        media_type=media_type,
+        size=len(data),
+        relative_path=None,
+    )
+
+
+def read_normalized_episodes(path: Path) -> list[NormalizedEpisode]:
+    """Parse every record of a normalized JSONL file.
+
+    A missing final newline is a refusal rather than a shrug. The normalizer
+    writes whole records; a file that stops mid-line is a file that was
+    truncated, and parsing the part that survived would silently drop an
+    episode from a comparison that claims to cover every task.
+    """
+    try:
+        raw = path.read_bytes()
+    except OSError as error:
+        raise ValidationError(
+            "the normalized episode file could not be read",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path)},
+        ) from error
+
+    if not raw:
+        raise ValidationError(
+            "the normalized episode file is empty",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path)},
+        )
+    if not raw.endswith(b"\n"):
+        raise ValidationError(
+            "the normalized episode file does not end with a newline, so its "
+            "last record is incomplete",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path)},
+        )
+
+    episodes: list[NormalizedEpisode] = []
+    for number, line in enumerate(raw.decode("utf-8").splitlines(), start=1):
+        if not line.strip():
+            raise ValidationError(
+                f"the normalized episode file has a blank record at line {number}",
+                code=VARIANT_OUTPUT_INCOMPLETE,
+                details={"path": str(path), "line": number},
+            )
+        episodes.append(_parse_episode(line, path=path, number=number))
+    return episodes
+
+
+def _parse_episode(line: str, *, path: Path, number: int) -> NormalizedEpisode:
+    """Parse one normalized record, or refuse with its line number."""
+    try:
+        return NormalizedEpisode.model_validate_json(line)
+    except PydanticValidationError as error:
+        detail = error.errors()[0]["msg"]
+        raise ValidationError(
+            f"the normalized episode at line {number} is malformed: {detail}",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path), "line": number},
+        ) from error
+    except json.JSONDecodeError as error:
+        raise ValidationError(
+            f"the normalized episode at line {number} is not JSON",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"path": str(path), "line": number},
+        ) from error
+
+
+# ---------------------------------------------------------------------------
+# The engine's own interpretation
+# ---------------------------------------------------------------------------
+
+
+def normalize_eval_output(
+    *,
+    engine_registry: EngineRegistry,
+    engine_digest: Digest,
+    engine_runner: EngineRunner,
+    output_dir: Path,
+    taskset_lock_path: Path,
+    experiment_manifest_path: Path,
+    destination: Path,
+    timeout: float = DEFAULT_NORMALIZE_TIMEOUT_SECONDS,
+) -> Path:
+    """Run the engine's normalizer over one variant's raw output.
+
+    Nothing about the upstream record is interpreted on this side of the
+    boundary. The helper runs under the engine's interpreter, refuses anything
+    it cannot join onto the Campaign's committed membership, and writes the
+    projection ordered by that membership rather than by completion order.
+    """
+    script = engine_registry.tool_path(engine_digest, NORMALIZE_EVAL_OUTPUT_TOOL)
+    result = engine_runner.run_python_script(
+        script,
+        [
+            "--output-dir",
+            str(output_dir),
+            "--membership",
+            str(taskset_lock_path),
+            "--experiment-manifest",
+            str(experiment_manifest_path),
+            "--output",
+            str(destination),
+        ],
+        timeout=timeout,
+    )
+    if result.exit_code != 0:
+        raise EngineError(
+            "the engine could not normalize the evaluation output: "
+            f"{_last_line(result.stderr) or _last_line(result.stdout)}",
+            code="eval_normalization_failed",
+            details={
+                "engine_digest": engine_digest,
+                "output_dir": str(output_dir),
+                "exit_code": result.exit_code,
+            },
+        )
+    return destination
+
+
+def build_variant_result(
+    *,
+    plan: VariantExecutionPlan,
+    outcome: ChildProcessOutcome,
+    image_resolution: SubjectImageResolution,
+    engine_registry: EngineRegistry,
+    engine_digest: Digest,
+    engine_runner: EngineRunner,
+    taskset_lock_path: Path,
+    timeout: float = DEFAULT_NORMALIZE_TIMEOUT_SECONDS,
+) -> VariantExecutionResult:
+    """Assemble one variant's complete result from what the child left behind.
+
+    Raw evidence and its projection are both retained and both hashed. The
+    projection is what later stages read; the raw files are what makes the
+    projection checkable, and a receipt that cited only the tidy document would
+    be asking to be trusted rather than verified.
+    """
+    if outcome.variant is not plan.variant:
+        raise ValidationError(
+            f"a {plan.variant.value} plan cannot be completed by a "
+            f"{outcome.variant.value} child process",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={"plan": plan.variant.value, "outcome": outcome.variant.value},
+        )
+
+    output_dir = Path(plan.verifiers_output_dir)
+    paths = require_output_files(output_dir)
+    destination = output_dir / NORMALIZED_EPISODES_FILENAME
+
+    normalize_eval_output(
+        engine_registry=engine_registry,
+        engine_digest=engine_digest,
+        engine_runner=engine_runner,
+        output_dir=output_dir,
+        taskset_lock_path=taskset_lock_path,
+        experiment_manifest_path=Path(plan.experiment_manifest_path),
+        destination=destination,
+        timeout=timeout,
+    )
+    episodes = read_normalized_episodes(destination)
+
+    if len(episodes) != plan.task_count:
+        raise ValidationError(
+            f"the {plan.variant.value} variant normalized {len(episodes)} "
+            f"episodes for {plan.task_count} committed tasks",
+            code=VARIANT_OUTPUT_INCOMPLETE,
+            details={
+                "variant": plan.variant.value,
+                "normalized": len(episodes),
+                "expected": plan.task_count,
+            },
+        )
+
+    return VariantExecutionResult(
+        variant=plan.variant,
+        experiment_manifest_digest=plan.experiment_manifest_digest,
+        resolved_verifiers_config=artifact_for(
+            paths["config"], RESOLVED_CONFIG_MEDIA_TYPE
+        ),
+        raw_traces=artifact_for(paths["traces"], TRACES_MEDIA_TYPE),
+        eval_log=artifact_for(paths["eval_log"], EVAL_LOG_MEDIA_TYPE),
+        normalized_episodes=artifact_for(destination, NORMALIZED_EPISODES_MEDIA_TYPE),
+        child_outcome=outcome,
+        image_resolution=image_resolution,
+        episodes=episodes,
+    )
+
+
+def _last_line(stream: str) -> str:
+    """Return the last line that says something."""
+    for line in reversed(stream.splitlines()):
+        if line.strip():
+            return line.strip()
+    return ""

techtree/verifiers/progress.py

@@ -0,0 +1,192 @@
+"""How far one variant has got, read from its own evidence. Spec section 6.11.
+
+Progress is derived from the file the engine is already writing rather than
+from anything the child says on its way past. ``traces.jsonl`` is append-only:
+upstream serializes one whole Episode, appends it under a run-wide lock in a
+shielded worker thread, and ends every record with a newline
+(``docs/verifiers-eval.md``). Counting the records that are complete is
+therefore an exact, cheap, read-only measurement that a concurrent writer cannot
+be disturbed by — provided the reader ignores a final line that has not been
+terminated yet, and never writes to the file to "repair" it.
+
+One rule governs how the number may be used. **Line position is never task
+position.** Records land in completion order, and the preflight watched four
+tasks arrive in exactly reverse order (``docs/verifiers-eval.md``, finding E4).
+A count of finished episodes is honest; an inference from line seven to task
+seven is not. Pairing happens later, by task hash, in the engine's own
+normalizer.
+
+The file's absence means *pending*, not broken. Upstream truncates
+``traces.jsonl`` to empty only after the taskset package has imported and the
+environment has been constructed, roughly a second and a half in on a trivial
+taskset, so a poller that treated "no file yet" as failure would fail every run
+it watched.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Final, Literal
+
+from techtree.models.run import VariantProgress
+from techtree.runs.events import (
+    DETAIL_COMPLETED,
+    DETAIL_ERRORED,
+    DETAIL_RUNNING,
+    DETAIL_STATE,
+    DETAIL_TOTAL,
+    DETAIL_VARIANT,
+    VARIANT_PROGRESS,
+)
+from techtree.runs.store import RunStore
+from techtree.verifiers.child import CANCELLATION_EXIT_CODE
+from techtree.verifiers.models import VariantName
+
+__all__ = [
+    "count_complete_jsonl_records",
+    "emit_progress_if_changed",
+    "inspect_progress",
+    "pending_progress",
+]
+
+#: Reading the tail of a growing file is done in whole chunks rather than line
+#: by line so that one poll is one pass, however many episodes have landed.
+_READ_CHUNK_BYTES: Final = 1 << 20
+
+type _VariantLiteral = Literal["baseline", "candidate"]
+type _StateLiteral = Literal["pending", "running", "completed", "failed", "cancelled"]
+
+#: ``VariantProgress`` is a projection model and names its variants as literals.
+#: Spelling the correspondence out keeps the enum the single source of the two
+#: names while still handing the model the exact type it declares.
+_VARIANT_LITERALS: Final[dict[VariantName, _VariantLiteral]] = {
+    VariantName.BASELINE: "baseline",
+    VariantName.CANDIDATE: "candidate",
+}
+
+
+def count_complete_jsonl_records(path: Path) -> int:
+    """Count the whole, valid JSON object records in an append-only JSONL file.
+
+    A final line without its newline is a record still being written, and is not
+    counted. Neither is a line that is newline-terminated but not yet a valid
+    JSON object, which is what a partial write looks like on a filesystem that
+    reorders. Nothing is rewritten: the file belongs to the child, and a reader
+    that repaired it would be destroying the evidence it came to measure.
+
+    A missing file counts as zero, because the engine creates it a moment after
+    the child starts and before the first rollout.
+    """
+    try:
+        handle = path.open("rb")
+    except FileNotFoundError:
+        return 0
+    except OSError:
+        return 0
+
+    complete = 0
+    remainder = b""
+    with handle:
+        while chunk := handle.read(_READ_CHUNK_BYTES):
+            remainder += chunk
+            *lines, remainder = remainder.split(b"\n")
+            complete += sum(1 for line in lines if _is_complete_record(line))
+    return complete
+
+
+def _is_complete_record(line: bytes) -> bool:
+    """Whether one newline-terminated line is a whole JSON object."""
+    if not line.strip():
+        return False
+    try:
+        return isinstance(json.loads(line), dict)
+    except (ValueError, UnicodeDecodeError):
+        return False
+
+
+def pending_progress(variant: VariantName, total: int) -> VariantProgress:
+    """The progress of a variant whose child has not been started yet."""
+    return VariantProgress(
+        variant=_VARIANT_LITERALS[variant],
+        completed=0,
+        total=total,
+        running=0,
+        errored=0,
+        state="pending",
+    )
+
+
+def inspect_progress(
+    *,
+    variant: VariantName,
+    traces_path: Path,
+    total: int,
+    child_exit_code: int | None,
+    max_concurrent: int = 1,
+) -> VariantProgress:
+    """Measure one variant without interpreting any reward.
+
+    ``completed`` is the number of episodes the engine has finished writing.
+    ``running`` is how many can be in flight given what is left and how many
+    permits the variant was compiled with — the file records finished episodes
+    only, so an exact in-flight count is not observable and a bound that cannot
+    exceed the truth is the honest answer.
+
+    ``errored`` stays zero here. Whether an episode is scientifically usable is
+    settled by :mod:`techtree.verifiers.verify` against the normalized
+    projection, and a progress line that guessed at it early would be reporting
+    a verdict the run has not reached.
+    """
+    completed = min(count_complete_jsonl_records(traces_path), total)
+    remaining = max(total - completed, 0)
+
+    state: _StateLiteral
+    if child_exit_code is None:
+        state = "running" if traces_path.exists() else "pending"
+        running = min(remaining, max(max_concurrent, 0))
+    elif child_exit_code == CANCELLATION_EXIT_CODE:
+        state, running = "cancelled", 0
+    elif child_exit_code == 0 and remaining == 0:
+        state, running = "completed", 0
+    else:
+        state, running = "failed", 0
+
+    return VariantProgress(
+        variant=_VARIANT_LITERALS[variant],
+        completed=completed,
+        total=total,
+        running=running,
+        errored=0,
+        state=state,
+    )
+
+
+def emit_progress_if_changed(
+    *,
+    run_store: RunStore,
+    run_id: str,
+    previous: VariantProgress | None,
+    current: VariantProgress,
+) -> None:
+    """Append one ``variant.progress`` event only when something moved.
+
+    A poller runs several times a second and a run's event log is the record
+    other processes rebuild their picture of the run from. Appending an
+    unchanged projection would grow that log without adding a fact to it.
+    """
+    if previous is not None and previous == current:
+        return
+    run_store.append(
+        run_id,
+        phase=None,
+        kind=VARIANT_PROGRESS,
+        details={
+            DETAIL_VARIANT: current.variant,
+            DETAIL_COMPLETED: current.completed,
+            DETAIL_TOTAL: current.total,
+            DETAIL_RUNNING: current.running,
+            DETAIL_ERRORED: current.errored,
+            DETAIL_STATE: current.state,
+        },
+    )