ageval-cli 0.7.1__py3-none-any.whl

ageval/viewer/trials/fs.py

@@ -0,0 +1,386 @@
+"""Trial file tree and file preview for viewer."""
+
+from __future__ import annotations
+
+import contextlib
+import mimetypes
+from pathlib import Path
+from typing import Any
+
+from ageval.config.errors import ConfigError
+from ageval.viewer.jobs import get_job, safe_id_segment
+from ageval.viewer.trials.constants import MAX_FILE_BYTES, MAX_TREE_ENTRIES, TEXT_SUFFIXES
+from ageval.viewer.trials.paths import (
+    _read_json_object,
+    _safe_run_id,
+    _safe_under,
+    resolve_evidence_root,
+)
+
+
+def _scope_base(evidence: Path, scope: str) -> Path:
+    scope = (scope or "root").strip().lower()
+    mapping = {
+        "root": evidence,
+        "agent": evidence / "agent",
+        "eval": evidence / "evaluation",
+        "evaluation": evidence / "evaluation",
+        "verifier": evidence / "evaluation",
+        "artifacts": evidence / "harness",
+        "harness": evidence / "harness",
+        "lock": evidence,  # single file handled by caller
+    }
+    if scope not in mapping:
+        raise ConfigError(
+            "invalid_package",
+            f"unknown tree scope: {scope!r}",
+            location="scope",
+        )
+    return mapping[scope]
+
+
+def trial_tree(
+    dataset_root: Path,
+    job_id: str,
+    task_id: str,
+    run_id: str,
+    *,
+    scope: str = "root",
+) -> dict[str, Any]:
+    root = dataset_root.expanduser().resolve(strict=False)
+    safe_id_segment(job_id, field="job_id")
+    task_id = safe_id_segment(task_id, field="task_id")
+    rid = _safe_run_id(run_id)
+    # Ensure job exists (sandbox + membership)
+    get_job(root, job_id)
+    evidence = resolve_evidence_root(root, rid, task_id=task_id, require_task_match=True)
+    scope_norm = (scope or "root").strip().lower()
+
+    # Special multi-root scopes for verifier
+    if scope_norm in {"verifier", "eval", "evaluation"}:
+        from ageval.evidence.attempt_record import RESULT_FILENAME, result_path
+
+        entries: list[dict[str, Any]] = []
+        for sub in (
+            evidence / "evaluation",
+            evidence / "eval_staging",
+            result_path(evidence),
+        ):
+            if sub.is_file():
+                st = sub.stat()
+                rel = (
+                    RESULT_FILENAME
+                    if sub.name == RESULT_FILENAME
+                    else (sub.name if sub.parent == evidence else str(sub.relative_to(evidence)))
+                )
+                entries.append(
+                    {
+                        "path": rel,
+                        "name": sub.name,
+                        "type": "file",
+                        "size": st.st_size,
+                    }
+                )
+            elif sub.is_dir():
+                remain = MAX_TREE_ENTRIES - len(entries)
+                entries.extend(_walk_tree(evidence, sub, max_entries=remain))
+        return {
+            "ok": True,
+            "run_id": rid,
+            "scope": "verifier",
+            "entries": entries[:MAX_TREE_ENTRIES],
+            "truncated": len(entries) > MAX_TREE_ENTRIES,
+        }
+
+    if scope_norm == "artifacts":
+        # Publishable / harness-produced product files — not root runtime bookkeeping.
+        # Prefer package artifacts/ then harness/ subtree (e.g. terminal.json, published JSON).
+        entries = []
+        for sub in (
+            evidence / "artifacts",
+            evidence / "harness",
+            evidence / "agent" / "artifacts",
+        ):
+            if sub.is_dir():
+                entries.extend(
+                    _walk_tree(evidence, sub, max_entries=MAX_TREE_ENTRIES - len(entries))
+                )
+            elif sub.is_file():
+                with contextlib.suppress(OSError, ValueError):
+                    entries.append(
+                        {
+                            "path": str(sub.relative_to(evidence)),
+                            "name": sub.name,
+                            "type": "file",
+                            "size": sub.stat().st_size,
+                        }
+                    )
+        return {
+            "ok": True,
+            "run_id": rid,
+            "scope": "artifacts",
+            "entries": entries[:MAX_TREE_ENTRIES],
+            "truncated": len(entries) > MAX_TREE_ENTRIES,
+            "note": "publishable outputs under artifacts/ and harness/; "
+            "runtime bookkeeping is under Runtime tab",
+        }
+
+    if scope_norm == "lock":
+        lock_path = evidence / "lock.json"
+        entries = []
+        if lock_path.is_file():
+            entries.append(
+                {
+                    "path": "lock.json",
+                    "name": "lock.json",
+                    "type": "file",
+                    "size": lock_path.stat().st_size,
+                }
+            )
+        return {"ok": True, "run_id": rid, "scope": "lock", "entries": entries, "truncated": False}
+
+    if scope_norm in {"runtime", "log"}:  # "log" accepted as alias
+        entries = []
+        for name in ("effects.jsonl", "cleanup.json", "summary.json", "agent.json", "harness.json"):
+            p = evidence / name
+            if p.is_file():
+                entries.append(
+                    {
+                        "path": name,
+                        "name": name,
+                        "type": "file",
+                        "size": p.stat().st_size,
+                    }
+                )
+        return {
+            "ok": True,
+            "run_id": rid,
+            "scope": "runtime",
+            "entries": entries,
+            "truncated": False,
+        }
+
+    base = _scope_base(evidence, scope_norm)
+    if not base.exists():
+        return {
+            "ok": True,
+            "run_id": rid,
+            "scope": scope_norm,
+            "entries": [],
+            "truncated": False,
+            "note": f"no files under scope {scope_norm!r}",
+        }
+    if base.is_file():
+        entries = [
+            {
+                "path": str(base.relative_to(evidence)),
+                "name": base.name,
+                "type": "file",
+                "size": base.stat().st_size,
+            }
+        ]
+    else:
+        entries = _walk_tree(evidence, base, max_entries=MAX_TREE_ENTRIES)
+
+    # Agent scope: attach profile_id / group labels for virtual SPA folders (#27).
+    groups: list[dict[str, Any]] | None = None
+    if scope_norm == "agent":
+        entries, groups = _annotate_agent_tree_profiles(evidence, entries)
+
+    return {
+        "ok": True,
+        "run_id": rid,
+        "scope": scope_norm,
+        "entries": entries,
+        "groups": groups,
+        "truncated": len(entries) >= MAX_TREE_ENTRIES,
+    }
+
+
+def _annotate_agent_tree_profiles(
+    evidence: Path,
+    entries: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None]:
+    """Stamp profile_id on agent/invocations/* entries; build group list.
+
+    Disk layout stays flat; groups are read-side only. Real relative paths
+    are preserved for file preview.
+    """
+    inv_root = evidence / "agent" / "invocations"
+    if not inv_root.is_dir():
+        return entries, None
+
+    inv_meta: dict[str, dict[str, Any]] = {}
+    try:
+        inv_dirs = sorted(p for p in inv_root.iterdir() if p.is_dir())
+    except OSError:
+        inv_dirs = []
+    for inv in inv_dirs:
+        meta = _read_json_object(inv / "metadata.json") or {}
+        pid = meta.get("profile_id")
+        inv_meta[inv.name] = {
+            "profile_id": pid if isinstance(pid, str) else None,
+            "model": meta.get("model") or meta.get("locked_model"),
+            "dirname": inv.name,
+        }
+
+    if not inv_meta:
+        return entries, None
+
+    annotated: list[dict[str, Any]] = []
+    for e in entries:
+        path = str(e.get("path") or "")
+        # agent/invocations/<dirname>/...
+        parts = path.split("/")
+        profile_id = None
+        inv_dirname = None
+        if len(parts) >= 3 and parts[0] == "agent" and parts[1] == "invocations":
+            inv_dirname = parts[2]
+            meta = inv_meta.get(inv_dirname) or {}
+            profile_id = meta.get("profile_id")
+        ne = dict(e)
+        if profile_id:
+            ne["profile_id"] = profile_id
+        if inv_dirname:
+            ne["invocation"] = inv_dirname
+        annotated.append(ne)
+
+    # Stable group order: first appearance in inv dir sort order.
+    seen: list[str] = []
+    groups: list[dict[str, Any]] = []
+    for inv_name in sorted(inv_meta.keys()):
+        meta = inv_meta[inv_name]
+        pid = meta.get("profile_id")
+        key = pid if isinstance(pid, str) and pid else inv_name
+        if key in seen:
+            continue
+        seen.append(key)
+        groups.append(
+            {
+                "key": key,
+                "profile_id": pid if isinstance(pid, str) else None,
+                "label": pid if isinstance(pid, str) else inv_name,
+            }
+        )
+    return annotated, groups if groups else None
+
+
+def _walk_tree(evidence: Path, base: Path, *, max_entries: int) -> list[dict[str, Any]]:
+    out: list[dict[str, Any]] = []
+    if max_entries <= 0:
+        return out
+    try:
+        paths = sorted(base.rglob("*"), key=lambda p: str(p).lower())
+    except OSError:
+        return out
+    for p in paths:
+        if len(out) >= max_entries:
+            break
+        try:
+            rel = str(p.relative_to(evidence))
+            if p.is_dir():
+                out.append({"path": rel, "name": p.name, "type": "dir", "size": None})
+            elif p.is_file():
+                out.append(
+                    {
+                        "path": rel,
+                        "name": p.name,
+                        "type": "file",
+                        "size": p.stat().st_size,
+                    }
+                )
+        except (OSError, ValueError):
+            continue
+    return out
+
+
+def trial_file(
+    dataset_root: Path,
+    job_id: str,
+    task_id: str,
+    run_id: str,
+    *,
+    relpath: str,
+) -> dict[str, Any]:
+    root = dataset_root.expanduser().resolve(strict=False)
+    safe_id_segment(job_id, field="job_id")
+    task_id = safe_id_segment(task_id, field="task_id")
+    rid = _safe_run_id(run_id)
+    get_job(root, job_id)
+    evidence = resolve_evidence_root(root, rid, task_id=task_id, require_task_match=True)
+    path = _safe_under(evidence, relpath)
+    if not path.is_file():
+        raise ConfigError(
+            "unknown_task",
+            f"file not found: {relpath}",
+            location=relpath,
+        )
+    size = path.stat().st_size
+    mime, _ = mimetypes.guess_type(str(path))
+    mime = mime or "application/octet-stream"
+    suffix = path.suffix.lower()
+    # Never preview env/secret-like basenames even if under evidence
+    if path.name in {".env", ".env.local", ".env.production"} or path.name.startswith(".env."):
+        return {
+            "ok": True,
+            "run_id": rid,
+            "path": relpath,
+            "name": path.name,
+            "size": size,
+            "media_type": mime,
+            "encoding": "redacted",
+            "truncated": False,
+            "content": None,
+            "note": "secret-like filename; content not shown",
+        }
+    is_text = (
+        suffix in TEXT_SUFFIXES
+        or mime.startswith("text/")
+        or mime in {"application/json", "application/xml", "application/x-yaml"}
+    )
+    if not is_text:
+        return {
+            "ok": True,
+            "run_id": rid,
+            "path": relpath,
+            "name": path.name,
+            "size": size,
+            "media_type": mime,
+            "encoding": "binary",
+            "truncated": False,
+            "content": None,
+            "note": "binary file; preview not shown",
+        }
+    if size > MAX_FILE_BYTES:
+        raw = path.read_bytes()[:MAX_FILE_BYTES]
+        try:
+            text = raw.decode("utf-8")
+        except UnicodeDecodeError:
+            text = raw.decode("utf-8", errors="replace")
+        return {
+            "ok": True,
+            "run_id": rid,
+            "path": relpath,
+            "name": path.name,
+            "size": size,
+            "media_type": mime,
+            "encoding": "utf-8",
+            "truncated": True,
+            "content": text,
+            "note": f"truncated to first {MAX_FILE_BYTES} bytes",
+        }
+    try:
+        text = path.read_text(encoding="utf-8")
+    except UnicodeDecodeError:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    return {
+        "ok": True,
+        "run_id": rid,
+        "path": relpath,
+        "name": path.name,
+        "size": size,
+        "media_type": mime,
+        "encoding": "utf-8",
+        "truncated": False,
+        "content": text,
+    }

ageval/viewer/trials/meta.py

@@ -0,0 +1,257 @@
+"""Trial list and detail APIs for the local viewer."""
+
+from __future__ import annotations
+
+import contextlib
+import json
+from pathlib import Path
+from typing import Any
+
+from ageval.config.dataset import load_dataset_manifest
+from ageval.config.errors import ConfigError
+from ageval.viewer.browse import commands_for
+from ageval.viewer.jobs import get_job, get_job_task, safe_id_segment
+from ageval.viewer.trials.paths import (
+    _safe_run_id,
+    resolve_evidence_root,
+)
+from ageval.viewer.trials.surface import _trial_meta_from_evidence
+
+
+def list_task_trials(
+    dataset_root: Path,
+    job_id: str,
+    task_id: str,
+) -> dict[str, Any]:
+    """List trials for a task within a suite job, enriched with local evidence when present."""
+    root = dataset_root.expanduser().resolve(strict=False)
+    job_id = safe_id_segment(job_id, field="job_id")
+    task_id = safe_id_segment(task_id, field="task_id")
+    task_payload = get_job_task(root, job_id, task_id)
+    job = task_payload["job"]
+    suite_trials = list(task_payload.get("trials") or [])
+
+    by_run: dict[str, dict[str, Any]] = {}
+    for tr in suite_trials:
+        rid = tr.get("run_id") or tr.get("trial_id")
+        if not rid:
+            continue
+        rid_s = str(rid)
+        by_run[rid_s] = {
+            "trial_id": rid_s,
+            "run_id": rid_s,
+            "task_id": task_id,
+            "status": tr.get("status"),
+            "score": tr.get("score") if tr.get("score") is not None else tr.get("reward"),
+            "reward": tr.get("reward") if tr.get("reward") is not None else tr.get("score"),
+            "error": tr.get("error"),
+            "exit_code": tr.get("exit_code"),
+            "duration": tr.get("duration"),
+            "started": tr.get("started") or job.get("started"),
+            "has_evidence": False,
+            "available_tabs": [],
+            "note": "from suite summary",
+        }
+
+    # Scan dataset-level and task-local runs matching this task_id via lock.json
+    candidates: list[Path] = []
+    from ageval.evidence.locators import default_runs_root
+
+    db_runs = default_runs_root(root)
+    if db_runs.is_dir():
+        candidates.extend(p for p in db_runs.iterdir() if p.is_dir())
+    tasks_root_name = "tasks"
+    with contextlib.suppress(ConfigError):
+        man = load_dataset_manifest(root)
+        tasks_root_name = man.tasks_root or "tasks"
+    task_runs = default_runs_root(root / tasks_root_name / task_id)
+    if task_runs.is_dir():
+        candidates.extend(p for p in task_runs.iterdir() if p.is_dir())
+
+    for run_dir in candidates:
+        rid = run_dir.name
+        # Only enrich this job's attempts — extra local runs of the same task
+        # belong on the jobs list as single jobs, not this suite's trial list.
+        if rid not in by_run:
+            continue
+        suite_row = by_run.get(rid, {})
+        meta = _trial_meta_from_evidence(
+            run_dir,
+            run_id=rid,
+            task_id=task_id,
+            suite_row=suite_row,
+        )
+        try:
+            rel = str(run_dir.resolve(strict=False).relative_to(root))
+        except ValueError:
+            rel = str(run_dir)
+        meta["evidence_relpath"] = rel
+        meta["started"] = meta.get("started") or suite_row.get("started") or job.get("started")
+        by_run[rid] = meta
+
+    trials = sorted(
+        by_run.values(),
+        key=lambda t: (t.get("started") or "", t.get("run_id") or ""),
+        reverse=True,
+    )
+    cmds = commands_for(root, task_id=task_id)
+    return {
+        "ok": True,
+        "job": job,
+        "task": task_payload["task"],
+        "trials": trials,
+        "count": len(trials),
+        "commands": cmds,
+        "run_command": cmds.get("run_task") or cmds.get("run_suite"),
+        "breadcrumb": [
+            {"label": "Jobs", "href": "/"},
+            {"label": job_id, "href": f"/jobs/{job_id}"},
+            {"label": task_id, "href": f"/jobs/{job_id}/tasks/{task_id}"},
+            {"label": "trials", "href": None},
+        ],
+        "note": None,
+    }
+
+
+def get_trial(
+    dataset_root: Path,
+    job_id: str,
+    task_id: str,
+    run_id: str,
+) -> dict[str, Any]:
+    root = dataset_root.expanduser().resolve(strict=False)
+    job_id = safe_id_segment(job_id, field="job_id")
+    task_id = safe_id_segment(task_id, field="task_id")
+    rid = _safe_run_id(run_id)
+    job_payload = get_job(root, job_id)
+    job = job_payload["job"]
+    suite_row: dict[str, Any] = {}
+    for row in job_payload.get("tasks") or []:
+        if row.get("task_id") != task_id:
+            continue
+        extra = row.get("attempt_run_ids")
+        extra_ids = (
+            {str(x).strip() for x in extra if x is not None and str(x).strip()}
+            if isinstance(extra, list)
+            else set()
+        )
+        prev_ids: set[str] = set()
+        raw_prev = row.get("previous")
+        if isinstance(raw_prev, list):
+            for item in raw_prev:
+                if isinstance(item, dict) and item.get("run_id"):
+                    prev_ids.add(str(item["run_id"]).strip())
+        if str(row.get("run_id") or "") != rid and rid not in extra_ids and rid not in prev_ids:
+            continue
+        suite_row = dict(row)
+        for attempt in row.get("attempts") or []:
+            if isinstance(attempt, dict) and str(attempt.get("run_id") or "") == rid:
+                suite_row = {**row, **attempt}
+                break
+        break
+
+    evidence: Path | None = None
+    with contextlib.suppress(ConfigError):
+        evidence = resolve_evidence_root(root, rid, task_id=task_id, require_task_match=True)
+
+    result_preview: dict[str, Any] | None = None
+    if evidence is None:
+        if not suite_row:
+            raise ConfigError(
+                "unknown_task",
+                f"trial {rid!r} not found for task {task_id!r} in job {job_id!r}",
+                location=rid,
+            )
+        status = suite_row.get("status")
+        meta = {
+            "trial_id": rid,
+            "run_id": rid,
+            "task_id": task_id,
+            "status": str(status).upper() if status else None,
+            "score": suite_row.get("score"),
+            "reward": suite_row.get("score"),
+            "error": suite_row.get("error"),
+            "exit_code": suite_row.get("exit_code"),
+            "duration": suite_row.get("duration"),
+            "started": suite_row.get("started") or job.get("started"),
+            "evidence_relpath": None,
+            "has_evidence": False,
+            "available_tabs": [],
+            "note": "no local evidence tree for this run_id",
+        }
+    else:
+        meta = _trial_meta_from_evidence(evidence, run_id=rid, task_id=task_id, suite_row=suite_row)
+        try:
+            meta["evidence_relpath"] = str(evidence.resolve(strict=False).relative_to(root))
+        except ValueError:
+            meta["evidence_relpath"] = str(evidence)
+        from ageval.evidence.attempt_record import read_attempt_result
+
+        result_preview = read_attempt_result(evidence)
+        if result_preview and "metrics" in result_preview:
+            metrics = result_preview.get("metrics")
+            if isinstance(metrics, dict) and len(json.dumps(metrics)) > 8_000:
+                result_preview = {**result_preview, "metrics": {"_truncated": True}}
+
+    listed = list_task_trials(root, job_id, task_id)
+    sibling_ids = [str(t.get("run_id")) for t in listed["trials"] if t.get("run_id")]
+    if rid not in sibling_ids and rid not in {
+        str(item.get("run_id") or "").strip()
+        for item in (suite_row.get("previous") or [])
+        if isinstance(item, dict)
+    }:
+        sibling_ids.insert(0, rid)
+    from ageval.application.suite.suite_metrics import attempt_started_at
+    from ageval.viewer.jobs import _started_from_run_dir
+
+    slot_previous = [
+        dict(item) for item in (suite_row.get("previous") or []) if isinstance(item, dict)
+    ]
+    for item in slot_previous:
+        if item.get("started_at"):
+            continue
+        prev_id = str(item.get("run_id") or "").strip()
+        if prev_id:
+            started = _started_from_run_dir(root, prev_id)
+            if started:
+                item["started_at"] = started
+    slot_current = str(suite_row.get("run_id") or "") or (sibling_ids[0] if sibling_ids else rid)
+    slot_current_started = None
+    for attempt in suite_row.get("attempts") or []:
+        if isinstance(attempt, dict) and str(attempt.get("run_id") or "") == slot_current:
+            slot_current_started = attempt_started_at(attempt)
+            break
+    if not slot_current_started:
+        slot_current_started = attempt_started_at(suite_row)
+    if not slot_current_started and slot_current:
+        slot_current_started = _started_from_run_dir(root, slot_current)
+    try:
+        idx = sibling_ids.index(rid)
+    except ValueError:
+        idx = -1
+    prev_id = sibling_ids[idx - 1] if idx > 0 else None
+    next_id = sibling_ids[idx + 1] if 0 <= idx < len(sibling_ids) - 1 else None
+    cmds = commands_for(root, task_id=task_id)
+
+    return {
+        "ok": True,
+        "job": job,
+        "task_id": task_id,
+        "trial": meta,
+        "result": result_preview,
+        "prev_run_id": prev_id,
+        "next_run_id": next_id,
+        "sibling_run_ids": sibling_ids,
+        "slot_current_run_id": slot_current,
+        "slot_current_started_at": slot_current_started,
+        "slot_previous": slot_previous,
+        "commands": cmds,
+        "run_command": cmds.get("run_task") or cmds.get("run_suite"),
+        "breadcrumb": [
+            {"label": "Jobs", "href": "/"},
+            {"label": job_id, "href": f"/jobs/{job_id}"},
+            {"label": task_id, "href": f"/jobs/{job_id}/tasks/{task_id}"},
+            {"label": rid, "href": None},
+        ],
+        "note": meta.get("note"),
+    }