eval-builder 0.1.0__py3-none-any.whl

eval_builder/select.py

@@ -0,0 +1,412 @@
+"""Pick a small, diverse, representative set of traces without calling any model.
+
+Pipeline: exact dedupe (normalized text hash) -> near-duplicate merge (character
+n-gram cosine) -> k-means topic clusters -> failure oversampling ->
+stratum coverage -> one central example per cluster -> proportional fill with
+farthest-point sampling. Every pick records the reasons it was picked.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+from scipy import sparse
+from sklearn.cluster import KMeans
+from sklearn.feature_extraction.text import TfidfVectorizer
+
+from .ingest import load_traces
+from .io import write_json
+from .schema import Trace
+from .workspace import Workspace
+
+DEFAULT_STRATA = ("route", "tools", "error", "feedback")
+
+_WS = re.compile(r"\s+")
+
+
+def normalize_text(s: str) -> str:
+    return _WS.sub(" ", s.lower()).strip()
+
+
+def text_hash(s: str) -> str:
+    return hashlib.sha256(normalize_text(s).encode("utf-8")).hexdigest()
+
+
+def is_failure(t: Trace) -> bool:
+    return bool(t.error) or t.feedback == "negative"
+
+
+def failure_kind(t: Trace) -> str:
+    kinds = []
+    if t.error:
+        kinds.append("error flag")
+    if t.feedback == "negative":
+        kinds.append("negative user feedback")
+    return " + ".join(kinds)
+
+
+def user_text(t: Trace) -> str:
+    """Everything the user said in the conversation, in order.
+
+    Using only the last user message would merge unrelated conversations that end
+    with the same generic follow-up ("make it shorter").
+    """
+    turns = [m["content"] for m in t.messages if m.get("role") == "user"]
+    return "\n".join(turns) if turns else t.input
+
+
+def _key_text(t: Trace, dedupe_on: str) -> str:
+    return user_text(t) if dedupe_on == "input" else f"{user_text(t)}\n{t.output}"
+
+
+class _UnionFind:
+    def __init__(self, n: int) -> None:
+        self.p = list(range(n))
+
+    def find(self, x: int) -> int:
+        while self.p[x] != x:
+            self.p[x] = self.p[self.p[x]]
+            x = self.p[x]
+        return x
+
+    def union(self, a: int, b: int) -> None:
+        ra, rb = self.find(a), self.find(b)
+        if ra != rb:
+            self.p[max(ra, rb)] = min(ra, rb)
+
+
+def near_duplicate_groups(texts: list[str], threshold: float, chunk: int = 1000) -> list[list[int]]:
+    """Group texts whose character 3-5 gram cosine similarity is >= threshold."""
+    n = len(texts)
+    if n < 2:
+        return [[i] for i in range(n)]
+    # Plain term-frequency vectors (no IDF) so the similarity of two texts does not
+    # depend on what else is in the corpus.
+    vec = TfidfVectorizer(analyzer="char_wb", ngram_range=(3, 5), lowercase=True, use_idf=False)
+    try:
+        x = vec.fit_transform(texts)
+    except ValueError:  # all empty
+        return [[i] for i in range(n)]
+    uf = _UnionFind(n)
+    for start in range(0, n, chunk):
+        coo = sparse.coo_matrix(x[start : start + chunk] @ x.T)
+        for r, c, v in zip(coo.row, coo.col, coo.data, strict=True):
+            i = start + int(r)
+            if int(c) > i and v >= threshold:
+                uf.union(i, int(c))
+    groups: dict[int, list[int]] = defaultdict(list)
+    for i in range(n):
+        groups[uf.find(i)].append(i)
+    return sorted(groups.values(), key=lambda g: g[0])
+
+
+def _pick_rep(members: list[int], traces: list[Trace]) -> int:
+    for i in members:
+        if is_failure(traces[i]):
+            return i
+    return members[0]
+
+
+def _strata_values(t: Trace, field: str) -> list[str]:
+    if field == "tools":
+        return [f"{tool}" for tool in t.tools] or ["(none)"]
+    if field == "error":
+        return [str(bool(t.error)).lower()]
+    if field == "feedback":
+        return [t.feedback or "(none)"]
+    if field == "route":
+        return [t.route] if t.route else []
+    if field == "model":
+        return [t.model] if t.model else []
+    cur: Any = t.metadata
+    for part in field.split("."):
+        if isinstance(cur, dict) and part in cur:
+            cur = cur[part]
+        else:
+            return []
+    if isinstance(cur, list):
+        return [str(x) for x in cur]
+    return [str(cur)] if cur is not None else []
+
+
+def select(
+    workspace: str | Path,
+    n: int = 50,
+    seed: int = 0,
+    clusters: int | None = None,
+    failure_share: float = 0.3,
+    near_dup_threshold: float = 0.9,
+    dedupe_on: str = "input",
+    stratify: list[str] | None = None,
+) -> dict[str, Any]:
+    if n < 1:
+        raise ValueError("n must be >= 1")
+    if dedupe_on not in ("input", "input+output"):
+        raise ValueError("dedupe_on must be 'input' or 'input+output'")
+    ws = Workspace.at(workspace)
+    traces = load_traces(ws.root)
+    if not traces:
+        raise ValueError("no traces in workspace; run ingest first")
+    result = select_traces(
+        traces,
+        n=n,
+        seed=seed,
+        clusters=clusters,
+        failure_share=failure_share,
+        near_dup_threshold=near_dup_threshold,
+        dedupe_on=dedupe_on,
+        stratify=stratify,
+    )
+    write_json(ws.selection, result)
+    return result
+
+
+def select_traces(
+    traces: list[Trace],
+    n: int = 50,
+    seed: int = 0,
+    clusters: int | None = None,
+    failure_share: float = 0.3,
+    near_dup_threshold: float = 0.9,
+    dedupe_on: str = "input",
+    stratify: list[str] | None = None,
+) -> dict[str, Any]:
+    # 1. exact dedupe
+    exact: dict[str, list[int]] = defaultdict(list)
+    for i, t in enumerate(traces):
+        exact[text_hash(_key_text(t, dedupe_on))].append(i)
+    exact_groups = sorted(exact.values(), key=lambda g: g[0])
+    reps = [_pick_rep(g, traces) for g in exact_groups]
+    members_of: dict[int, list[int]] = {r: g for r, g in zip(reps, exact_groups, strict=True)}
+
+    # 2. near-duplicate merge among exact representatives
+    near = near_duplicate_groups(
+        [_key_text(traces[r], dedupe_on) for r in reps], near_dup_threshold
+    )
+    unique: list[int] = []
+    group_members: dict[int, list[int]] = {}
+    near_merged = 0
+    for g in near:
+        all_members = [m for gi in g for m in members_of[reps[gi]]]
+        rep = _pick_rep(sorted(all_members), traces)
+        unique.append(rep)
+        group_members[rep] = sorted(all_members)
+        if len(g) > 1:
+            near_merged += len(g) - 1
+    unique.sort()
+    u_traces = [traces[i] for i in unique]
+    nu = len(unique)
+
+    # 3. topic clusters over the user side of each conversation
+    vec = TfidfVectorizer(
+        ngram_range=(1, 2), sublinear_tf=True, stop_words="english", max_features=20000, min_df=1
+    )
+    try:
+        x = vec.fit_transform([user_text(t) for t in u_traces])
+        terms = np.array(vec.get_feature_names_out())
+    except ValueError:
+        x = sparse.csr_matrix(np.ones((nu, 1)))
+        terms = np.array(["(empty)"])
+    k = clusters or max(2, round(math.sqrt(nu)))
+    k = max(1, min(k, nu, n))
+    if k >= 2:
+        km = KMeans(n_clusters=k, n_init=10, random_state=seed)
+        labels = km.fit_predict(x)
+        centers = km.cluster_centers_
+    else:
+        labels = np.zeros(nu, dtype=int)
+        centers = np.asarray(x.mean(axis=0))
+    dense_dist = np.asarray(
+        [np.linalg.norm(x[i].toarray().ravel() - centers[labels[i]]) for i in range(nu)]
+    )
+    cluster_info: list[dict[str, Any]] = []
+    for c in range(k):
+        idx = np.where(labels == c)[0]
+        top = [str(terms[j]) for j in np.argsort(-centers[c])[:5] if centers[c][j] > 0]
+        cluster_info.append(
+            {"id": c, "size": int(len(idx)), "share": round(len(idx) / nu, 4), "terms": top}
+        )
+
+    # helpers
+    selected: dict[int, list[str]] = {}  # position in u_traces -> reasons
+
+    def add(pos: int, reason: str) -> None:
+        selected.setdefault(pos, []).append(reason)
+
+    def central_order(positions: list[int]) -> list[int]:
+        return sorted(positions, key=lambda p: (dense_dist[p], p))
+
+    def cluster_note(c: int) -> str:
+        info = cluster_info[c]
+        terms = ", ".join(info["terms"][:3])
+        return f"cluster {c} ({info['size']} traces, {info['share']:.0%}; {terms})"
+
+    # 4. failure oversampling
+    fail_pos = [p for p, t in enumerate(u_traces) if is_failure(t)]
+    pop_fail_share = len(fail_pos) / nu if nu else 0.0
+    fail_budget = min(
+        len(fail_pos), n, max(math.ceil(failure_share * n), round(pop_fail_share * n))
+    )
+    if fail_budget:
+        by_cluster: dict[int, list[int]] = defaultdict(list)
+        for p in central_order(fail_pos):
+            by_cluster[int(labels[p])].append(p)
+        order = sorted(by_cluster, key=lambda c: (-cluster_info[c]["size"], c))
+        picked = 0
+        while picked < fail_budget:
+            progressed = False
+            for c in order:
+                if by_cluster[c] and picked < fail_budget:
+                    p = by_cluster[c].pop(0)
+                    add(
+                        p,
+                        f"failure ({failure_kind(u_traces[p])}); {pop_fail_share:.0%} of unique "
+                        f"traces are failures and at least {failure_share:.0%} of picks are "
+                        f"reserved for them",
+                    )
+                    picked += 1
+                    progressed = True
+            if not progressed:
+                break
+
+    # 5. stratum coverage
+    fields = list(DEFAULT_STRATA) + [f for f in (stratify or []) if f not in DEFAULT_STRATA]
+    strata_pop: dict[str, Counter[str]] = {}
+    skipped_fields: dict[str, str] = {}
+    max_card = max(10, n // 2)
+    for f in fields:
+        counts: Counter[str] = Counter()
+        for t in u_traces:
+            counts.update(_strata_values(t, f))
+        if not counts or (len(counts) == 1 and f in DEFAULT_STRATA):
+            continue
+        if len(counts) > max_card:
+            skipped_fields[f] = f"{len(counts)} distinct values (limit {max_card})"
+            continue
+        strata_pop[f] = counts
+    for f, counts in strata_pop.items():
+        for value, cnt in sorted(counts.items(), key=lambda kv: (kv[1], kv[0])):
+            if len(selected) >= n:
+                break
+            members = [p for p, t in enumerate(u_traces) if value in _strata_values(t, f)]
+            if any(p in selected for p in members):
+                continue
+            p = central_order(members)[0]
+            add(p, f"covers {f}={value} ({cnt} unique traces, {cnt / nu:.0%})")
+
+    # 6. one central example per uncovered cluster
+    covered = {int(labels[p]) for p in selected}
+    for c in sorted(range(k), key=lambda c: (-cluster_info[c]["size"], c)):
+        if len(selected) >= n:
+            break
+        if c in covered:
+            continue
+        members = [p for p in range(nu) if labels[p] == c]
+        add(central_order(members)[0], f"central example of {cluster_note(c)}")
+
+    # 7. proportional fill with farthest-point sampling inside each cluster
+    remaining = n - len(selected)
+    if remaining > 0:
+        quotas = {c: n * cluster_info[c]["size"] / nu for c in range(k)}
+        have = Counter(int(labels[p]) for p in selected)
+        need = {c: max(0.0, quotas[c] - have[c]) for c in range(k)}
+        floor = {c: int(need[c]) for c in range(k)}
+        left = remaining - sum(floor.values())
+        for c in sorted(range(k), key=lambda c: (-(need[c] - floor[c]), c)):
+            if left <= 0:
+                break
+            floor[c] += 1
+            left -= 1
+        for c in range(k):
+            for _ in range(floor[c]):
+                if len(selected) >= n:
+                    break
+                far = _farthest(
+                    x,
+                    [q for q in range(nu) if labels[q] == c and q not in selected],
+                    [q for q in selected if labels[q] == c],
+                )
+                if far is None:
+                    break
+                add(
+                    far,
+                    f"adds variety within {cluster_note(c)}; least similar to cases already "
+                    f"picked there",
+                )
+        while len(selected) < min(n, nu):
+            far = _farthest(x, [q for q in range(nu) if q not in selected], list(selected))
+            if far is None:
+                break
+            add(far, "adds variety: least similar to every case already picked")
+
+    # 8. assemble
+    picks = []
+    for p in sorted(selected, key=lambda p: (int(labels[p]), p)):
+        t = u_traces[p]
+        grp = group_members[unique[p]]
+        picks.append(
+            {
+                "trace_id": t.id,
+                "reasons": selected[p],
+                "cluster": int(labels[p]),
+                "represents": len(grp),
+                "duplicates": [traces[i].id for i in grp if i != unique[p]],
+                "failure": is_failure(t),
+                "failing_in_group": sum(1 for i in grp if is_failure(traces[i])),
+                "strata": {f: _strata_values(t, f) for f in strata_pop},
+            }
+        )
+    sel_counts = Counter(int(labels[p]) for p in selected)
+    for info in cluster_info:
+        info["selected"] = sel_counts.get(info["id"], 0)
+    strata_report = {
+        f: {
+            v: {
+                "population": cnt,
+                "selected": sum(1 for p in selected if v in _strata_values(u_traces[p], f)),
+            }
+            for v, cnt in sorted(counts.items())
+        }
+        for f, counts in strata_pop.items()
+    }
+    return {
+        "params": {
+            "n": n,
+            "seed": seed,
+            "clusters": k,
+            "failure_share": failure_share,
+            "near_dup_threshold": near_dup_threshold,
+            "dedupe_on": dedupe_on,
+            "stratify": fields,
+        },
+        "population": {
+            "traces": len(traces),
+            "exact_duplicates_removed": len(traces) - len(exact_groups),
+            "near_duplicates_merged": near_merged,
+            "unique": nu,
+            "failures_unique": len(fail_pos),
+        },
+        "clusters": cluster_info,
+        "strata": strata_report,
+        "strata_skipped": skipped_fields,
+        "selected_count": len(picks),
+        "selected_failures": sum(1 for p in picks if p["failure"]),
+        "selected": picks,
+    }
+
+
+def _farthest(x: sparse.csr_matrix, candidates: list[int], chosen: list[int]) -> int | None:
+    if not candidates:
+        return None
+    if not chosen:
+        return candidates[0]
+    sims = (x[candidates] @ x[chosen].T).toarray()
+    max_sim = sims.max(axis=1)
+    best = int(np.argmin(max_sim))  # first index wins ties: deterministic
+    return candidates[best]

eval_builder/setup_agents.py

@@ -0,0 +1,190 @@
+"""`eval-builder setup`: register the MCP server with Claude Code, Codex and Cursor.
+
+Shows every change first. Applies only with --yes. Backs up any file it edits.
+Running it twice changes nothing the second time.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+import subprocess
+import tomllib
+from dataclasses import dataclass, field
+from datetime import datetime
+from importlib import resources
+from pathlib import Path
+from typing import Any
+
+NAME = "eval-builder"
+DEFAULT_SERVER = ["uvx", NAME, "mcp"]
+
+
+@dataclass
+class Action:
+    agent: str
+    kind: str  # "command" | "edit" | "copy" | "skip"
+    target: str
+    detail: str
+    apply_fn: Any = field(default=None, repr=False)
+
+    def as_dict(self) -> dict[str, str]:
+        return {
+            "agent": self.agent,
+            "kind": self.kind,
+            "target": self.target,
+            "detail": self.detail,
+        }
+
+
+def skill_text() -> str:
+    try:
+        return resources.files("eval_builder").joinpath("data/SKILL.md").read_text("utf-8")
+    except (FileNotFoundError, ModuleNotFoundError):
+        repo = Path(__file__).resolve().parents[2] / "skills" / NAME / "SKILL.md"
+        return repo.read_text("utf-8")
+
+
+def _backup(path: Path) -> Path | None:
+    if not path.exists():
+        return None
+    stamp = datetime.now().strftime("%Y%m%d%H%M%S")
+    dest = path.with_name(f"{path.name}.bak-{NAME}-{stamp}")
+    shutil.copy2(path, dest)
+    return dest
+
+
+def _claude_action(server: list[str], path_env: str | None) -> Action:
+    exe = shutil.which("claude", path=path_env)
+    if not exe:
+        return Action(
+            "Claude Code",
+            "skip",
+            "claude CLI",
+            "claude CLI not found on PATH; to add later run: claude mcp add --scope "
+            f"user {NAME} -- {' '.join(server)}",
+        )
+    probe = subprocess.run([exe, "mcp", "get", NAME], capture_output=True, text=True, timeout=60)
+    if probe.returncode == 0:
+        return Action("Claude Code", "skip", "claude mcp", f"{NAME} already registered")
+    argv = [exe, "mcp", "add", "--scope", "user", NAME, "--", *server]
+
+    def run() -> str:
+        res = subprocess.run(argv, capture_output=True, text=True, timeout=60)
+        if res.returncode != 0:
+            raise RuntimeError(res.stderr.strip() or res.stdout.strip())
+        return res.stdout.strip()
+
+    return Action("Claude Code", "command", "claude mcp", "run: claude " + " ".join(argv[1:]), run)
+
+
+def _codex_action(home: Path, server: list[str], path_env: str | None) -> Action:
+    codex_dir = home / ".codex"
+    if not codex_dir.exists() and not shutil.which("codex", path=path_env):
+        return Action("Codex", "skip", str(codex_dir), "Codex not detected")
+    cfg = codex_dir / "config.toml"
+    existing = cfg.read_text("utf-8") if cfg.exists() else ""
+    try:
+        parsed = tomllib.loads(existing) if existing else {}
+    except tomllib.TOMLDecodeError as e:
+        return Action(
+            "Codex", "skip", str(cfg), f"config.toml does not parse ({e}); not touching it"
+        )
+    if NAME in (parsed.get("mcp_servers") or {}):
+        return Action("Codex", "skip", str(cfg), f"[mcp_servers.{NAME}] already present")
+    block = f'\n[mcp_servers.{NAME}]\ncommand = "{server[0]}"\nargs = {json.dumps(server[1:])}\n'
+
+    def run() -> str:
+        codex_dir.mkdir(parents=True, exist_ok=True)
+        b = _backup(cfg)
+        with open(cfg, "a", encoding="utf-8") as f:
+            if existing and not existing.endswith("\n"):
+                f.write("\n")
+            f.write(block)
+        return f"appended block; backup: {b}" if b else "created config.toml"
+
+    return Action("Codex", "edit", str(cfg), f"append:{block}", run)
+
+
+def _cursor_action(home: Path, server: list[str], path_env: str | None) -> Action:
+    cursor_dir = home / ".cursor"
+    if not cursor_dir.exists() and not shutil.which("cursor", path=path_env):
+        return Action("Cursor", "skip", str(cursor_dir), "Cursor not detected")
+    cfg = cursor_dir / "mcp.json"
+    data: dict[str, Any] = {}
+    if cfg.exists():
+        try:
+            data = json.loads(cfg.read_text("utf-8") or "{}")
+        except json.JSONDecodeError as e:
+            return Action(
+                "Cursor", "skip", str(cfg), f"mcp.json does not parse ({e}); not touching it"
+            )
+    entry = {"command": server[0], "args": server[1:]}
+    if (data.get("mcpServers") or {}).get(NAME) == entry:
+        return Action("Cursor", "skip", str(cfg), f"{NAME} already present")
+
+    def run() -> str:
+        cursor_dir.mkdir(parents=True, exist_ok=True)
+        b = _backup(cfg)
+        data.setdefault("mcpServers", {})[NAME] = entry
+        cfg.write_text(json.dumps(data, indent=2) + "\n", encoding="utf-8")
+        return f"wrote mcpServers.{NAME}; backup: {b}" if b else f"created {cfg}"
+
+    return Action("Cursor", "edit", str(cfg), f"set mcpServers.{NAME} = {json.dumps(entry)}", run)
+
+
+def _skill_actions(home: Path) -> list[Action]:
+    text = skill_text()
+    out = []
+    for agent, base in (("Claude Code", home / ".claude"), ("Codex", home / ".codex")):
+        dest = base / "skills" / NAME / "SKILL.md"
+        if not base.exists():
+            out.append(Action(agent, "skip", str(dest), f"{base} not found"))
+            continue
+        if dest.exists() and dest.read_text("utf-8") == text:
+            out.append(Action(agent, "skip", str(dest), "skill already up to date"))
+            continue
+
+        def run(dest: Path = dest) -> str:
+            dest.parent.mkdir(parents=True, exist_ok=True)
+            b = _backup(dest)
+            dest.write_text(text, encoding="utf-8")
+            return f"wrote skill; backup: {b}" if b else "wrote skill"
+
+        out.append(Action(agent, "copy", str(dest), "install agent instructions (SKILL.md)", run))
+    return out
+
+
+def plan(
+    home: Path | None = None, server: list[str] | None = None, path_env: str | None = None
+) -> list[Action]:
+    home = home or Path.home()
+    server = server or DEFAULT_SERVER
+    return [
+        _claude_action(server, path_env),
+        _codex_action(home, server, path_env),
+        _cursor_action(home, server, path_env),
+        *_skill_actions(home),
+    ]
+
+
+def setup(
+    yes: bool = False,
+    home: Path | None = None,
+    server: list[str] | None = None,
+    path_env: str | None = None,
+) -> dict[str, Any]:
+    actions = plan(home, server, path_env)
+    result: dict[str, Any] = {"applied": False, "actions": [a.as_dict() for a in actions]}
+    if not yes:
+        result["next"] = "re-run with --yes to apply the changes above"
+        return result
+    for a, d in zip(actions, result["actions"], strict=True):
+        if a.apply_fn is None:
+            continue
+        try:
+            d["result"] = a.apply_fn()
+        except Exception as e:  # report and continue with the other agents
+            d["result"] = f"failed: {e}"
+    result["applied"] = True
+    return result

eval_builder/status.py

@@ -0,0 +1,32 @@
+"""Summarize which steps have run in a workspace and suggest the next one."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from .workspace import Workspace
+
+
+def status(workspace: str | Path) -> dict[str, Any]:
+    ws = Workspace.at(workspace)
+    steps = {
+        "ingest": ws.traces.exists(),
+        "select": ws.selection.exists(),
+        "draft": ws.cases.exists(),
+        "judge_plan": ws.judge_requests.exists(),
+        "judgments": ws.judgments.exists(),
+        "labels": ws.labels.exists(),
+        "judge_check": ws.judge_check.exists(),
+        "export": (ws.exports / "manifest.json").exists(),
+        "report": ws.report_md.exists(),
+    }
+    order = [
+        ("ingest", "eval-builder ingest <logs>"),
+        ("select", "eval-builder select"),
+        ("draft", "eval-builder draft"),
+        ("export", "fill cases, then eval-builder export"),
+        ("report", "eval-builder report"),
+    ]
+    nxt = next((cmd for step, cmd in order if not steps[step]), "done")
+    return {"workspace": str(ws.root), "steps": steps, "next": nxt}

eval_builder/workspace.py

@@ -0,0 +1,67 @@
+"""Standard file layout inside a workspace directory. Every step reads and writes here."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Workspace:
+    root: Path
+
+    @classmethod
+    def at(cls, path: str | Path) -> Workspace:
+        return cls(Path(path))
+
+    def ensure(self) -> Workspace:
+        self.root.mkdir(parents=True, exist_ok=True)
+        return self
+
+    @property
+    def traces(self) -> Path:
+        return self.root / "traces.jsonl"
+
+    @property
+    def ingest_report(self) -> Path:
+        return self.root / "ingest.json"
+
+    @property
+    def selection(self) -> Path:
+        return self.root / "selection.json"
+
+    @property
+    def cases(self) -> Path:
+        return self.root / "cases.yaml"
+
+    @property
+    def rubric(self) -> Path:
+        return self.root / "rubric.yaml"
+
+    @property
+    def judge_requests(self) -> Path:
+        return self.root / "judge_requests.jsonl"
+
+    @property
+    def judgments(self) -> Path:
+        return self.root / "judgments.jsonl"
+
+    @property
+    def labels(self) -> Path:
+        return self.root / "labels.jsonl"
+
+    @property
+    def judge_check(self) -> Path:
+        return self.root / "judge_check.json"
+
+    @property
+    def exports(self) -> Path:
+        return self.root / "exports"
+
+    @property
+    def report_md(self) -> Path:
+        return self.root / "report.md"
+
+    @property
+    def report_json(self) -> Path:
+        return self.root / "report.json"