blitz-cli 0.12.0__tar.gz → 0.13.0__tar.gz

{blitz_cli-0.12.0 → blitz_cli-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: blitz-cli
-Version: 0.12.0
+Version: 0.13.0
 Summary: Developer CLI for Blitz locker-room: spec-driven playbooks, a codebase/huddle ontology graph with an MCP tool, and the composable watcher/trainer/graph containers.
 License: MIT
 License-File: LICENSE

blitz_cli-0.13.0/blitz_cli/_collect.py

@@ -0,0 +1,383 @@
+"""DPO data-collection store and preference-pair tooling for
+`blitz model execute --collect` and the `blitz model pairs` command family.
+
+`CollectStore` owns four tables (`collect_runs`, `collect_attempts`,
+`dpo_pairs`, `sft_queue`) in the same locker-room activity database that
+`_corpus.py`'s `TurnStore` writes its transcript corpus to (default
+`data/activity.db`, env override `ACTIVITY_DB_PATH`, WAL journal). Like
+`TurnStore`, it is the one writer of its tables and creates its schema on
+first connect.
+
+The execute loop inserts one `collect_runs` row per `--collect` invocation
+and one `collect_attempts` row per model call, capturing the exact prompt
+messages sent to the model verbatim. `extract_pairs` turns those attempts
+into chosen/rejected DPO preference pairs (pass/fail contrast on byte
+identical prompt lineage, or judge-ranked green-vs-green), `export_pairs`
+emits TRL-shaped JSONL, and `stats` reports collection health per run.
+`sft_queue` quarantines placeholders for tasks no chain solved; it never
+feeds pairs.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sqlite3
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+_SCHEMA = """
+CREATE TABLE IF NOT EXISTS collect_runs (
+    run_id TEXT PRIMARY KEY,
+    repo_path TEXT NOT NULL,
+    spec TEXT NOT NULL,
+    tasks_file TEXT NOT NULL,
+    model TEXT NOT NULL,
+    endpoint TEXT NOT NULL,
+    temperature REAL NOT NULL,
+    max_retries INTEGER NOT NULL,
+    k INTEGER NOT NULL,
+    started_at TEXT NOT NULL,
+    finished_at TEXT
+);
+CREATE INDEX IF NOT EXISTS idx_collect_runs_started_at ON collect_runs(started_at);
+
+CREATE TABLE IF NOT EXISTS collect_attempts (
+    attempt_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL REFERENCES collect_runs(run_id),
+    task_id TEXT NOT NULL,
+    sample_index INTEGER NOT NULL,
+    attempt_index INTEGER NOT NULL,
+    prompt_messages_json TEXT NOT NULL,
+    reply TEXT NOT NULL,
+    actions_json TEXT NOT NULL,
+    passed INTEGER NOT NULL,
+    error_detail TEXT,
+    guard_violation TEXT,
+    input_tokens INTEGER,
+    output_tokens INTEGER,
+    env_failure INTEGER NOT NULL DEFAULT 0,
+    judge_rank INTEGER,
+    judge_notes TEXT,
+    created_at TEXT NOT NULL,
+    UNIQUE(run_id, task_id, sample_index, attempt_index)
+);
+CREATE INDEX IF NOT EXISTS idx_attempts_task_lineage
+    ON collect_attempts(task_id, prompt_messages_json);
+CREATE INDEX IF NOT EXISTS idx_attempts_run ON collect_attempts(run_id);
+
+CREATE TABLE IF NOT EXISTS dpo_pairs (
+    pair_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    chosen_attempt_id INTEGER NOT NULL REFERENCES collect_attempts(attempt_id),
+    rejected_attempt_id INTEGER NOT NULL REFERENCES collect_attempts(attempt_id),
+    task_id TEXT NOT NULL,
+    chosen_source TEXT NOT NULL CHECK (chosen_source IN ('local', 'judge-ranked')),
+    status TEXT NOT NULL DEFAULT 'candidate'
+        CHECK (status IN ('candidate', 'approved', 'rejected')),
+    judge_notes TEXT,
+    created_at TEXT NOT NULL,
+    UNIQUE(chosen_attempt_id, rejected_attempt_id),
+    CHECK (chosen_attempt_id != rejected_attempt_id)
+);
+
+CREATE TABLE IF NOT EXISTS sft_queue (
+    item_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    run_id TEXT NOT NULL REFERENCES collect_runs(run_id),
+    task_id TEXT NOT NULL,
+    prompt_messages_json TEXT NOT NULL,
+    solution_text TEXT,
+    source TEXT NOT NULL DEFAULT 'placeholder',
+    status TEXT NOT NULL DEFAULT 'needs_solution',
+    created_at TEXT NOT NULL,
+    UNIQUE(run_id, task_id)
+);
+"""
+
+
+def default_db_path() -> Path:
+    """The activity database path: ACTIVITY_DB_PATH env override, else the
+    conventional data/activity.db relative to the CWD (same idiom as _train.py)."""
+    return Path(os.environ.get("ACTIVITY_DB_PATH", "data/activity.db"))
+
+
+def _now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class CollectStore:
+    """Writer for the DPO collection tables. Owns its schema and creates it on
+    first connect, the same writer-owns-schema pattern as `_corpus.TurnStore`,
+    against the same database file."""
+
+    def __init__(self, db_path: Path):
+        self._db_path = Path(db_path)
+        self._schema_ready = False
+
+    def _connect(self) -> sqlite3.Connection:
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+        conn = sqlite3.connect(self._db_path)
+        conn.row_factory = sqlite3.Row
+        if not self._schema_ready:
+            conn.execute("PRAGMA journal_mode=WAL")
+            conn.executescript(_SCHEMA)
+            conn.commit()
+            self._schema_ready = True
+        return conn
+
+    @property
+    def db_path(self) -> Path:
+        return self._db_path
+
+    def check_writable(self) -> None:
+        """Create the schema and probe write access with BEGIN IMMEDIATE.
+        Raises sqlite3.OperationalError or OSError when the database is
+        unwritable or locked, so a --collect run can fail loudly before any
+        task runs instead of dropping attempt rows mid-run."""
+        conn = self._connect()
+        try:
+            conn.execute("BEGIN IMMEDIATE")
+            conn.execute("ROLLBACK")
+        finally:
+            conn.close()
+
+    def insert_run(
+        self,
+        run_id: str,
+        repo_path: str,
+        spec: str,
+        tasks_file: str,
+        model: str,
+        endpoint: str,
+        temperature: float,
+        max_retries: int,
+        k: int,
+    ) -> None:
+        conn = self._connect()
+        try:
+            conn.execute(
+                "INSERT INTO collect_runs (run_id, repo_path, spec, tasks_file, model, "
+                "endpoint, temperature, max_retries, k, started_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (run_id, repo_path, spec, tasks_file, model, endpoint,
+                 temperature, max_retries, k, _now_iso()),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def finish_run(self, run_id: str) -> None:
+        conn = self._connect()
+        try:
+            conn.execute(
+                "UPDATE collect_runs SET finished_at = ? WHERE run_id = ?",
+                (_now_iso(), run_id),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def insert_attempt(
+        self,
+        run_id: str,
+        task_id: str,
+        sample_index: int,
+        attempt_index: int,
+        prompt_messages_json: str,
+        reply: str,
+        actions_json: str,
+        passed: int,
+        error_detail: Optional[str] = None,
+        guard_violation: Optional[str] = None,
+        input_tokens: Optional[int] = None,
+        output_tokens: Optional[int] = None,
+    ) -> int:
+        conn = self._connect()
+        try:
+            cur = conn.execute(
+                "INSERT INTO collect_attempts (run_id, task_id, sample_index, "
+                "attempt_index, prompt_messages_json, reply, actions_json, passed, "
+                "error_detail, guard_violation, input_tokens, output_tokens, created_at) "
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+                (run_id, task_id, sample_index, attempt_index, prompt_messages_json,
+                 reply, actions_json, passed, error_detail, guard_violation,
+                 input_tokens, output_tokens, _now_iso()),
+            )
+            conn.commit()
+            return cur.lastrowid
+        finally:
+            conn.close()
+
+    def insert_sft_placeholder(
+        self, run_id: str, task_id: str, prompt_messages_json: str
+    ) -> None:
+        """Queue a repair placeholder for a task no chain solved. INSERT OR
+        IGNORE plus UNIQUE(run_id, task_id) keeps it one row per task per run."""
+        conn = self._connect()
+        try:
+            conn.execute(
+                "INSERT OR IGNORE INTO sft_queue "
+                "(run_id, task_id, prompt_messages_json, created_at) "
+                "VALUES (?, ?, ?, ?)",
+                (run_id, task_id, prompt_messages_json, _now_iso()),
+            )
+            conn.commit()
+        finally:
+            conn.close()
+
+
+def extract_pairs(store: CollectStore, run_id: Optional[str] = None) -> dict:
+    """Materialize eligible DPO preference pairs from collected attempts.
+
+    Pass/fail pairs join a passing attempt (chosen) with a failing attempt
+    (rejected) of the same task on byte-identical prompt_messages_json, both
+    free of the judge's env_failure flag, chosen_source 'local'. Green-vs-green
+    pairs exist only where the judge has ranked both passing attempts (lower
+    judge_rank wins), chosen_source 'judge-ranked'. sft_queue is never read.
+    Idempotent: UNIQUE(chosen_attempt_id, rejected_attempt_id) plus INSERT OR
+    IGNORE makes repeated runs, with or without a run filter, no-ops.
+
+    Returns {"created_local": n, "created_judge": n, "existing": n}."""
+    run_filter = ""
+    args: tuple = ()
+    if run_id is not None:
+        run_filter = " AND a.run_id = ? AND b.run_id = ?"
+        args = (run_id, run_id)
+
+    local_sql = (
+        "SELECT a.attempt_id, b.attempt_id, a.task_id "
+        "FROM collect_attempts a JOIN collect_attempts b "
+        "ON a.task_id = b.task_id "
+        "AND a.prompt_messages_json = b.prompt_messages_json "
+        "WHERE a.passed = 1 AND b.passed = 0 "
+        "AND a.env_failure = 0 AND b.env_failure = 0" + run_filter
+    )
+    judge_sql = (
+        "SELECT a.attempt_id, b.attempt_id, a.task_id "
+        "FROM collect_attempts a JOIN collect_attempts b "
+        "ON a.task_id = b.task_id "
+        "AND a.prompt_messages_json = b.prompt_messages_json "
+        "WHERE a.passed = 1 AND b.passed = 1 "
+        "AND a.judge_rank IS NOT NULL AND b.judge_rank IS NOT NULL "
+        "AND a.judge_rank < b.judge_rank "
+        "AND a.env_failure = 0 AND b.env_failure = 0" + run_filter
+    )
+
+    counts = {"created_local": 0, "created_judge": 0, "existing": 0}
+    conn = store._connect()
+    try:
+        for sql, source, key in (
+            (local_sql, "local", "created_local"),
+            (judge_sql, "judge-ranked", "created_judge"),
+        ):
+            for chosen_id, rejected_id, task_id in conn.execute(sql, args).fetchall():
+                cur = conn.execute(
+                    "INSERT OR IGNORE INTO dpo_pairs (chosen_attempt_id, "
+                    "rejected_attempt_id, task_id, chosen_source, status, created_at) "
+                    "VALUES (?, ?, ?, ?, 'candidate', ?)",
+                    (chosen_id, rejected_id, task_id, source, _now_iso()),
+                )
+                if cur.rowcount:
+                    counts[key] += 1
+                else:
+                    counts["existing"] += 1
+        conn.commit()
+    finally:
+        conn.close()
+    return counts
+
+
+def export_pairs(
+    store: CollectStore, out_path: Path, statuses: Optional[list] = None
+) -> int:
+    """Write pairs as dpo-jsonl (contracts/export-format.md): one JSON object
+    per line with exactly the keys prompt (the pair's shared message list),
+    chosen, and rejected. Returns the number of lines written; no matching
+    pairs writes an empty file."""
+    sql = (
+        "SELECT c.prompt_messages_json AS prompt, c.reply AS chosen, "
+        "r.reply AS rejected "
+        "FROM dpo_pairs p "
+        "JOIN collect_attempts c ON c.attempt_id = p.chosen_attempt_id "
+        "JOIN collect_attempts r ON r.attempt_id = p.rejected_attempt_id"
+    )
+    args: tuple = ()
+    if statuses:
+        sql += f" WHERE p.status IN ({', '.join('?' * len(statuses))})"
+        args = tuple(statuses)
+    sql += " ORDER BY p.pair_id"
+
+    conn = store._connect()
+    try:
+        rows = conn.execute(sql, args).fetchall()
+    finally:
+        conn.close()
+
+    out_path = Path(out_path)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with out_path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(
+                {
+                    "prompt": json.loads(row["prompt"]),
+                    "chosen": row["chosen"],
+                    "rejected": row["rejected"],
+                },
+                ensure_ascii=False,
+            ) + "\n")
+    return len(rows)
+
+
+def stats(store: CollectStore, run_id: Optional[str] = None) -> list:
+    """Per-run collection health: attempt count, per-task pass rates with the
+    20 to 80 percent band flag, pair yield (pairs whose chosen attempt belongs
+    to the run), and repair-queue depth (needs_solution rows for the run)."""
+    conn = store._connect()
+    try:
+        if run_id is not None:
+            runs = conn.execute(
+                "SELECT run_id, started_at FROM collect_runs WHERE run_id = ? "
+                "ORDER BY started_at", (run_id,),
+            ).fetchall()
+        else:
+            runs = conn.execute(
+                "SELECT run_id, started_at FROM collect_runs ORDER BY started_at"
+            ).fetchall()
+
+        out = []
+        for run in runs:
+            rid = run["run_id"]
+            task_rows = conn.execute(
+                "SELECT task_id, COUNT(*) AS attempts, SUM(passed) AS passes "
+                "FROM collect_attempts WHERE run_id = ? GROUP BY task_id "
+                "ORDER BY task_id", (rid,),
+            ).fetchall()
+            tasks = []
+            for t in task_rows:
+                rate = (t["passes"] or 0) / t["attempts"]
+                tasks.append({
+                    "task_id": t["task_id"],
+                    "attempts": t["attempts"],
+                    "passes": t["passes"] or 0,
+                    "pass_rate": rate,
+                    "in_band": 0.2 <= rate <= 0.8,
+                })
+            pair_yield = conn.execute(
+                "SELECT COUNT(*) FROM dpo_pairs p JOIN collect_attempts a "
+                "ON a.attempt_id = p.chosen_attempt_id WHERE a.run_id = ?", (rid,),
+            ).fetchone()[0]
+            repair_depth = conn.execute(
+                "SELECT COUNT(*) FROM sft_queue WHERE run_id = ? "
+                "AND status = 'needs_solution'", (rid,),
+            ).fetchone()[0]
+            out.append({
+                "run_id": rid,
+                "started_at": run["started_at"],
+                "attempts": sum(t["attempts"] for t in tasks),
+                "tasks": tasks,
+                "pair_yield": pair_yield,
+                "repair_depth": repair_depth,
+            })
+        return out
+    finally:
+        conn.close()