py-harness-cli 0.3.0__py3-none-any.whl

harness/observe/eval_tasks.py

@@ -0,0 +1,399 @@
+"""Held-out execution tasks. None of these prompts are in the 45 train pairs."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Task:
+    id: str
+    prompt: str
+    reference: str
+    argv: tuple[str, ...] = ()
+    stdin: str = ""
+    expect_stdout: str = ""
+    files: tuple[tuple[str, str], ...] = ()
+    timeout: float = 8.0
+
+
+RUN_PREFIX = (
+    "Reply with one fenced ```python block only. "
+    "Call main() from `if __name__ == '__main__'` so running the file prints. "
+    "Stdlib only. Read extra args from sys.argv. "
+    "Print exactly what is asked — no extra text.\n\n"
+)
+
+REPAIR_PREFIX = (
+    "The script failed when I ran it. Fix it.\n"
+    "Reply with one complete fenced python block.\n"
+)
+
+
+def all_tasks() -> tuple[Task, ...]:
+    return _TASKS
+
+
+_TASKS: tuple[Task, ...] = (
+    Task(
+        id="weekday",
+        prompt="Print the English weekday name for a YYYY-MM-DD date in sys.argv[1].",
+        argv=("2026-08-29",),
+        expect_stdout="Saturday\n",
+        reference="""
+import sys
+from datetime import date
+
+def main() -> None:
+    y, m, d = sys.argv[1].split("-")
+    print(date(int(y), int(m), int(d)).strftime("%A"))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="count-ext",
+        prompt=(
+            "Count files in directory sys.argv[1] whose names end with sys.argv[2] "
+            "(example: .md). Do not recurse. Print the integer count."
+        ),
+        argv=(".", ".md"),
+        files=(("a.md", "x"), ("b.md", "y"), ("c.txt", "z"), ("notes.md.bak", "q")),
+        expect_stdout="2\n",
+        reference="""
+import sys
+from pathlib import Path
+
+def main() -> None:
+    root = Path(sys.argv[1])
+    suffix = sys.argv[2]
+    print(sum(1 for p in root.iterdir() if p.is_file() and p.name.endswith(suffix)))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="fizzbuzz",
+        prompt=(
+            "Print FizzBuzz for 1..N inclusive, N from sys.argv[1]. "
+            "Fizz on multiples of 3, Buzz on 5, FizzBuzz on both. One value per line."
+        ),
+        argv=("15",),
+        expect_stdout=(
+            "1\n2\nFizz\n4\nBuzz\nFizz\n7\n8\nFizz\nBuzz\n11\nFizz\n13\n14\nFizzBuzz\n"
+        ),
+        reference="""
+import sys
+
+def main() -> None:
+    n = int(sys.argv[1])
+    for i in range(1, n + 1):
+        out = ""
+        if i % 3 == 0:
+            out += "Fizz"
+        if i % 5 == 0:
+            out += "Buzz"
+        print(out or i)
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="clamp",
+        prompt="Print clamp(x, lo, hi) for three integers sys.argv[1:4] (x, lo, hi).",
+        argv=("12", "0", "10"),
+        expect_stdout="10\n",
+        reference="""
+import sys
+
+def main() -> None:
+    x, lo, hi = (int(a) for a in sys.argv[1:4])
+    print(min(hi, max(lo, x)))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="slugify",
+        prompt=(
+            "Slugify sys.argv[1]: lowercase, replace each run of non-ascii-letters/"
+            "digits with one hyphen, strip leading/trailing hyphens. Print the slug."
+        ),
+        argv=("Hello, World!",),
+        expect_stdout="hello-world\n",
+        reference="""
+import re
+import sys
+
+def main() -> None:
+    text = re.sub(r"[^a-z0-9]+", "-", sys.argv[1].lower()).strip("-")
+    print(text)
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="median",
+        prompt="Print the median of the integers in sys.argv[1:]. For even n, print the lower middle (integer).",
+        argv=("1", "3", "2", "9", "5"),
+        expect_stdout="3\n",
+        reference="""
+import sys
+
+def main() -> None:
+    nums = sorted(int(a) for a in sys.argv[1:])
+    print(nums[(len(nums) - 1) // 2])
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="hhmmss",
+        prompt="Convert a non-negative integer number of seconds (sys.argv[1]) to HH:MM:SS with zero-padded fields.",
+        argv=("3661",),
+        expect_stdout="01:01:01\n",
+        reference="""
+import sys
+
+def main() -> None:
+    total = int(sys.argv[1])
+    h, rem = divmod(total, 3600)
+    m, s = divmod(rem, 60)
+    print(f"{h:02d}:{m:02d}:{s:02d}")
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="rotate",
+        prompt=(
+            "Left-rotate the words in sys.argv[2:] by k=int(sys.argv[1]) positions. "
+            "Print the words joined by a single space."
+        ),
+        argv=("2", "a", "b", "c", "d"),
+        expect_stdout="c d a b\n",
+        reference="""
+import sys
+
+def main() -> None:
+    k = int(sys.argv[1])
+    words = sys.argv[2:]
+    k %= len(words)
+    print(" ".join(words[k:] + words[:k]))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="unique-order",
+        prompt="Print the words in sys.argv[1:] with duplicates removed, first occurrence kept, space-separated.",
+        argv=("a", "b", "a", "c", "b"),
+        expect_stdout="a b c\n",
+        reference="""
+import sys
+
+def main() -> None:
+    seen: list[str] = []
+    for word in sys.argv[1:]:
+        if word not in seen:
+            seen.append(word)
+    print(" ".join(seen))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="palindrome",
+        prompt=(
+            "Print yes or no: is sys.argv[1] a palindrome if you lowercase it and "
+            "drop every character that is not a-z or 0-9?"
+        ),
+        argv=("RaceCar",),
+        expect_stdout="yes\n",
+        reference="""
+import sys
+
+def main() -> None:
+    chars = [c for c in sys.argv[1].lower() if c.isalnum() and c.isascii()]
+    print("yes" if chars == chars[::-1] else "no")
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="fib",
+        prompt="Print F(n) where F(0)=0, F(1)=1, n=int(sys.argv[1]).",
+        argv=("10",),
+        expect_stdout="55\n",
+        reference="""
+import sys
+
+def main() -> None:
+    n = int(sys.argv[1])
+    a, b = 0, 1
+    for _ in range(n):
+        a, b = b, a + b
+    print(a)
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="sum-even",
+        prompt="Print the sum of the even integers in sys.argv[1:].",
+        argv=("1", "2", "3", "4", "5", "6"),
+        expect_stdout="12\n",
+        reference="""
+import sys
+
+def main() -> None:
+    print(sum(int(a) for a in sys.argv[1:] if int(a) % 2 == 0))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="csv-col",
+        prompt=(
+            "Read CSV from stdin (no quotes). Print 0-based column sys.argv[1], "
+            "one cell per line."
+        ),
+        argv=("1",),
+        stdin="a,1\nb,2\n",
+        expect_stdout="1\n2\n",
+        reference="""
+import sys
+
+def main() -> None:
+    idx = int(sys.argv[1])
+    for line in sys.stdin:
+        line = line.rstrip("\\n")
+        if not line:
+            continue
+        print(line.split(",")[idx])
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="indent4",
+        prompt="Read stdin and print it again with four spaces prepended to every line. Keep the last newline.",
+        stdin="foo\nbar\n",
+        expect_stdout="    foo\n    bar\n",
+        reference="""
+import sys
+
+def main() -> None:
+    for line in sys.stdin:
+        if line.endswith("\\n"):
+            print("    " + line[:-1])
+        else:
+            print("    " + line, end="")
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="anagram",
+        prompt=(
+            "Print yes or no: are sys.argv[1] and sys.argv[2] anagrams after "
+            "lowercasing and dropping spaces?"
+        ),
+        argv=("listen", "silent"),
+        expect_stdout="yes\n",
+        reference="""
+import sys
+
+def norm(text: str) -> list[str]:
+    return sorted(c for c in text.lower() if c != " ")
+
+def main() -> None:
+    print("yes" if norm(sys.argv[1]) == norm(sys.argv[2]) else "no")
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="wrap",
+        prompt=(
+            "Word-wrap stdin to width N=int(sys.argv[1]). Split on spaces. "
+            "If a word is longer than N, put it on its own line. Print one wrapped line per row."
+        ),
+        argv=("5",),
+        stdin="hello world\n",
+        expect_stdout="hello\nworld\n",
+        reference="""
+import sys
+
+def main() -> None:
+    width = int(sys.argv[1])
+    words = sys.stdin.read().split()
+    line: list[str] = []
+    size = 0
+    for word in words:
+        extra = len(word) if not line else len(word) + 1
+        if line and size + extra > width:
+            print(" ".join(line))
+            line = [word]
+            size = len(word)
+        else:
+            line.append(word)
+            size += extra
+    if line:
+        print(" ".join(line))
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="iso-date",
+        prompt="Print YYYY-MM-DD from an ISO-8601 timestamp in sys.argv[1] (may include time and timezone).",
+        argv=("2026-09-05T17:27:00",),
+        expect_stdout="2026-09-05\n",
+        reference="""
+import sys
+from datetime import datetime
+
+def main() -> None:
+    raw = sys.argv[1]
+    if raw.endswith("Z"):
+        raw = raw[:-1] + "+00:00"
+    print(datetime.fromisoformat(raw).date().isoformat())
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+    Task(
+        id="relpath",
+        prompt="Print path sys.argv[2] relative to directory sys.argv[1], using forward slashes.",
+        argv=("/Users/x/proj", "/Users/x/proj/src/a.py"),
+        expect_stdout="src/a.py\n",
+        reference="""
+import sys
+from pathlib import Path
+
+def main() -> None:
+    print(Path(sys.argv[2]).relative_to(sys.argv[1]).as_posix())
+
+if __name__ == "__main__":
+    main()
+""",
+    ),
+)

harness/observe/report_md.py

@@ -0,0 +1,102 @@
+"""Turn batch-review JSONL into markdown. No model. Not a second harness."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def load_rows(path: Path) -> list[dict]:
+    rows: list[dict] = []
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if line:
+            rows.append(json.loads(line))
+    return rows
+
+
+def _clean_review(text: str) -> str:
+    first = (text or "").strip()
+    if first.lower().startswith("no issue"):
+        return "no issues"
+    return first.split("\n", 1)[0].strip() or "(empty)"
+
+
+def render_markdown(
+    rows: list[dict],
+    *,
+    project: str,
+    extra: dict | None = None,
+) -> str:
+    extra = extra or {}
+    clean = 0
+    flagged = 0
+    applied = 0
+    lines = [
+        "# py-harness batch review",
+        "",
+        f"Project: `{project}`",
+        "",
+        "This file is **formatted from JSONL**. The 0.5B model reviewed one "
+        "small `.py` file at a time. `PythonVibeGuard` only gates the draft "
+        "(empty / keys / `curl|sh`). It does not judge review quality. "
+        "There is no separate markdown harness.",
+        "",
+    ]
+    if extra:
+        bits = [f"{k}={v}" for k, v in extra.items()]
+        lines.append("Run: " + ", ".join(bits))
+        lines.append("")
+    body: list[str] = ["| File | Bytes | Review | Applied |", "| --- | ---: | --- | --- |"]
+    extras: list[str] = []
+    for row in rows:
+        review = str(row.get("review") or "")
+        label = _clean_review(review)
+        if label == "no issues":
+            clean += 1
+        else:
+            flagged += 1
+        if row.get("applied"):
+            applied += 1
+        rel = row.get("file", "")
+        body.append(
+            f"| `{rel}` | {row.get('bytes', '')} | {label} | "
+            f"{'yes' if row.get('applied') else 'no'} |"
+        )
+        if "```" in review or (label == "no issues" and len(review) > 40):
+            extras.append(
+                f"### `{rel}`\n\nThe model said no issues but also emitted extra text "
+                f"(ignored for apply).\n"
+            )
+    lines += [
+        f"- Files: **{len(rows)}**",
+        f"- Said no issues: **{clean}**",
+        f"- Other review text: **{flagged}**",
+        f"- Applied rewrites: **{applied}**",
+        "",
+        "## Files",
+        "",
+    ]
+    lines.extend(body)
+    if extras:
+        lines += ["", "## Extra model text (not applied)", ""]
+        lines.extend(extras)
+    lines += [
+        "",
+        "## How to read this",
+        "",
+        "A hundred `no issues` on 200–350 byte `__init__.py` / constants / "
+        "verifiers is the expected 0.5B outcome: the files are tiny re-exports. "
+        "It is not a sign OpenSRE was audited. For a real review, open a module "
+        "over ~1 KB with `--file` or raise `--min-bytes`.",
+        "",
+    ]
+    return "\n".join(lines) + "\n"
+
+
+def write_report(jsonl: Path, dest: Path, *, project: str, extra: dict | None = None) -> Path:
+    dest.write_text(
+        render_markdown(load_rows(jsonl), project=project, extra=extra),
+        encoding="utf-8",
+    )
+    return dest

harness/observe/trace_record.py

@@ -0,0 +1,79 @@
+"""Append redacted agent turns. Never store raw keys or home paths."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+
+from harness.secrets import secret_in
+
+# The four shapes are shared with the guard and the outbound check, so
+# a shape learned once is known everywhere. These two are extra to this
+# file: a trace is written to disk and kept, so it redacts wider than a
+# refusal needs to.
+_ALSO_REDACT = re.compile(r"(HF_TOKEN=|-----BEGIN )")
+_HOME = re.compile(r"/(Users|home)/[^/\s]+")
+_URL_HOST = re.compile(r"\b([a-z][a-z0-9+.-]*://)(?:[^/@\s]+@)?([^/\s]+)", re.IGNORECASE)
+# A hostname with no scheme in front of it. `.home` and `.local` are
+# deliberately absent from the list without a port: `Path.home()` is
+# standard Python and `settings.local.json` is a real file name, and a
+# trace is training data, so mangling either teaches the model a
+# mistake. With a port there is no ambiguity — `box.local:8443` is a
+# host and `Path.home()` never carries one.
+_BARE_HOST = re.compile(
+    r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*"
+    r"(?:\.(?:lan|internal|corp)(?::\d+)?|\.(?:local|home):\d+)\b"
+)
+
+
+def redact(text: str) -> str:
+    if secret_in(text) or _ALSO_REDACT.search(text):
+        return "[redacted]"
+    text = _HOME.sub(lambda match: f"/{match.group(1)}/you", text)
+    text = _URL_HOST.sub(lambda match: f"{match.group(1)}[host]", text)
+    return _BARE_HOST.sub("[host]", text)
+
+
+# Where a run writes its turns when nobody says otherwise. Inside the
+# project, because the traces are about that project's code, and hidden
+# because nobody wants it in a listing.
+TRACE_DIR = ".python-vibe"
+TRACE_FILE = "traces.jsonl"
+
+
+def default_trace_path(project: Path) -> Path:
+    """Where turns go when no --record is given."""
+    return Path(project) / TRACE_DIR / TRACE_FILE
+
+
+def render_last(project: Path, *, limit: int = 8) -> str:
+    """The most recent recorded turns, or a line saying there are none."""
+    path = default_trace_path(project)
+    if not path.is_file():
+        return f"no traces at {path}"
+    rows = [
+        json.loads(line)
+        for line in path.read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    if not rows:
+        return f"no traces at {path}"
+    lines = [f"{len(rows)} turns in {path}", ""]
+    for row in rows[-limit:]:
+        action = (row.get("action") or "-").strip() or "-"
+        text = (row.get("assistant") or row.get("user") or "").strip()
+        text = " ".join(text.split())
+        if len(text) > 80:
+            text = text[:77] + "..."
+        lines.append(f"{action}: {text}" if text else action)
+    return "\n".join(lines)
+
+
+def append_turn(path: Path, row: dict[str, object]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    clean = {key: redact(str(value)) for key, value in row.items()}
+    if any(v == "[redacted]" for v in clean.values()):
+        return
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(clean, ensure_ascii=False) + "\n")

harness/openai_api.py

@@ -0,0 +1,81 @@
+"""The shapes an OpenAI-compatible client sends and expects.
+
+Request parsing and reply payloads for `server.py`, so an editor can
+talk to this project using the API it already speaks.
+
+This is the request and reply shape, not the model. It knows what a chat request
+looks like and nothing about weights, which is why it sits beside the
+server rather than inside `model/`: that package is only the code that
+talks to a model, and the CLI and the server do not reach into it.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from finetune.everyday import DEFAULT_EVERYDAY_OLLAMA, is_tiny_model
+
+
+def parse_chat_body(raw: bytes) -> dict[str, Any]:
+    body = json.loads(raw or b"{}")
+    if not isinstance(body, dict):
+        raise ValueError("json object required")
+    messages = body.get("messages")
+    if not isinstance(messages, list) or not messages:
+        raise ValueError("messages required")
+    model = str(body.get("model") or DEFAULT_EVERYDAY_OLLAMA)
+    return {"model": model, "messages": messages, "stream": bool(body.get("stream"))}
+
+
+def warn_tiny(model: str) -> str | None:
+    if is_tiny_model(model):
+        return (
+            f"{model} is the 0.5B sidecar. Everyday laptop use should be "
+            f"{DEFAULT_EVERYDAY_OLLAMA} (or qwen2.5-coder:7b / 14b)."
+        )
+    return None
+
+
+def ollama_openai_url(host: str) -> str:
+    return host.rstrip("/") + "/v1/chat/completions"
+
+
+def models_payload(model: str) -> dict[str, Any]:
+    return {
+        "object": "list",
+        "data": [{"id": model, "object": "model", "owned_by": "ollama"}],
+    }
+
+
+def last_user_text(messages: list[Any]) -> str:
+    """The last user turn, as plain text. Editors send a string or parts."""
+    for message in reversed(messages):
+        if not isinstance(message, dict) or message.get("role") != "user":
+            continue
+        content = message.get("content")
+        if isinstance(content, str):
+            return content.strip()
+        if isinstance(content, list):
+            parts = [
+                str(item.get("text") or "")
+                for item in content
+                if isinstance(item, dict) and item.get("type") == "text"
+            ]
+            return "\n".join(part for part in parts if part).strip()
+    return ""
+
+
+def chat_completion_payload(content: str, model: str) -> dict[str, Any]:
+    return {
+        "id": "py-harness",
+        "object": "chat.completion",
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": content},
+                "finish_reason": "stop",
+            }
+        ],
+        "model": model,
+    }