py-harness-cli 0.3.0__py3-none-any.whl

harness/model/ollama_generate.py

@@ -0,0 +1,60 @@
+"""Call a local/cloud Ollama model. Weights stay in Ollama; this process is tiny."""
+
+from __future__ import annotations
+
+from harness.model.chat_backend import ChatBackend
+
+# How much the model is told it may read. Ollama's own default is 4096,
+# small enough that a twenty-step run silently lost its opening.
+CONTEXT_TOKENS = 8192
+
+import json
+import os
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+
+
+class OllamaGenerate(ChatBackend):
+    def __init__(
+        self,
+        model: str,
+        system: str,
+        host: str | None = None,
+        timeout: float = 180,
+    ) -> None:
+        super().__init__(model, system, timeout=timeout)
+        self.host = (host or os.environ.get("OLLAMA_HOST") or "http://127.0.0.1:11434").rstrip(
+            "/"
+        )
+
+    num_ctx = CONTEXT_TOKENS
+
+    def url(self) -> str:
+        return f"{self.host}/api/chat"
+
+    def body(self, messages: list[dict[str, str]]) -> dict[str, object]:
+        return {
+            "model": self.model,
+            "stream": False,
+            "messages": messages,
+            # Say the size rather than take the server's default. That
+            # default is 4096 for weights that accept 131072, and a run
+            # crossing it had its oldest messages dropped by the server
+            # without saying so. The harness decides what to forget.
+            "options": {"num_ctx": self.num_ctx},
+        }
+
+    def reply_from(self, payload: dict[str, object]) -> str:
+        message = payload.get("message") or {}
+        return str(message.get("content") or "")  # type: ignore[union-attr]
+
+    def unreachable(self, exc: Exception) -> str:
+        return f"ollama {self.host} unreachable: {exc}"
+
+    def healthy(self) -> bool:
+        try:
+            with urllib.request.urlopen(f"{self.host}/api/tags", timeout=2) as resp:
+                return resp.status == 200
+        except (urllib.error.URLError, TimeoutError, OSError):
+            return False

harness/model/openai_generate.py

@@ -0,0 +1,156 @@
+"""Call an OpenAI-compatible chat endpoint. Weights stay on that host.
+
+The laptop harness does not change. This is how a 14B–70B that timed out
+locally is reached: Hugging Face Inference, a rented vLLM box, or any
+other /v1/chat/completions server. Tokens come from the environment.
+They are never written into a trace, a test, or an error string.
+"""
+
+from __future__ import annotations
+
+from harness.model.chat_backend import ChatBackend
+from harness.model.outbound import refuse_to_send, what_was_sent
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+
+HF_ROUTER = "https://router.huggingface.co/v1"
+
+
+def chat_url(base: str) -> str:
+    """POST target. Accepts a host, a /v1 root, or the full chat path."""
+    base = base.rstrip("/")
+    if base.endswith("/chat/completions"):
+        return base
+    return base + "/chat/completions"
+
+
+def resolve_openai_endpoint(
+    *,
+    base_url: str | None = None,
+    api_key: str | None = None,
+) -> tuple[str, str]:
+    """(base_url, api_key). Raises ValueError with no secret in the text."""
+    base = (
+        base_url
+        or os.environ.get("PYTHON_VIBE_BASE_URL")
+        or os.environ.get("OPENAI_BASE_URL")
+        or ""
+    ).strip()
+    key = (
+        api_key
+        or os.environ.get("PYTHON_VIBE_API_KEY")
+        or os.environ.get("HF_TOKEN")
+        or os.environ.get("HUGGING_FACE_HUB_TOKEN")
+        or os.environ.get("OPENAI_API_KEY")
+        or ""
+    ).strip()
+    if not base and key:
+        base = HF_ROUTER
+    if not base:
+        raise ValueError(
+            "PYTHON_VIBE_BASE_URL is required for --engine openai "
+            "(or set HF_TOKEN to use the Hugging Face router)"
+        )
+    return base.rstrip("/"), key
+
+
+def looks_like_an_ollama_tag(model: str) -> bool:
+    """`llama3.1:8b` names a local pull, not a model on someone's server.
+
+    The engine falls back to the everyday Ollama name when --model is not
+    given, so the first remote run sends a tag no remote host knows and
+    comes back with a bare 400. Saying so is cheaper than guessing.
+    """
+    return ":" in model and "/" not in model
+
+
+class OpenAIGenerate(ChatBackend):
+    def __init__(
+        self,
+        model: str,
+        system: str,
+        *,
+        max_tokens: int = 700,
+        base_url: str | None = None,
+        api_key: str | None = None,
+        timeout: float | None = None,
+    ) -> None:
+        self.max_tokens = max_tokens
+        self.base_url, self.api_key = resolve_openai_endpoint(
+            base_url=base_url, api_key=api_key
+        )
+        if timeout is None:
+            raw = os.environ.get("PYTHON_VIBE_TIMEOUT", "180")
+            try:
+                timeout = float(raw)
+            except ValueError:
+                timeout = 180.0
+        self.last_sent = ""
+        self._said = False
+        super().__init__(model, system, timeout=timeout)
+
+    def _hint(self, code: int) -> str:
+        """Say the likely cause. Never quote the key or the headers."""
+        if code in {401, 403}:
+            return (
+                ". Check the token in PYTHON_VIBE_API_KEY or HF_TOKEN, "
+                "and that it may use this host"
+            )
+        if code in {400, 404} and looks_like_an_ollama_tag(self.model):
+            return (
+                f". `{self.model}` is an Ollama tag; a remote host wants its "
+                "own id, such as meta-llama/Llama-3.1-8B-Instruct. "
+                "Pass --model"
+            )
+        return ""
+
+    def url(self) -> str:
+        return chat_url(self.base_url)
+
+    def body(self, messages: list[dict[str, str]]) -> dict[str, object]:
+        return {
+            "model": self.model,
+            "stream": False,
+            "max_tokens": self.max_tokens,
+            "messages": messages,
+        }
+
+    def headers(self) -> dict[str, str]:
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def before_send(self, messages: list[dict[str, str]]) -> None:
+        """Nothing goes to somebody else's host unread.
+
+        The summary is said once, on the first send. A run takes up to
+        twenty turns and a line per turn would be noise, but a person
+        who wants to know whether their file left the laptop should not
+        have to go looking.
+        """
+        blocked = refuse_to_send(messages, self.base_url)
+        if blocked:
+            raise RuntimeError(blocked)
+        self.last_sent = what_was_sent(messages, self.base_url)
+        if self.last_sent and not self._said:
+            self._said = True
+            print(self.last_sent, file=sys.stderr)
+
+    def reply_from(self, payload: dict[str, object]) -> str:
+        choices = payload.get("choices") or []
+        if not choices:
+            return ""
+        message = choices[0].get("message") or {}  # type: ignore[index]
+        return str(message.get("content") or "")
+
+    def unreachable(self, exc: Exception) -> str:
+        return "remote model unreachable"
+
+    def refused(self, code: int) -> str:
+        return f"remote model HTTP {code}{self._hint(code)}"

harness/model/outbound.py

@@ -0,0 +1,83 @@
+"""What may leave this machine, and what was sent when it did.
+
+The default engine talks to Ollama on this laptop, so nothing leaves and
+none of this runs. `--engine openai` posts to a host somebody else runs,
+and until now nothing looked at the bytes on the way out. The guard read
+drafts coming back and never the prompt going out.
+
+Two questions are answered here. May this go: not if it carries a secret
+shape, and not if it is far larger than a task should need. And what
+went: a sentence a person can check against what they expected, without
+the prompt itself being printed back at them.
+"""
+
+from __future__ import annotations
+
+import os
+from urllib.parse import urlparse
+
+from harness.secrets import secret_in
+
+# Hosts that are this machine. Sending to one of these is not sending.
+LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "0.0.0.0", ""})
+
+# A single large file is ordinary. A whole repository is a mistake, and
+# the point of a cap is to catch the mistake, not to ration the work.
+DEFAULT_MAX_CHARS = 200_000
+
+
+def leaves_this_machine(base_url: str) -> bool:
+    """True when this host is somebody else's."""
+    host = urlparse(base_url if "//" in base_url else f"//{base_url}").hostname
+    return (host or "") not in LOCAL_HOSTS
+
+
+def max_chars() -> int:
+    """The cap, which a caller who means it can raise."""
+    try:
+        return int(os.environ.get("PYTHON_VIBE_MAX_SEND", DEFAULT_MAX_CHARS))
+    except ValueError:
+        return DEFAULT_MAX_CHARS
+
+
+def _joined(messages: list[dict[str, str]]) -> str:
+    return "\n".join(str(m.get("content") or "") for m in messages)
+
+
+def refuse_to_send(messages: list[dict[str, str]], base_url: str) -> str:
+    """Why these messages must not go to that host. "" when they may."""
+    if not leaves_this_machine(base_url):
+        return ""
+    text = _joined(messages)
+    found = secret_in(text)
+    if found:
+        return (
+            f"Refusing to send: the prompt contains {found}. It would go "
+            "to a host this machine does not control. Take it out of the "
+            "file, or run without --engine openai."
+        )
+    size = len(text)
+    cap = max_chars()
+    if size > cap:
+        return (
+            f"Refusing to send {size} characters to a remote host; the "
+            f"limit is {cap}. Narrow the task with --scope, or raise "
+            "PYTHON_VIBE_MAX_SEND if that is really the intent."
+        )
+    return ""
+
+
+def what_was_sent(messages: list[dict[str, str]], base_url: str) -> str:
+    """One line naming where it went and how much of it. "" when local.
+
+    Deliberately a size and a destination rather than the prompt. A
+    person who wants to know whether their file went can read this; a
+    person who wants the prompt back has it already.
+    """
+    if not leaves_this_machine(base_url):
+        return ""
+    host = urlparse(base_url if "//" in base_url else f"//{base_url}").hostname
+    text = _joined(messages)
+    return (
+        f"sent {len(text)} characters in {len(messages)} message(s) to {host}"
+    )

harness/model/route.py

@@ -0,0 +1,90 @@
+"""Which local weight a task should use. Deterministic. No LLM router.
+
+Papers call this routing (pick one model first) versus cascading (cheap
+model first, escalate if a judge fails). This harness already cascades on
+compiler oracles, same model. The router only names a *lane*. It never
+selects the 0.5B sidecar or the 30B that timed out on this laptop.
+"""
+
+from __future__ import annotations
+
+from harness.task import (
+    looks_like_add_feature,
+    looks_like_bugfix,
+    looks_like_design_loop,
+    looks_like_everyday_code,
+    looks_like_fix_smell,
+    looks_like_question,
+    looks_like_review_code,
+    looks_like_ship,
+    looks_like_write_tests,
+    task_paths,
+)
+
+EVERYDAY = "llama3.1:8b"
+TINY = "qwen2.5-coder:0.5b"
+FAST = "qwen2.5-coder:1.5b"
+CODER = "qwen2.5-coder:7b"
+HEAVY = "qwen3coder:latest"
+
+LANES = ("none", "read", "write", "structure")
+
+
+def model_lane(task: str) -> str:
+    """One of LANES. Ship and one-file reviews are not write jobs."""
+    if looks_like_ship(task):
+        return "none"
+    if looks_like_question(task):
+        return "read"
+    if looks_like_design_loop(task) and not task_paths(task):
+        return "structure"
+    if looks_like_review_code(task) and task_paths(task):
+        return "read"
+    if (
+        looks_like_add_feature(task)
+        or looks_like_everyday_code(task)
+        or looks_like_write_tests(task)
+        or looks_like_fix_smell(task)
+        or looks_like_bugfix(task)
+    ):
+        return "write"
+    if looks_like_design_loop(task):
+        return "structure"
+    return "write"
+
+
+def suggest_ollama(task: str) -> str:
+    """Ollama name to run, or empty when the harness needs no model."""
+    if model_lane(task) == "none":
+        return ""
+    return EVERYDAY
+
+
+def route_advice(task: str) -> str:
+    """Human-readable pick. Safe to print from `brief` / `route`."""
+    lane = model_lane(task)
+    model = suggest_ollama(task)
+    if lane == "none":
+        return (
+            "Lane: none. No model. Ship actions are limited git/gh. "
+            "Do not pull a larger weight for this."
+        )
+    extra = {
+        "read": (
+            f"Lane: read. Use {model}. "
+            "A chat 8B is enough for a typed question. "
+            "Do not use the 0.5B sidecar (misses Action:). "
+            f"{FAST} is on disk but unproven on this protocol."
+        ),
+        "write": (
+            f"Lane: write. Use {model}. "
+            f"Optional specialist when pulled: {CODER} via --model. "
+            f"Do not auto-switch to {HEAVY} (180s timeout on this laptop). "
+            "Oracles stay on the same model — do not load a second weight mid-run."
+        ),
+        "structure": (
+            f"Lane: structure. Use {model}. "
+            "The design scan is deterministic. A 30B does not replace it."
+        ),
+    }
+    return extra[lane]

harness/observe/__init__.py

@@ -0,0 +1,6 @@
+"""Record what a run did.
+
+Holds the redacted trace writer, the Markdown report writer, the
+offline checks used as the merge gate, and the held-out exact-stdout
+scorer.
+"""

harness/observe/eval_gate.py

@@ -0,0 +1,80 @@
+"""Offline everyday gate. Live Ollama comparison is scripts/measure/eval_everyday.py --live."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from harness.act.parse import parse_turn
+from harness.paths import EVAL_DIR, REPO_ROOT
+from harness.act.code import apply_source, write_and_run
+
+ROOT = REPO_ROOT
+EVAL = EVAL_DIR
+
+
+def action_parse_rate(path: Path | None = None) -> tuple[int, int]:
+    drafts = path or EVAL / "action_drafts.jsonl"
+    ok = 0
+    total = 0
+    for line in drafts.read_text(encoding="utf-8").splitlines():
+        if not line.strip():
+            continue
+        row = json.loads(line)
+        total += 1
+        turn = parse_turn(row["draft"])
+        if turn and turn.action == row["action"]:
+            ok += 1
+    return ok, total
+
+
+def held_out_run_pass(gold_dir: Path | None = None) -> tuple[int, int]:
+    import tempfile
+
+    gold = gold_dir or EVAL / "gold"
+    ok = 0
+    tasks = list((EVAL / "held_out").glob("*.json")) if (EVAL / "held_out").is_dir() else []
+    if not tasks:
+        return 0, 0
+    for spec_path in tasks:
+        spec = json.loads(spec_path.read_text(encoding="utf-8"))
+        script = gold / spec["script"]
+        argv: list[str] = []
+        for item in spec.get("argv") or []:
+            cand = EVAL / str(item)
+            argv.append(str(cand) if cand.exists() else str(item))
+        with tempfile.TemporaryDirectory() as tmp:
+            dest = Path(tmp) / script.name
+            result = write_and_run(
+                script.read_text(encoding="utf-8"),
+                dest,
+                argv,
+            )
+        expected = spec["stdout"].strip()
+        if result.code == 0 and result.stdout.strip() == expected:
+            ok += 1
+    return ok, len(tasks)
+
+
+def bugfix_fixture_ready(project: Path | None = None) -> tuple[bool, str]:
+    root = project or EVAL / "fixtures" / "nameerror_pkg"
+    broken = root / "pkg" / "util_stats.py"
+    gold = EVAL / "gold" / "util_stats.py"
+    if not broken.is_file() or not gold.is_file():
+        return False, "missing fixture or gold"
+    if broken.stat().st_size < 1000:
+        return False, f"{broken} is under 1 KB"
+    original = broken.read_text(encoding="utf-8")
+    if "tota" not in original or "return tota" not in original:
+        return False, "fixture no longer has the NameError"
+    apply_source(broken, gold.read_text(encoding="utf-8"), original=original)
+    try:
+        text = broken.read_text(encoding="utf-8")
+        if "return total" not in text and "return sum(" not in text:
+            return False, "gold apply did not fix the name"
+    finally:
+        broken.write_text(original, encoding="utf-8")
+        bak = broken.with_suffix(broken.suffix + ".bak")
+        if bak.is_file():
+            bak.unlink()
+    return True, "ok"

harness/observe/eval_loop.py

@@ -0,0 +1,185 @@
+"""Score one draft against a held-out task. No model. No network."""
+
+from __future__ import annotations
+
+import subprocess
+import tempfile
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+
+from harness.act.code import RunResult, extract_python, write_and_run
+from harness.guard.fallbacks import PYTHON_VIBE_FALLBACK
+from harness.guard.python_vibe import PythonVibeGuard
+from harness.guard.run import complete
+from harness.guard.types import Outcome
+from harness.observe.eval_tasks import REPAIR_PREFIX, RUN_PREFIX, Task
+
+GenerateFn = Callable[[str], str]
+
+
+@dataclass(frozen=True)
+class Score:
+    task_id: str
+    passed: bool
+    repaired: bool
+    reason: str
+    stdout: str
+    stderr: str
+    exit_code: int
+    fallback: bool
+
+
+def stdout_matches(expect: str, got: str) -> bool:
+    return got.rstrip("\n") == expect.rstrip("\n")
+
+
+def _remember(generate: GenerateFn, prompt: str, draft: str) -> None:
+    memory = getattr(generate, "memory", None)
+    if memory is None:
+        return
+    memory.remember(prompt, draft)
+
+
+def _reset(generate: GenerateFn) -> None:
+    memory = getattr(generate, "memory", None)
+    if memory is not None:
+        memory.clear()
+        return
+    history = getattr(generate, "history", None)
+    if history is not None:
+        history.clear()
+
+
+def run_source(task: Task, source: str, dest: Path) -> tuple[bool, RunResult]:
+    work = dest.parent
+    for rel, content in task.files:
+        path = work / rel
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(content, encoding="utf-8")
+    try:
+        result = write_and_run(
+            source,
+            dest,
+            list(task.argv),
+            cwd=work,
+            timeout=task.timeout,
+            stdin=task.stdin or None,
+        )
+    except subprocess.TimeoutExpired as exc:
+        result = RunResult(
+            124,
+            (exc.stdout or "") if isinstance(exc.stdout, str) else "",
+            (exc.stderr or "") if isinstance(exc.stderr, str) else "timed out",
+        )
+    ok = result.code == 0 and stdout_matches(task.expect_stdout, result.stdout)
+    return ok, result
+
+
+def score_source(task: Task, source: str) -> Score:
+    if not source.strip():
+        return Score(task.id, False, False, "empty source", "", "", 1, False)
+    with tempfile.TemporaryDirectory(prefix=f"pv-eval-{task.id}-") as tmp:
+        dest = Path(tmp) / "task.py"
+        ok, result = run_source(task, source, dest)
+    return Score(
+        task.id,
+        ok,
+        False,
+        (
+            "pass"
+            if ok
+            else "timeout"
+            if result.code == 124
+            else "wrong output"
+            if result.code == 0
+            else "nonzero exit"
+        ),
+        result.stdout,
+        result.stderr,
+        result.code,
+        False,
+    )
+
+
+def _draft_source(outcome: Outcome) -> tuple[str | None, Score | None]:
+    if outcome.fallback or not outcome.output:
+        return None, Score(
+            "",
+            False,
+            False,
+            "guard fallback" if outcome.fallback else "empty draft",
+            "",
+            "",
+            1,
+            outcome.fallback,
+        )
+    source = extract_python(outcome.output)
+    if not source:
+        return None, Score("", False, False, "no python block", "", "", 1, False)
+    return source, None
+
+
+def score_generate(
+    task: Task,
+    generate: GenerateFn,
+    *,
+    repair: bool = False,
+    guard: PythonVibeGuard | None = None,
+) -> Score:
+    checker = guard or PythonVibeGuard()
+    prompt = RUN_PREFIX + task.prompt
+    try:
+        outcome = complete(generate, checker, PYTHON_VIBE_FALLBACK, prompt)
+    except (TimeoutError, RuntimeError, OSError) as exc:
+        return Score(task.id, False, False, f"generate error: {exc}", "", str(exc), 1, False)
+    source, early = _draft_source(outcome)
+    if early is not None:
+        return Score(task.id, early.passed, False, early.reason, "", "", 1, early.fallback)
+    assert source is not None
+    _remember(generate, prompt, outcome.output or source)
+    first = score_source(task, source)
+    if first.passed or not repair:
+        return first
+    err = (first.stderr or first.stdout).strip() or first.reason
+    repair_prompt = f"{REPAIR_PREFIX}```\n{err}\n```"
+    try:
+        repaired = complete(generate, checker, PYTHON_VIBE_FALLBACK, repair_prompt)
+    except (TimeoutError, RuntimeError, OSError) as exc:
+        return Score(
+            task.id, False, True, f"generate error: {exc}", first.stdout, first.stderr, first.exit_code, False
+        )
+    source2, early2 = _draft_source(repaired)
+    if early2 is not None:
+        return Score(
+            task.id, False, True, early2.reason, first.stdout, first.stderr, first.exit_code, early2.fallback
+        )
+    assert source2 is not None
+    second = score_source(task, source2)
+    return Score(
+        task.id,
+        second.passed,
+        True,
+        f"repair {second.reason}",
+        second.stdout,
+        second.stderr,
+        second.exit_code,
+        False,
+    )
+
+
+def run_repeats(
+    tasks: list[Task],
+    generate: GenerateFn,
+    *,
+    repair: bool,
+    repeats: int,
+    reset: Callable[[], None] | None = None,
+) -> Iterator[Score]:
+    for _ in range(repeats):
+        for task in tasks:
+            if reset is not None:
+                reset()
+            else:
+                _reset(generate)
+            yield score_generate(task, generate, repair=repair)