py-harness-cli 0.3.0__py3-none-any.whl

harness/mcp_stdio.py

@@ -0,0 +1,306 @@
+"""Local MCP over stdio so an editor can apply the write limit without a tunnel.
+
+This is the editor calling py-harness. It is not an Action the 8B may emit.
+Stdout is JSON-RPC only. Logs go to stderr.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any, TextIO
+
+from harness.agent import Agent, AgentOptions
+
+PROTOCOL = "2024-11-05"
+PROMPTS = (
+    {
+        "name": "ask",
+        "description": "Read-only question. py-harness does not change files.",
+        "arguments": [
+            {"name": "task", "description": "The question", "required": True}
+        ],
+    },
+    {
+        "name": "run",
+        "description": "Explore, edit and run inside the project folder.",
+        "arguments": [
+            {"name": "task", "description": "What to do", "required": True}
+        ],
+    },
+)
+
+TOOLS = (
+    {
+        "name": "ask",
+        "description": (
+            "Read-only question about the project. Uses the py-harness write limit. "
+            "Does not change files."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {"task": {"type": "string"}},
+            "required": ["task"],
+        },
+    },
+    {
+        "name": "run",
+        "description": (
+            "Explore, edit and run inside the project folder. Writes only when "
+            "the server was started with --allow-writes."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "task": {"type": "string"},
+                "scope": {"type": "string"},
+            },
+            "required": ["task"],
+        },
+    },
+    # These two need no model and take under a second, which makes them
+    # the ones worth reaching for in an editor. Only ask and run were
+    # offered, so the only tools an editor had were the two slowest and
+    # least certain.
+    {
+        "name": "brief",
+        "description": (
+            "Summarise this project: how large it is, what is in it, and "
+            "whether it is small enough to read whole. Needs no model."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {"scope": {"type": "string"}},
+        },
+    },
+    {
+        "name": "layout",
+        "description": (
+            "Report what makes this project hard to read: import cycles, "
+            "a folder with no grouping, an oversized module. Needs no model."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {"scope": {"type": "string"}},
+        },
+    },
+)
+
+
+def _text(rpc_id, body: str) -> dict[str, Any]:
+    """One plain answer, in the shape an editor expects."""
+    return {
+        "jsonrpc": "2.0",
+        "id": rpc_id,
+        "result": {"content": [{"type": "text", "text": body}]},
+    }
+
+
+def _describe(name: str, project: Path, scope: str) -> str:
+    """Answer brief or layout. Neither loads a model."""
+    from harness.scan.layout import render_layout
+    from harness.scan.project_brief import (
+        classify_project,
+        render_brief_for_person,
+        resolve_scope,
+    )
+
+    if name == "brief":
+        return render_brief_for_person(
+            classify_project(project, scope), scope=scope
+        )
+    base = resolve_scope(project, scope) if scope else project
+    return render_layout(base)
+
+
+def handle_rpc(
+    message: dict[str, Any],
+    *,
+    project: Path,
+    allow_writes: bool,
+    model: str,
+) -> dict[str, Any] | None:
+    """One JSON-RPC message. Notifications return None."""
+    method = str(message.get("method") or "")
+    rpc_id = message.get("id")
+    if method == "initialize":
+        return {
+            "jsonrpc": "2.0",
+            "id": rpc_id,
+            "result": {
+                "protocolVersion": PROTOCOL,
+                "capabilities": {"tools": {}, "prompts": {}},
+                "serverInfo": {"name": "py-harness", "version": "0.1.0"},
+            },
+        }
+    if method == "notifications/initialized":
+        return None
+    if method == "tools/list":
+        return {"jsonrpc": "2.0", "id": rpc_id, "result": {"tools": list(TOOLS)}}
+    if method == "prompts/list":
+        return {"jsonrpc": "2.0", "id": rpc_id, "result": {"prompts": list(PROMPTS)}}
+    if method == "prompts/get":
+        params = message.get("params") or {}
+        name = str(params.get("name") or "")
+        args = params.get("arguments") or {}
+        task = str(args.get("task") or "").strip() or "what does this project do?"
+        if name not in {"ask", "run"}:
+            return _error(rpc_id, f"unknown prompt {name}")
+        verb = "ask" if name == "ask" else "run"
+        return {
+            "jsonrpc": "2.0",
+            "id": rpc_id,
+            "result": {
+                "description": f"Call the py-harness {verb} tool",
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": {
+                            "type": "text",
+                            "text": f"Use the py-harness {verb} tool with task: {task}",
+                        },
+                    }
+                ],
+            },
+        }
+    if method == "tools/call":
+        params = message.get("params") or {}
+        name = str(params.get("name") or "")
+        args = params.get("arguments") or {}
+        scope = str(args.get("scope") or "").strip()
+        if name in {"brief", "layout"}:
+            return _text(rpc_id, _describe(name, project, scope))
+        task = str(args.get("task") or "").strip()
+        if not task:
+            return _error(rpc_id, "task required")
+        if name == "run" and not allow_writes:
+            return {
+                "jsonrpc": "2.0",
+                "id": rpc_id,
+                "result": {
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": (
+                                "this server is read-only. Restart it with "
+                                "--allow-writes, or run py-harness run in the terminal."
+                            ),
+                        }
+                    ],
+                    "isError": True,
+                },
+            }
+        if name not in {"ask", "run"}:
+            return _error(rpc_id, f"unknown tool {name}")
+        options = AgentOptions(
+            project=project,
+            task=task,
+            model=model,
+            scope=str(args.get("scope") or ""),
+            allow_writes=allow_writes and name == "run",
+        )
+        try:
+            result = Agent(options).run()
+        except (ValueError, OSError) as exc:
+            return _error(rpc_id, str(exc))
+        return {
+            "jsonrpc": "2.0",
+            "id": rpc_id,
+            "result": {
+                "content": [{"type": "text", "text": describe(result)}],
+                "isError": not result.ok,
+            },
+        }
+    if rpc_id is None:
+        return None
+    return _error(rpc_id, f"unknown method {method}")
+
+
+def describe(result) -> str:
+    """What to show in the editor: the answer, and what it left behind.
+
+    The summary alone is not enough when a run stops early. A person who
+    asked for a feature and got a question back also needs to know that
+    files were changed and that the tests no longer pass, or they will
+    find out later and blame the wrong thing.
+    """
+    lines = [result.summary or "done"]
+    if result.writes:
+        changed = ", ".join(sorted(set(result.writes)))
+        lines.append(f"\nChanged: {changed}")
+    if not result.ok:
+        if result.stopped == "question":
+            lines.append(
+                "This run stopped to ask. Nothing further was done, so the "
+                "change above may be half finished — run the tests before "
+                "relying on it."
+            )
+        elif result.stopped == "steps":
+            lines.append(
+                "This run ran out of steps before it finished. Check what "
+                "changed above."
+            )
+    return "\n".join(lines)
+
+
+def _error(rpc_id: Any, text: str) -> dict[str, Any]:
+    return {"jsonrpc": "2.0", "id": rpc_id, "error": {"code": -32600, "message": text}}
+
+
+def _read_message(stdin: TextIO) -> dict[str, Any] | None:
+    """Read one message. Lines are the transport; headers are tolerated."""
+    first = stdin.readline()
+    if first == "":
+        return None
+    if first.lstrip().startswith("{"):
+        return json.loads(first)
+    headers: dict[str, str] = {}
+    line = first
+    while line not in ("", "\r\n", "\n"):
+        key, _, value = line.partition(":")
+        headers[key.strip().lower()] = value.strip()
+        line = stdin.readline()
+        if line == "" and "content-length" not in headers:
+            return None
+    length = int(headers.get("content-length") or "0")
+    raw = stdin.read(length) if length else ""
+    return json.loads(raw) if raw else None
+
+
+def _write_message(stdout: TextIO, payload: dict[str, Any]) -> None:
+    """One message per line, as the stdio transport requires.
+
+    The specification says messages are delimited by newlines and must not
+    contain embedded newlines. Content-Length headers are the convention in
+    the Language Server Protocol, not this one, and a client that expects
+    lines cannot read them.
+    """
+    # json.dumps escapes a newline inside a value, so the serialized form is
+    # already a single line.
+    stdout.write(json.dumps(payload, separators=(",", ":")) + "\n")
+    stdout.flush()
+
+
+def serve_stdio(
+    project: Path, *, allow_writes: bool = False, model: str = ""
+) -> int:
+    print(
+        f"py-harness mcp  project {project}  "
+        f"{'read-write' if allow_writes else 'read-only'}",
+        file=sys.stderr,
+        flush=True,
+    )
+    while True:
+        try:
+            message = _read_message(sys.stdin)
+        except (json.JSONDecodeError, ValueError) as exc:
+            print(f"bad mcp message: {exc}", file=sys.stderr)
+            continue
+        if message is None:
+            return 0
+        reply = handle_rpc(
+            message, project=project, allow_writes=allow_writes, model=model
+        )
+        if reply is not None:
+            _write_message(sys.stdout, reply)

harness/memory/__init__.py

@@ -0,0 +1,5 @@
+"""What a run remembers between turns."""
+
+from harness.memory.conversation import Conversation
+
+__all__ = ["Conversation"]

harness/memory/conversation.py

@@ -0,0 +1,104 @@
+"""What a run remembers between turns, and what it is allowed to forget.
+
+Memory used to be a bare list on the generate function. Every turn
+appended the prompt and the reply, nothing was ever removed, and the
+request grew about 130 tokens a turn on top of an opening that is
+usually over a thousand.
+
+Nobody decided where that stopped. The harness sent no context size, so
+Ollama applied its own default — 4096 tokens for a model whose weights
+accept 131072 — and when a run crossed it the oldest messages were
+dropped by the server. The oldest message is the opening: the file the
+harness located and the instruction saying what to do with it. The run
+lost exactly the part it had done work to assemble, and said nothing.
+
+This decides instead. The opening is kept whatever else goes, because it
+is the only turn that carries the code. Recent turns are kept because
+that is where the run is. What goes is the middle, which is where a
+model has already been told it used the wrong verb four times.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+# Roughly four characters to a token. Only used to decide what to drop,
+# so it does not need to match a tokeniser.
+CHARS_PER_TOKEN = 4
+
+
+@dataclass
+class Conversation:
+    """The messages one run sends, and the budget they have to fit.
+
+    Fields:
+        budget_tokens: how much the model will read. Everything above
+            this is dropped here, in the open, rather than by the server.
+        system: the instruction that opens every request.
+        opening: the first prompt, which carries the located file. Kept
+            for as long as the run lasts.
+        turns: prompt and reply for every turn after the first.
+        dropped: how many middle turns have been let go, so a caller can
+            say so rather than guess.
+    """
+
+    budget_tokens: int = 8192
+    system: str = ""
+    opening: str = ""
+    turns: list[tuple[str, str]] = field(default_factory=list)
+    dropped: int = 0
+
+    @property
+    def budget_chars(self) -> int:
+        return self.budget_tokens * CHARS_PER_TOKEN
+
+    def remember(self, prompt: str, reply: str) -> None:
+        """Record one exchange. The first prompt becomes the opening."""
+        if not self.opening:
+            self.opening = prompt
+            self.turns.append(("", reply))
+            return
+        self.turns.append((prompt, reply))
+
+    def clear(self) -> None:
+        """Forget the run, keep the budget and the system line."""
+        self.opening = ""
+        self.turns.clear()
+        self.dropped = 0
+
+    def messages(self, prompt: str) -> list[dict[str, str]]:
+        """What to send for `prompt`, trimmed from the middle to fit."""
+        head: list[dict[str, str]] = []
+        if self.system:
+            head.append({"role": "system", "content": self.system})
+        if self.opening:
+            head.append({"role": "user", "content": self.opening})
+
+        body: list[dict[str, str]] = []
+        for asked, answered in self.turns:
+            if asked:
+                body.append({"role": "user", "content": asked})
+            if answered:
+                body.append({"role": "assistant", "content": answered})
+        tail = [{"role": "user", "content": prompt}]
+
+        fixed = _size(head) + _size(tail)
+        room = self.budget_chars - fixed
+        if room <= 0:
+            # Even the opening does not fit. Send the instruction and the
+            # question: a truncated file is worse than none, because the
+            # model reads half a function and answers about it.
+            self.dropped = len(self.turns)
+            return ([head[0]] if self.system else []) + tail
+
+        kept: list[dict[str, str]] = []
+        for message in reversed(body):
+            if _size(kept) + len(message["content"]) > room:
+                break
+            kept.insert(0, message)
+        self.dropped = max(0, (len(body) - len(kept) + 1) // 2)
+        return head + kept + tail
+
+
+def _size(messages: list[dict[str, str]]) -> int:
+    return sum(len(message["content"]) for message in messages)

harness/model/__init__.py

@@ -0,0 +1,6 @@
+"""Load and call a language model.
+
+Supports a LoRA adapter through MLX and a local model through Ollama. This
+is the only package whose output is not deterministic. It imports no other
+package in the harness, so everything else can be tested without a model.
+"""

harness/model/chat_backend.py

@@ -0,0 +1,100 @@
+"""What every model this project talks to has in common.
+
+Both hosts speak the same shape: post a list of messages, read one reply
+out of what comes back. Sixty-four per cent of the two files was the
+same lines — building the request, opening it, turning a failure into a
+message, pulling the text out.
+
+What actually differs is small and named here: where to post, what the
+body looks like, what headers it needs, where the reply sits in the
+answer, and what a status code means. A new host is those five things,
+not another copy of the transport.
+
+This is a base class rather than a function because the pieces vary
+together and are named by what they are. Most of this project is plain
+functions, which is right when a thing takes values and returns one; a
+model backend holds a host, a model name and a timeout, and answers
+several questions about them.
+"""
+
+from __future__ import annotations
+
+import json
+import urllib.error
+import urllib.request
+from collections.abc import Sequence
+from typing import Any
+
+
+class ChatBackend:
+    """One model, reachable over HTTP, that answers a list of messages."""
+
+    def __init__(self, model: str, system: str, *, timeout: float) -> None:
+        self.model = model
+        self.system = system
+        self.timeout = timeout
+
+    # -- what a host has to say for itself -----------------------------
+
+    def url(self) -> str:
+        raise NotImplementedError
+
+    def body(self, messages: list[dict[str, str]]) -> dict[str, Any]:
+        raise NotImplementedError
+
+    def headers(self) -> dict[str, str]:
+        return {"Content-Type": "application/json"}
+
+    def reply_from(self, payload: dict[str, Any]) -> str:
+        raise NotImplementedError
+
+    def unreachable(self, exc: Exception) -> str:
+        return f"{self.model} unreachable"
+
+    def refused(self, code: int) -> str:
+        return f"remote model HTTP {code}"
+
+    def before_send(self, messages: list[dict[str, str]]) -> None:
+        """Last look at what is about to leave. Raise to stop it.
+
+        Local hosts have nothing to check, so this does nothing by
+        default. A backend that posts somewhere else overrides it.
+        """
+
+    # -- what they all do the same way ---------------------------------
+
+    def __call__(
+        self, prompt: str, history: Sequence[dict[str, str]] | None = None
+    ) -> str:
+        """Answer one prompt, with earlier turns in front of it."""
+        messages: list[dict[str, str]] = [
+            {"role": "system", "content": self.system}
+        ]
+        if history:
+            messages.extend(history)
+        messages.append({"role": "user", "content": prompt})
+        return self.send(messages)
+
+    def send(self, messages: list[dict[str, str]]) -> str:
+        """Post exactly these messages. The caller decides what they are.
+
+        The conversation is assembled by `harness.memory`, which knows
+        what to keep and what to let go. This only sends it.
+        """
+        self.before_send(messages)
+        request = urllib.request.Request(
+            self.url(),
+            data=json.dumps(self.body(messages)).encode("utf-8"),
+            headers=self.headers(),
+            method="POST",
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=self.timeout) as answer:
+                payload = json.loads(answer.read().decode("utf-8"))
+        except urllib.error.HTTPError as exc:
+            # `from None` on purpose: the original carries the full URL,
+            # and a token can be in it.
+            raise RuntimeError(self.refused(exc.code)) from None
+        except urllib.error.URLError as exc:
+            raise RuntimeError(self.unreachable(exc)) from exc
+        return self.reply_from(payload)

harness/model/engine.py

@@ -0,0 +1,165 @@
+"""Load MLX LoRA or Ollama once; reuse for a batch of prompts."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+from collections.abc import Callable
+from pathlib import Path
+
+from finetune.huggingface_store import BEST_ADAPTER, ensure_adapters
+from finetune.models import SPECS
+from finetune.paths import PROJECT_ROOT
+from harness.paths import venv_python
+
+
+def mlx_pythons() -> list[str]:
+    extra = os.environ.get("MLX_PYTHON", "")
+    return [
+        sys.executable,
+        str(venv_python(PROJECT_ROOT / ".venv")),
+        *([extra] if extra else []),
+    ]
+
+
+def has_mlx(exe: str) -> bool:
+    if not Path(exe).is_file():
+        return False
+    probe = subprocess.run([exe, "-c", "import mlx_lm"], capture_output=True)
+    return probe.returncode == 0
+
+
+def reexec_for_mlx() -> None:
+    try:
+        import mlx_lm  # noqa: F401
+        return
+    except ImportError:
+        pass
+    for exe in mlx_pythons():
+        if exe == sys.executable:
+            continue
+        if has_mlx(exe):
+            os.execv(exe, [exe, *sys.argv])
+    sys.exit("mlx-lm missing. Use the Homebrew 3.13 venv or pass --engine ollama")
+
+
+def _stage_best(adapter_dir: Path) -> Path:
+    ckpt = adapter_dir / BEST_ADAPTER
+    if not ckpt.is_file():
+        sys.exit(f"no best checkpoint: {ckpt}")
+    staging = adapter_dir.parent / "python-vibe-100"
+    staging.mkdir(parents=True, exist_ok=True)
+    dest = staging / "adapters.safetensors"
+    cfg = staging / "adapter_config.json"
+    src_cfg = adapter_dir / "adapter_config.json"
+    if dest.exists() or dest.is_symlink():
+        dest.unlink()
+    dest.symlink_to(ckpt.resolve())
+    if src_cfg.is_file():
+        if cfg.exists() or cfg.is_symlink():
+            cfg.unlink()
+        cfg.symlink_to(src_cfg.resolve())
+    return staging
+
+
+def make_generate(
+    engine: str,
+    max_tokens: int,
+    *,
+    model: str | None = None,
+    system: str | None = None,
+    memory=None,
+    adapters: bool = True,
+) -> tuple[str, Callable[[str], str]]:
+    """A label and a function that answers one prompt.
+
+    `memory` is whatever the harness is using to remember the run. It
+    only has to answer `messages(prompt)`. Nothing here decides what is
+    remembered or forgotten: that belongs to the harness, and this
+    package does not import it.
+    """
+    if engine == "auto":
+        engine = "mlx" if any(has_mlx(p) for p in mlx_pythons()) else "ollama"
+    if engine == "mlx":
+        return _mlx_generate(
+            max_tokens, system=system, memory=memory, adapters=adapters
+        )
+    if engine == "openai":
+        return _openai_generate(
+            max_tokens, model=model, system=system, memory=memory
+        )
+    if engine != "ollama":
+        sys.exit(f"unknown engine {engine!r}: use ollama, mlx, or openai")
+    return _ollama_generate(model=model, system=system, memory=memory)
+
+
+def _mlx_generate(
+    max_tokens: int,
+    *,
+    system: str | None = None,
+    memory=None,
+    adapters: bool = True,
+) -> tuple[str, Callable[[str], str]]:
+    reexec_for_mlx()
+    from mlx_lm import generate, load
+
+    spec = SPECS["python-vibe"]
+    adapter_path: str | None = None
+    label = f"mlx-base:{spec.mlx_base}"
+    if adapters:
+        local = ensure_adapters(spec)
+        adapter = _stage_best(local) if (local / BEST_ADAPTER).is_file() else local
+        adapter_path = str(adapter)
+        label = f"mlx-lora:{adapter.name}"
+    model, tokenizer = load(spec.mlx_base, adapter_path=adapter_path)
+
+    def generate_once(prompt: str) -> str:
+        text = tokenizer.apply_chat_template(
+            memory.messages(prompt), tokenize=False, add_generation_prompt=True
+        )
+        return generate(model, tokenizer, prompt=text, max_tokens=max_tokens)
+
+    generate_once.memory = memory  # type: ignore[attr-defined]
+    return label, generate_once
+
+
+def _openai_generate(
+    max_tokens: int,
+    *,
+    model: str | None = None,
+    system: str | None = None,
+    memory=None,
+) -> tuple[str, Callable[[str], str]]:
+    from harness.model.openai_generate import OpenAIGenerate
+
+    spec = SPECS["python-vibe"]
+    name = model or spec.ollama_base
+    try:
+        backend = OpenAIGenerate(
+            name, system or spec.system, max_tokens=max_tokens
+        )
+    except ValueError as exc:
+        sys.exit(str(exc))
+    def generate_once(prompt: str) -> str:
+        return backend.send(memory.messages(prompt))
+
+    generate_once.memory = memory  # type: ignore[attr-defined]
+    return f"openai:{name}", generate_once
+
+
+def _ollama_generate(
+    *, model: str | None = None, system: str | None = None, memory=None
+) -> tuple[str, Callable[[str], str]]:
+    from harness.model.ollama_generate import OllamaGenerate
+
+    spec = SPECS["python-vibe"]
+    name = model or spec.ollama_base
+    backend = OllamaGenerate(name, system or spec.system)
+    if not backend.healthy():
+        sys.exit(f"ollama {backend.host} is down")
+    def generate_once(prompt: str) -> str:
+        return backend.send(memory.messages(prompt))
+
+    generate_once.memory = memory  # type: ignore[attr-defined]
+    return f"ollama:{name}", generate_once