sid-harness 0.3.0__py3-none-any.whl

sid_harness/loop.py

@@ -0,0 +1,394 @@
+"""Drive SID's EXISTING native loop with SID as the only model backend.
+
+The loop body, its tool contract, and its bounds are ``native_agent``'s. This
+module only wires the ``complete`` callback to SID's ``/v1``, executes the
+human-approved test argv locally (SUB-80), and optionally forwards a tool
+registry (SUB-56) and configurable bounds (SUB-58) into the ONE loop.
+
+Local test execution is deliberately narrow: a LIST of tokens (never a shell
+string), a recognised test-runner allowlist, the approved workspace root as
+``cwd``, a bounded wall-clock timeout, and a bounded output capture. The runner
+never commits and never pushes.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import subprocess
+import threading
+import time
+from dataclasses import replace
+from pathlib import Path
+from typing import Any
+
+from .client import SIDClient
+from .gateway import native_agent
+
+DEFAULT_TEST_TIMEOUT_SECONDS = 120.0
+MAX_TEST_TIMEOUT_SECONDS = 900.0
+MAX_TEST_OUTPUT_CHARS = 6000
+TEST_BACKEND = "local"
+
+TEST_NOT_EXECUTED_DETAIL = (
+    "no approved test command was provided; pass one (sid run --test-cmd ...) "
+    "to run tests locally."
+)
+
+# The approved command must be a recognised test-runner invocation. Matching is
+# prefix-based on the normalised executable plus the tokens that identify the
+# runner, so extra flags (``-q``, ``--maxfail=1``) are allowed but an arbitrary
+# executable is not. This is the mechanical "human-approved" boundary: a test
+# command, not a shell.
+_ALLOWED_TEST_PREFIXES: tuple[tuple[str, ...], ...] = (
+    ("python", "-m", "pytest"),
+    ("python3", "-m", "pytest"),
+    ("pytest",),
+    ("py.test",),
+    ("python", "-m", "unittest"),
+    ("python3", "-m", "unittest"),
+    ("npm", "test"),
+    ("npm", "run", "test"),
+    ("yarn", "test"),
+    ("pnpm", "test"),
+    ("cargo", "test"),
+    ("go", "test"),
+)
+
+_PYTHON_VERSIONED = re.compile(r"^python3(\.[0-9]+)?$")
+_WINDOWS_ABSOLUTE = re.compile(r"^[A-Za-z]:")
+
+
+def _empty_result(**overrides: Any) -> dict[str, Any]:
+    result: dict[str, Any] = {
+        "executed": False,
+        "passed": None,
+        "isolated": False,
+        "backend": "none",
+        "argv": None,
+        "cwd": None,
+        "exit_code": None,
+        "timed_out": False,
+        "duration_ms": 0,
+        "output": "",
+        "output_truncated": False,
+        "detail": None,
+    }
+    result.update(overrides)
+    return result
+
+
+def not_executed_test(*, detail: str = TEST_NOT_EXECUTED_DETAIL) -> dict[str, Any]:
+    """Result for a run with no approved test command — never claims a test ran."""
+    return _empty_result(detail=detail)
+
+
+def _program_name(token: str) -> str:
+    """Normalise an executable token to its bare program name.
+
+    ``C:\\Python312\\python.exe``, ``python3.12``, and ``python`` all normalise to
+    a name the allowlist can match, so a caller may pass ``sys.executable``.
+    """
+    name = token.replace("\\", "/").rsplit("/", 1)[-1].lower()
+    if name.endswith(".exe"):
+        name = name[: -len(".exe")]
+    if _PYTHON_VERSIONED.match(name):
+        return "python3" if name == "python3" else "python"
+    return name
+
+
+def match_test_allowlist(argv: list[str]) -> tuple[str, ...] | None:
+    """Return the allowlisted runner prefix matching ``argv``, else ``None``."""
+    if not argv:
+        return None
+    tokens = [_program_name(str(argv[0])), *[str(token) for token in argv[1:]]]
+    for prefix in _ALLOWED_TEST_PREFIXES:
+        if len(tokens) < len(prefix):
+            continue
+        if all(tokens[index] == prefix[index] for index in range(len(prefix))):
+            return prefix
+    return None
+
+
+def _escapes_workspace(token: str) -> bool:
+    normalized = token.replace("\\", "/")
+    if normalized.startswith("/") or _WINDOWS_ABSOLUTE.match(normalized):
+        return True
+    return ".." in normalized.split("/")
+
+
+def _capture_bounded(
+    argv: list[str], *, cwd: Path, timeout: float, output_limit: int
+) -> tuple[int | None, bool, bytes, bool]:
+    """Run ``argv`` in ``cwd``; return (exit, timed_out, output, truncated).
+
+    Output is drained on a reader thread and only the first ``output_limit``
+    bytes are kept, so a chatty test command cannot exhaust memory.
+    """
+    process = subprocess.Popen(
+        list(argv),
+        cwd=str(cwd),
+        shell=False,
+        stdin=subprocess.DEVNULL,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+    )
+    kept: list[bytes] = []
+    state = {"kept": 0, "truncated": False}
+
+    def drain() -> None:
+        stream = process.stdout
+        if stream is None:
+            return
+        while True:
+            block = stream.read(65536)
+            if not block:
+                break
+            room = output_limit - state["kept"]
+            if room > 0:
+                chunk = block[:room]
+                kept.append(chunk)
+                state["kept"] += len(chunk)
+                if len(chunk) < len(block):
+                    state["truncated"] = True
+            else:
+                state["truncated"] = True
+
+    reader = threading.Thread(target=drain, name="sid-test-output", daemon=True)
+    reader.start()
+    timed_out = False
+    try:
+        exit_code: int | None = process.wait(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        timed_out = True
+        process.kill()
+        try:
+            exit_code = process.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            exit_code = None
+    reader.join(timeout=10)
+    if process.stdout is not None:
+        try:
+            process.stdout.close()
+        except OSError:
+            pass
+    return exit_code, timed_out, b"".join(kept), state["truncated"]
+
+
+def execute_test_argv(
+    argv: list[str] | tuple[str, ...] | None,
+    *,
+    root: str | Path | None,
+    timeout_seconds: float = DEFAULT_TEST_TIMEOUT_SECONDS,
+    output_limit: int = MAX_TEST_OUTPUT_CHARS,
+) -> dict[str, Any]:
+    """Execute the human-approved test argv in the workspace root.
+
+    Refuses a string (a shell command line), a non-allowlisted runner, or argv
+    that points outside the workspace. Bounded timeout and bounded output; never
+    touches git, never commits, never pushes.
+    """
+    if argv is None:
+        return not_executed_test()
+    if isinstance(argv, str):
+        return _empty_result(detail="test argv must be a list of tokens, never a shell string")
+    if not isinstance(argv, (list, tuple)) or not all(isinstance(token, str) for token in argv):
+        return _empty_result(detail="test argv must be a list of strings")
+    tokens = list(argv)
+    if not tokens:
+        return not_executed_test()
+    if any(not token for token in tokens):
+        return _empty_result(detail="test argv must not contain empty tokens")
+    if any("\x00" in token for token in tokens):
+        return _empty_result(detail="test argv must not contain NUL bytes")
+
+    prefix = match_test_allowlist(tokens)
+    if prefix is None:
+        allowed = ", ".join(" ".join(item) for item in _ALLOWED_TEST_PREFIXES)
+        return _empty_result(
+            argv=tokens,
+            detail=f"test command is not allowlisted ({tokens[0]!r}); allowed runners: {allowed}",
+        )
+    for token in tokens[1:]:
+        if _escapes_workspace(token):
+            return _empty_result(
+                argv=tokens,
+                detail=f"test argv must stay inside the workspace: {token!r}",
+            )
+
+    if root is None:
+        return _empty_result(
+            argv=tokens,
+            detail="no local workspace root was supplied; refusing to run tests on the host",
+        )
+    workspace_root = Path(root)
+    if not workspace_root.is_dir():
+        return _empty_result(
+            argv=tokens,
+            detail=f"workspace root is not a directory: {workspace_root}",
+        )
+
+    try:
+        timeout = float(timeout_seconds)
+    except (TypeError, ValueError):
+        timeout = DEFAULT_TEST_TIMEOUT_SECONDS
+    if timeout <= 0:
+        timeout = DEFAULT_TEST_TIMEOUT_SECONDS
+    timeout = min(timeout, MAX_TEST_TIMEOUT_SECONDS)
+    limit = max(1, int(output_limit))
+
+    started = time.monotonic()
+    try:
+        exit_code, timed_out, raw, truncated = _capture_bounded(
+            tokens, cwd=workspace_root, timeout=timeout, output_limit=limit
+        )
+    except FileNotFoundError:
+        return _empty_result(
+            backend=TEST_BACKEND,
+            argv=tokens,
+            cwd=str(workspace_root),
+            detail=f"test command not found: {tokens[0]!r}",
+        )
+    except OSError as exc:
+        return _empty_result(
+            backend=TEST_BACKEND,
+            argv=tokens,
+            cwd=str(workspace_root),
+            detail=f"could not start test command: {type(exc).__name__}: {exc}",
+        )
+    duration_ms = int((time.monotonic() - started) * 1000)
+    output = raw.decode("utf-8", errors="replace")
+    if len(output) > limit:
+        output = output[:limit]
+        truncated = True
+
+    if timed_out:
+        return _empty_result(
+            executed=True,
+            passed=False,
+            backend=TEST_BACKEND,
+            argv=tokens,
+            cwd=str(workspace_root),
+            exit_code=exit_code,
+            timed_out=True,
+            duration_ms=duration_ms,
+            output=output,
+            output_truncated=truncated,
+            detail=f"test command timed out after {timeout:g}s",
+        )
+    return _empty_result(
+        executed=True,
+        passed=exit_code == 0,
+        backend=TEST_BACKEND,
+        argv=tokens,
+        cwd=str(workspace_root),
+        exit_code=exit_code,
+        duration_ms=duration_ms,
+        output=output,
+        output_truncated=truncated,
+        detail="tests passed" if exit_code == 0 else f"test command exited {exit_code}",
+    )
+
+
+async def run_loop(
+    *,
+    client: SIDClient,
+    task: str,
+    workspace_files: dict[str, str],
+    acceptance_criteria: list[str],
+    out_of_scope: list[str],
+    test_argv: list[str],
+    workspace_root: str | Path | None = None,
+    test_timeout_seconds: float = DEFAULT_TEST_TIMEOUT_SECONDS,
+    test_output_limit: int = MAX_TEST_OUTPUT_CHARS,
+    model: str = "auto",
+    max_tokens: int = 8000,
+    reasoning_effort: str | None = None,
+    registry: Any | None = None,
+    bounds: Any | None = None,
+    shell_policy: Any | None = None,
+    plan_only: bool = False,
+    subagent_tasks: list[Any] | None = None,
+    subagent_max_concurrency: int = 4,
+) -> dict[str, Any]:
+    async def complete(messages: list[dict[str, Any]], tools: list[dict[str, Any]]) -> dict[str, Any]:
+        return await asyncio.to_thread(
+            client.chat,
+            messages=messages,
+            tools=tools,
+            model=model,
+            max_tokens=max_tokens,
+            reasoning_effort=reasoning_effort,
+        )
+
+    async def run_tests(files: dict[str, str]) -> dict[str, Any]:
+        # SUB-80: the approved argv runs locally against the real workspace
+        # root on a worker thread, so a slow suite never blocks the event loop.
+        return await asyncio.to_thread(
+            execute_test_argv,
+            test_argv,
+            root=workspace_root,
+            timeout_seconds=test_timeout_seconds,
+            output_limit=test_output_limit,
+        )
+
+    # SUB-94: wire the shipped subagent fan-out into the run path. Each sub-task
+    # runs through THIS same loop with the same callbacks and workspace (a copy
+    # per sub-run); the findings are handed to the main run as data. Subagent
+    # edits are never applied — only the main run's changed files reach the
+    # caller's write path.
+    subagent_results: list[dict[str, Any]] = []
+    if subagent_tasks:
+        from .tools.subagents import fan_out
+
+        subagent_results = await fan_out(
+            tasks=subagent_tasks,
+            complete=complete,
+            run_tests=run_tests,
+            workspace_files=workspace_files,
+            max_concurrency=subagent_max_concurrency,
+        )
+        findings = "\n".join(
+            f"- {result.get('id')}: {result.get('final_response') or '(no response)'}"
+            + (
+                f" (changed: {', '.join(result.get('changed') or [])})"
+                if result.get("changed")
+                else ""
+            )
+            for result in subagent_results
+        )
+        task = (
+            f"{task}\n\nPARALLEL SUBAGENT FINDINGS (read-only; their edits are "
+            f"not applied by this run):\n{findings}"
+        )
+
+    # SUB-58: forward configurable bounds into the one loop (defaults = module
+    # constants). SUB-56: forward a tool registry through the single dispatch
+    # seam; the registry routes base tools back to native_agent._tool_result.
+    native_kwargs: dict[str, Any] = dict(bounds.to_native_kwargs()) if bounds is not None else {}
+    tool_schemas = registry.schemas() if registry is not None else None
+    tool_dispatch = registry.dispatch if registry is not None else None
+
+    # Sandbox-by-default: the local workspace root IS the shell sandbox. A
+    # policy without an explicit root is filled from the run's root; a run with
+    # no root still refuses (the loop's run_shell_tool never runs on the host).
+    if shell_policy is not None and getattr(shell_policy, "root", None) is None:
+        if workspace_root is not None:
+            shell_policy = replace(shell_policy, root=str(workspace_root))
+
+    result = await native_agent.run_native_agent(
+        task=task,
+        workspace_files=workspace_files,
+        acceptance_criteria=acceptance_criteria,
+        out_of_scope=out_of_scope,
+        test_argv=test_argv,
+        complete=complete,
+        run_tests=run_tests,
+        tool_schemas=tool_schemas,
+        tool_dispatch=tool_dispatch,
+        shell_policy=shell_policy,
+        plan_only=plan_only,
+        **native_kwargs,
+    )
+    result["subagents"] = subagent_results
+    return result