lisa-harness-e2e 0.1.0__py3-none-any.whl

e2etest/__init__.py

@@ -0,0 +1 @@
+"""End-to-end scenario harness for Lisa packs (drives a full ``HostRuntime``)."""

e2etest/bundle.py

@@ -0,0 +1,67 @@
+"""Write a run bundle: verdict, raw outcome, and a readable transcript."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict
+from pathlib import Path
+from typing import Any
+
+from e2etest.oracles import Evidence, OracleResult
+from e2etest.scenario import Scenario
+
+_TEXT_LIMIT = 4000
+
+
+def verdict(scenario: Scenario, evidence: Evidence, results: list[OracleResult]) -> dict[str, Any]:
+    failed = [r for r in results if not r.passed]
+    return {
+        "scenario": scenario.scenario_id,
+        "scenario_file": str(scenario.source),
+        "run_dir": str(evidence.paths.run_dir),
+        "passed": not failed,
+        "first_failure": asdict(failed[0]) if failed else None,
+        "exchanges": len(evidence.outcome.exchanges),
+        "aborted": evidence.outcome.aborted,
+        "results": [asdict(r) for r in results],
+    }
+
+
+def _transcript(evidence: Evidence) -> str:
+    lines = ["# Transcript", ""]
+    for ex in evidence.outcome.exchanges:
+        lines.append(f"## #{ex.index} {ex.kind} ({ex.duration_s}s)")
+        lines += ["", "```text", ex.message, "```", ""]
+        flags = []
+        if ex.approval:
+            flags.append(
+                f"approval: {ex.approval.get('target_id')} ({ex.approval.get('request_id')})"
+            )
+        if ex.operator_exit:
+            flags.append(f"operator_exit: {ex.operator_exit}")
+        if ex.yielded_question:
+            flags.append(f"yielded: {ex.yielded_question}")
+        flags += [f"error: {e}" for e in ex.errors]
+        lines += [f"- {f}" for f in flags]
+        text = ex.text if len(ex.text) <= _TEXT_LIMIT else ex.text[:_TEXT_LIMIT] + "\n…[truncated]"
+        lines += ["", text, ""]
+    lines += ["# Thread entries", ""]
+    for entry in evidence.thread_entries:
+        receipts = [r.get("capability_id") for r in entry.get("effect_receipts") or ()]
+        provenance = entry.get("provenance") or {}
+        lines.append(
+            f"- turn {entry.get('turn')} mode={entry.get('mode_id')} "
+            f"handoff={entry.get('handoff_to') or '-'} model={provenance.get('model_id', '-')} "
+            f"exit={provenance.get('operator_exit') or '-'} receipts={receipts}"
+        )
+    return "\n".join(lines) + "\n"
+
+
+def write_bundle(run_dir: Path, payload: dict[str, Any], evidence: Evidence) -> Path:
+    (run_dir / "outcome.json").write_text(
+        json.dumps(evidence.outcome.to_json(), indent=1), encoding="utf-8"
+    )
+    (run_dir / "transcript.md").write_text(_transcript(evidence), encoding="utf-8")
+    path = run_dir / "verdict.json"
+    path.write_text(json.dumps(payload, indent=1), encoding="utf-8")
+    return path

e2etest/cli.py

@@ -0,0 +1,63 @@
+"""``lisa-e2e``: run one scenario end to end and write a verdict bundle."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from datetime import UTC, datetime
+from pathlib import Path
+
+from e2etest.bundle import verdict, write_bundle
+from e2etest.driver import run_conversation
+from e2etest.environment import prepare_run
+from e2etest.oracles import Evidence, evaluate
+from e2etest.packs_venv import ensure_packs_venv
+from e2etest.scenario import load_scenario
+
+_REPO_ROOT = Path(__file__).resolve().parents[3]
+_EXIT_FAIL = 1
+_EXIT_SETUP = 2
+
+
+def _parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="lisa-e2e", description=__doc__)
+    sub = parser.add_subparsers(dest="command", required=True)
+    run = sub.add_parser("run", help="Run a scenario against a live model.")
+    run.add_argument("scenario", type=Path)
+    run.add_argument("--runs-dir", type=Path, default=_REPO_ROOT / ".e2e" / "runs")
+    run.add_argument("--rebuild-venv", action="store_true")
+    return parser
+
+
+def _run(args: argparse.Namespace) -> int:
+    scenario = load_scenario(args.scenario)
+    venv = ensure_packs_venv(
+        scenario.packs, _REPO_ROOT / ".e2e" / "cache", rebuild=args.rebuild_venv
+    )
+    stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    run_dir = (args.runs_dir / scenario.scenario_id.replace("/", "__") / stamp).resolve()
+    paths = prepare_run(scenario, run_dir, venv)
+    print(f"run dir: {run_dir}", flush=True)
+    outcome = run_conversation(scenario, paths, venv)
+    evidence = Evidence(paths, outcome)
+    payload = verdict(scenario, evidence, evaluate(scenario.oracles, evidence))
+    path = write_bundle(run_dir, payload, evidence)
+    for result in payload["results"]:
+        mark = "PASS" if result["passed"] else "FAIL"
+        first = result["detail"].splitlines()[0] if result["detail"] else ""
+        print(f"{mark} {result['phase']}.{result['name']}: {first}")
+    print(f"verdict: {'PASS' if payload['passed'] else 'FAIL'} → {path}")
+    return 0 if payload["passed"] else _EXIT_FAIL
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _parser().parse_args(argv)
+    try:
+        return _run(args)
+    except (OSError, ValueError, RuntimeError) as exc:
+        print(f"setup error: {type(exc).__name__}: {exc}", file=sys.stderr)
+        return _EXIT_SETUP
+
+
+if __name__ == "__main__":
+    sys.exit(main())

e2etest/driver.py

@@ -0,0 +1,249 @@
+"""Drive a real ``HostRuntime`` through a scenario's conversation."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+import runtime.turn_hooks_builder
+from runtime.bootstrap import build_runtime
+from runtime.handle_message import handle_message
+from runtime.paths import inject_packs_venv_site_packages
+from runtime.types import HostCommandResult
+
+from e2etest.environment import RunPaths
+from e2etest.scenario import Scenario
+from e2etest.scripted import ScriptedGatewayProvider, load_script
+
+_APPROVAL_SCHEMA = "lisa.approval.v1"
+
+
+@dataclass
+class Exchange:
+    """One message sent to the host and what came back."""
+
+    index: int
+    kind: str
+    message: str
+    text: str = ""
+    is_error: bool = False
+    operator_exit: str = ""
+    yielded_question: str = ""
+    approval: dict[str, Any] | None = None
+    errors: list[str] = field(default_factory=list)
+    duration_s: float = 0.0
+
+
+@dataclass
+class DriverOutcome:
+    """Everything the driver observed; ``aborted`` names a blown budget."""
+
+    exchanges: list[Exchange] = field(default_factory=list)
+    stages: list[dict[str, Any]] = field(default_factory=list)
+    aborted: str = ""
+    model_calls: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_json(self) -> dict[str, Any]:
+        return {
+            "aborted": self.aborted,
+            "exchanges": [asdict(e) for e in self.exchanges],
+            "stages": self.stages,
+            "model_calls": self.model_calls,
+        }
+
+
+class RecordingTransport:
+    """``TransportCallbacks`` that records stages and errors instead of rendering."""
+
+    def __init__(self, outcome: DriverOutcome) -> None:
+        self._outcome = outcome
+        self.errors: list[str] = []
+
+    async def on_stage(self, label: str, data: dict[str, Any] | None = None) -> None:
+        self._outcome.stages.append(
+            {
+                "exchange": len(self._outcome.exchanges),
+                "t": round(time.monotonic(), 3),
+                "label": label,
+                "data": _jsonable(data or {}),
+            }
+        )
+
+    async def on_text_chunk(self, text: str, reasoning: str = "") -> None:
+        return None
+
+    async def on_response(self, text: str, data: dict[str, Any] | None = None) -> None:
+        return None
+
+    async def on_debug_context(self, context: dict[str, Any]) -> None:
+        return None
+
+    async def on_error(self, message: str, exc: Exception | None = None) -> None:
+        self.errors.append(message)
+
+
+def _jsonable(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(k): _jsonable(v) for k, v in value.items()}
+    if isinstance(value, list | tuple):
+        return [_jsonable(v) for v in value]
+    if value is None or isinstance(value, str | int | float | bool):
+        return value
+    return str(value)
+
+
+def approval_payload(result: HostCommandResult) -> dict[str, Any] | None:
+    """Return the structured approval request a turn paused on, if any."""
+    data = result.data or {}
+    nested = data.get("approval")
+    if isinstance(nested, dict) and nested.get("request_id"):
+        return nested
+    if data.get("schema") == _APPROVAL_SCHEMA and data.get("request_id"):
+        return data
+    return None
+
+
+class _Conversation:
+    def __init__(self, runtime: Any, scenario: Scenario, outcome: DriverOutcome) -> None:
+        self.runtime = runtime
+        self.scenario = scenario
+        self.outcome = outcome
+        self.transport = RecordingTransport(outcome)
+        self.decisions = 0
+        self.continues = 0
+        self.yield_replies = 0
+
+    async def send(self, kind: str, message: str) -> Exchange:
+        if len(self.outcome.exchanges) >= self.scenario.budgets.max_messages:
+            raise _BudgetExceededError(f"max_messages={self.scenario.budgets.max_messages}")
+        message = self._expand_message(message)
+        exchange = Exchange(index=len(self.outcome.exchanges), kind=kind, message=message)
+        self.outcome.exchanges.append(exchange)
+        self.transport.errors = []
+        started = time.monotonic()
+        result = await handle_message(self.runtime, message, self.transport)
+        exchange.duration_s = round(time.monotonic() - started, 2)
+        exchange.text = result.text
+        exchange.is_error = result.is_error
+        exchange.operator_exit = result.operator_exit
+        exchange.yielded_question = result.yielded_question
+        exchange.approval = approval_payload(result)
+        exchange.errors = list(self.transport.errors)
+        return exchange
+
+    def _expand_message(self, message: str) -> str:
+        """Resolve ``$RECOVERY_OPEN_ID`` from the run's recovery store.
+
+        Substitutes the work-item id of the first open (running/stalled/
+        indeterminate) recovery record, read straight from
+        ``data/state/recovery-items.json``. Left verbatim when there is no
+        open record (the substitution happens at send time, so a stall
+        created by an earlier turn resolves here). Lets scenarios drive
+        deterministic operator remediation by id without parsing model
+        prose.
+        """
+        if "$RECOVERY_OPEN_ID" not in message:
+            return message
+        path = self.runtime.paths.data_dir / "state" / "recovery-items.json"
+        if not path.is_file():
+            return message
+        try:
+            records = json.loads(path.read_text(encoding="utf-8"))
+        except (OSError, ValueError):
+            return message
+        if not isinstance(records, dict):
+            return message
+        for record in records.values():
+            if record.get("state") in ("running", "stalled", "indeterminate"):
+                return message.replace("$RECOVERY_OPEN_ID", str(record.get("work_item_id") or ""))
+        return message
+
+    async def follow_up(self, exchange: Exchange) -> Exchange | None:
+        """Answer whatever left the turn unfinished, or return ``None`` when it finished."""
+        budgets = self.scenario.budgets
+        if exchange.approval:
+            if self.decisions >= self.scenario.approvals.max_decisions:
+                raise _BudgetExceededError("approvals.max_decisions")
+            self.decisions += 1
+            verdict = self.scenario.approvals.decide(str(exchange.approval.get("target_id", "")))
+            return await self.send(verdict, f"/{verdict} {exchange.approval['request_id']}")
+        if exchange.yielded_question and self.yield_replies < budgets.max_yield_replies:
+            self.yield_replies += 1
+            return await self.send("yield_reply", self.scenario.yield_reply)
+        if exchange.operator_exit and self.continues < budgets.max_continues:
+            self.continues += 1
+            return await self.send("continue", self.scenario.continue_message)
+        return None
+
+    async def run(self) -> None:
+        for turn in self.scenario.turns:
+            exchange: Exchange | None = await self.send("turn", turn)
+            while exchange is not None:
+                exchange = await self.follow_up(exchange)
+
+
+class _BudgetExceededError(Exception):
+    pass
+
+
+async def _drive(runtime: Any, scenario: Scenario, outcome: DriverOutcome) -> None:
+    conversation = _Conversation(runtime, scenario, outcome)
+    await runtime.start()
+    try:
+        async with asyncio.timeout(scenario.budgets.wall_clock_s):
+            await conversation.run()
+    except _BudgetExceededError as exc:
+        outcome.aborted = f"budget exceeded: {exc}"
+    except TimeoutError:
+        outcome.aborted = f"budget exceeded: wall_clock_s={scenario.budgets.wall_clock_s}"
+    finally:
+        try:
+            await asyncio.wait_for(runtime.aclose(), timeout=30)
+        except TimeoutError:
+            outcome.aborted = outcome.aborted or "runtime.aclose() timed out"
+
+
+@contextmanager
+def _gateway_override(scripted: ScriptedGatewayProvider | None) -> Iterator[None]:
+    """Bind every provider gateway to the model script while the runtime is wired."""
+    if scripted is None:
+        yield
+        return
+    hooks = runtime.turn_hooks_builder
+    original = hooks._resolve_gateway_provider_for_runtime
+    hooks._resolve_gateway_provider_for_runtime = lambda _runtime: scripted
+    try:
+        yield
+    finally:
+        hooks._resolve_gateway_provider_for_runtime = original
+
+
+def run_conversation(scenario: Scenario, paths: RunPaths, packs_venv: Path) -> DriverOutcome:
+    """Boot a runtime against ``paths`` and play the scenario's turns."""
+    outcome = DriverOutcome()
+    if inject_packs_venv_site_packages(packs_venv) is None:
+        outcome.aborted = f"packs venv has no site-packages: {packs_venv}"
+        return outcome
+    scripted: ScriptedGatewayProvider | None = None
+    if scenario.model_script is not None:
+        scripted = load_script(scenario.model_script)
+        outcome.model_calls = scripted.log
+    with _gateway_override(scripted):
+        runtime = build_runtime(
+            host_kind="e2e",
+            data_dir=str(paths.data_dir),
+            session_context=str(paths.session_context),
+            streaming="none",
+            in_process_mcp=True,
+        )
+        if runtime.session.restart_required:
+            outcome.aborted = "runtime requested a restart during boot (pack auto-install?)"
+            return outcome
+        asyncio.run(_drive(runtime, scenario, outcome))
+    return outcome

e2etest/environment.py

@@ -0,0 +1,134 @@
+"""Isolated run directory: data dir, config, fixture repo, and process env."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from string import Template
+
+from dotenv import load_dotenv
+
+from e2etest.scenario import Scenario
+
+_DEFAULT_POLICY = """\
+version: 1
+mode: ask
+sandbox_downgrade: true
+bare_requires_approval: false
+allow: []
+ask: []
+deny: []
+"""
+# Operator location overrides that would point the run back at real state.
+_LOCATION_ENV = (
+    "LISA_PROFILE",
+    "LISA_THREAD_DIR",
+    "LISA_SESSION_CONTEXT_PATH",
+    "LISA_MCP_LOG_PATH",
+)
+_GIT_IDENTITY = {
+    "GIT_AUTHOR_NAME": "lisa-e2e",
+    "GIT_AUTHOR_EMAIL": "lisa-e2e@localhost",
+    "GIT_COMMITTER_NAME": "lisa-e2e",
+    "GIT_COMMITTER_EMAIL": "lisa-e2e@localhost",
+}
+
+
+@dataclass(frozen=True)
+class RunPaths:
+    """Every location a run touches; all live under ``run_dir``."""
+
+    run_dir: Path
+    data_dir: Path
+    workspace: Path | None
+    remote: Path | None
+    initial_main_sha: str
+    scenario_dir: Path | None = None
+
+    @property
+    def session_context(self) -> Path:
+        return self.data_dir / "config" / "session_context.md"
+
+    @property
+    def files_root(self) -> Path:
+        """The lisa-storage sandbox root (``StorageBackend.files_root()``)."""
+        return self.data_dir / "data" / "files"
+
+
+def base_data_dir() -> Path:
+    """The operator's real data dir; only models.yaml and .env are read from it."""
+    override = os.getenv("LISA_E2E_BASE_DATA_DIR", "").strip()
+    return Path(override).expanduser() if override else Path.home() / ".lisa"
+
+
+def git(cwd: Path, *args: str) -> str:
+    proc = subprocess.run(
+        ["git", *args],
+        cwd=cwd,
+        capture_output=True,
+        text=True,
+        check=True,
+        env={**os.environ, **_GIT_IDENTITY},
+    )
+    return proc.stdout.strip()
+
+
+def _seed_fixture(scenario: Scenario, run_dir: Path) -> tuple[Path | None, Path | None, str]:
+    if scenario.fixture_repo is None:
+        return None, None, ""
+    workspace = run_dir / "workspace" / scenario.fixture_repo.name
+    shutil.copytree(
+        scenario.fixture_repo, workspace, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")
+    )
+    git(workspace, "init", "--quiet", "--initial-branch", "main")
+    for key, value in (("user.name", "lisa-e2e"), ("user.email", "lisa-e2e@localhost")):
+        git(workspace, "config", key, value)
+    git(workspace, "add", "-A")
+    git(workspace, "commit", "--quiet", "-m", "fixture: initial state")
+    remote = run_dir / "remote.git"
+    git(run_dir, "init", "--quiet", "--bare", str(remote))
+    git(workspace, "remote", "add", "origin", str(remote))
+    git(workspace, "push", "--quiet", "origin", "main")
+    return workspace, remote, git(workspace, "rev-parse", "HEAD")
+
+
+def prepare_run(scenario: Scenario, run_dir: Path, packs_venv: Path) -> RunPaths:
+    """Create the run dir, seed config + fixture, and point the process env at it."""
+    data_dir = run_dir / "data"
+    config_dir = data_dir / "config"
+    config_dir.mkdir(parents=True)
+    base = base_data_dir()
+    models = base / "config" / "models.yaml"
+    if not models.is_file():
+        raise FileNotFoundError(f"base models.yaml not found: {models}")
+    shutil.copy2(models, config_dir / "models.yaml")
+    policy_text = (
+        scenario.policy.read_text(encoding="utf-8") if scenario.policy else _DEFAULT_POLICY
+    )
+    (config_dir / "policy.yaml").write_text(policy_text, encoding="utf-8")
+
+    workspace, remote, initial_sha = _seed_fixture(scenario, run_dir)
+    paths = RunPaths(run_dir, data_dir, workspace, remote, initial_sha, scenario.base_dir)
+    if scenario.fixture_data_files is not None:
+        shutil.copytree(scenario.fixture_data_files, paths.files_root)
+    template = Template(scenario.session_context.read_text(encoding="utf-8"))
+    paths.session_context.write_text(
+        template.substitute(
+            RUN_DIR=str(run_dir),
+            WORKSPACE=str(workspace or ""),
+            WORKSPACE_PARENT=str(workspace.parent if workspace else ""),
+        ),
+        encoding="utf-8",
+    )
+
+    load_dotenv(base / ".env", override=False)
+    for name in _LOCATION_ENV:
+        os.environ.pop(name, None)
+    os.environ["LISA_DATA_DIR"] = str(data_dir)
+    os.environ["LISA_PACKS_VENV"] = str(packs_venv)
+    os.environ.update(_GIT_IDENTITY)
+    os.environ.update(scenario.env)
+    return paths