marcus-toolkit 0.1.0__py3-none-any.whl

marcus_toolkit/integrity.py

@@ -0,0 +1,251 @@
+"""
+Run guards: prove a run executed the code its record names.
+
+* :func:`tree_hash` hashes the files that execute, the construction the
+  Minecraft image guard uses (sorted path, length, bytes); a missing file is
+  a refusal, never a quietly shorter hash.
+* :func:`committed_hash` hashes the same files as committed in a git
+  repository, so uncommitted edits cannot satisfy the guard.
+* :func:`git_version` is the ``marcus-version.txt`` record: checkout,
+  branch, commit, dirty file count.
+
+Lifted from the Minecraft plumbing's ``plumbing_hash.py``.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import subprocess  # nosec B404 - git on a path we own, fixed argv
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+
+def tree_hash(root: Path, files: Sequence[str]) -> str:
+    """
+    Hash the named files under ``root``.
+
+    Parameters
+    ----------
+    root : Path
+        The directory the run executes from.
+    files : Sequence[str]
+        The files that execute, relative to ``root``.
+
+    Returns
+    -------
+    str
+        A sha256 hex digest over each file's relative path, byte length and
+        bytes, in sorted order.
+
+    Raises
+    ------
+    FileNotFoundError
+        If ``root`` is not a directory or any named file is missing.
+    """
+    if not root.is_dir():
+        raise FileNotFoundError(f"not a directory: {root}")
+    digest = hashlib.sha256()
+    for name in sorted(files):
+        path = root / name
+        if not path.is_file():
+            raise FileNotFoundError(f"missing from {root}: {name}")
+        content = path.read_bytes()
+        digest.update(name.encode())
+        digest.update(b"\0")
+        digest.update(str(len(content)).encode())
+        digest.update(b"\0")
+        digest.update(content)
+    return digest.hexdigest()
+
+
+def committed_hash(
+    repo: Path, files: Sequence[str], subdir: str = "", ref: str = "HEAD"
+) -> str:
+    """
+    Hash the same files as they are committed in ``repo`` at ``ref``.
+
+    Parameters
+    ----------
+    repo : Path
+        The repository root.
+    files : Sequence[str]
+        The files, relative to ``subdir``.
+    subdir : str
+        Where they live inside the repository (``""`` for the root).
+    ref : str
+        The commit to read.
+
+    Returns
+    -------
+    str
+        The same construction as :func:`tree_hash`, over the committed bytes.
+
+    Raises
+    ------
+    FileNotFoundError
+        If git cannot be run, the ref does not exist, or a file is not in that commit.
+    """
+    digest = hashlib.sha256()
+    prefix = f"{subdir.rstrip('/')}/" if subdir else ""
+    for name in sorted(files):
+        try:
+            content = subprocess.run(  # nosec B603 B607
+                ["git", "-C", str(repo), "show", f"{ref}:{prefix}{name}"],
+                capture_output=True,
+                check=True,
+            ).stdout
+        except (OSError, subprocess.CalledProcessError) as error:
+            raise FileNotFoundError(
+                f"not in {repo} at {ref}: {prefix}{name} ({error})"
+            ) from error
+        digest.update(name.encode())
+        digest.update(b"\0")
+        digest.update(str(len(content)).encode())
+        digest.update(b"\0")
+        digest.update(content)
+    return digest.hexdigest()
+
+
+def uncommitted(repo: Path, files: Sequence[str], subdir: str = "") -> list[str]:
+    """
+    Which of the files have uncommitted changes in ``repo``.
+
+    Parameters
+    ----------
+    repo : Path
+        The repository root.
+    files : Sequence[str]
+        The files, relative to ``subdir``.
+    subdir : str
+        Where they live inside the repository.
+
+    Returns
+    -------
+    List[str]
+        Paths git reports as changed; empty when git cannot answer.
+    """
+    prefix = f"{subdir.rstrip('/')}/" if subdir else ""
+    try:
+        out = subprocess.run(  # nosec B603 B607
+            [
+                "git",
+                "-C",
+                str(repo),
+                "status",
+                "--porcelain",
+                "--",
+                *(f"{prefix}{n}" for n in files),
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        ).stdout
+    except (OSError, subprocess.CalledProcessError):
+        return []
+    return [line[3:].strip() for line in out.splitlines() if line.strip()]
+
+
+def git_version(checkout: Path) -> dict[str, str]:
+    """
+    Read the version record of a checkout: what ``marcus-version.txt`` holds.
+
+    Parameters
+    ----------
+    checkout : Path
+        A git working directory.
+
+    Returns
+    -------
+    Dict[str, str]
+        ``checkout``, ``branch``, ``commit`` and ``dirty`` (a count of changed
+        files, as text).
+    """
+
+    def git(*args: str) -> str:
+        try:
+            return subprocess.run(
+                ["git", "-C", str(checkout), *args],
+                capture_output=True,
+                text=True,
+                check=True,
+            ).stdout.strip()  # nosec B603 B607
+        except (OSError, subprocess.CalledProcessError):
+            return ""
+
+    dirty = git("status", "--porcelain")
+    return {
+        "checkout": str(checkout),
+        "branch": git("rev-parse", "--abbrev-ref", "HEAD"),
+        "commit": git("rev-parse", "HEAD"),
+        "dirty": str(len([line for line in dirty.splitlines() if line.strip()])),
+    }
+
+
+def format_version(version: dict[str, str]) -> str:
+    """Render :func:`git_version` the way the Minecraft run records print it."""
+    return (
+        f"checkout: {version['checkout']}\nbranch:   {version['branch']}\n"
+        f"commit:   {version['commit']}\ndirty:    {version['dirty']} files\n"
+    )
+
+
+def contains_commit(checkout: Path, sha: str) -> bool:
+    """
+    Whether ``checkout``'s HEAD descends from ``sha``.
+
+    Parameters
+    ----------
+    checkout : Path
+        A git working directory.
+    sha : str
+        A commit, full or abbreviated.
+
+    Returns
+    -------
+    bool
+    """
+    try:
+        subprocess.run(
+            ["git", "-C", str(checkout), "merge-base", "--is-ancestor", sha, "HEAD"],
+            capture_output=True,
+            check=True,
+        )  # nosec B603 B607
+        return True
+    except (OSError, subprocess.CalledProcessError):
+        return False
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Print the hash of a directory or of a repository's committed tree."""
+    args = list(sys.argv[1:] if argv is None else argv)
+    if len(args) >= 2 and args[0] != "--committed":
+        try:
+            print(tree_hash(Path(args[0]), args[1].split(",")))
+        except FileNotFoundError as error:
+            print(f"integrity: {error}", file=sys.stderr)
+            return 2
+        return 0
+    if len(args) >= 3 and args[0] == "--committed":
+        try:
+            print(
+                committed_hash(
+                    Path(args[1]),
+                    args[2].split(","),
+                    subdir=args[3] if len(args) > 3 else "",
+                )
+            )
+        except FileNotFoundError as error:
+            print(f"integrity: {error}", file=sys.stderr)
+            return 2
+        return 0
+    print(
+        "usage: marcus-toolkit-hash <dir> <file,file,...>"
+        " | --committed <repo> <file,file,...> [subdir]",
+        file=sys.stderr,
+    )
+    return 2
+
+
+if __name__ == "__main__":
+    sys.exit(main())

marcus_toolkit/livecheck.py

@@ -0,0 +1,223 @@
+"""
+The toolbox's own live check: a real server, a scripted agent, every reply read back.
+
+What it proves, the way every extension's free rehearsal proves it: a server
+under test starts with no key and its model off; a project is seeded with
+two tasks, the second depending on the first; one scripted agent registers,
+pulls the first, reports with evidence, pulls the second (now claimable),
+closes it, and hears the drain; the sampler read the board from outside
+before and after; the server log carries no model-provider host.
+
+Usage::
+
+    python -m marcus_toolkit.livecheck --checkout ~/dev/marcus --out livecheck-runs
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+from typing import Any
+
+from . import integrity, measure, record, sampler, seed, server
+from .agent import BoardAgent, jsonl_logger
+from .client import MarcusMCPClient
+
+TASKS: list[dict[str, Any]] = [
+    {
+        "name": "Say where you are",
+        "description": "Report your position on the board; nothing else to do.",
+        "labels": ["livecheck", "seeded"],
+        "completion_criteria": ["The completion report says where the agent is"],
+        "estimated_hours": 0.1,
+        "priority": "high",
+    },
+    {
+        "name": "Say it again, now that the first report is in",
+        "description": "Only claimable once the first task is done.",
+        "labels": ["livecheck", "seeded"],
+        "depends_on": ["Say where you are"],
+        "completion_criteria": ["The completion report repeats the position"],
+        "estimated_hours": 0.1,
+        "priority": "medium",
+    },
+]
+
+
+def run(checkout: Path, out: Path, python: str = sys.executable) -> dict[str, Any]:
+    """
+    Run the live check and return its verdict.
+
+    Parameters
+    ----------
+    checkout : Path
+        A Marcus checkout to serve from.
+    out : Path
+        The ``runs`` root; a ``<stamp>/`` directory is created under it.
+    python : str
+        The interpreter that runs the server.
+
+    Returns
+    -------
+    Dict[str, Any]
+        ``{"PASS": bool, "checks": {...}, "record": str}``.
+    """
+    rec = record.RunRecord(out)
+    data_dir = rec.dir / "data"
+    port = server.free_port()
+    config_path = rec.dir / "config_marcus.json"
+    server.write_config(
+        config_path,
+        port=port,
+        project_name="runner-livecheck",
+        sqlite_db_path=data_dir / "kanban.db",
+    )
+    version = integrity.git_version(checkout)
+    rec.write("marcus-version.txt", integrity.format_version(version))
+    for key, value in [
+        ("run_at", rec.stamp),
+        ("kind", "marcus-toolkit live check"),
+        ("port", port),
+        ("checkout", f"{checkout} @ {version['commit']} ({version['dirty']} dirty)"),
+        ("ai", "off, keys stripped"),
+    ]:
+        rec.config_line(key, value)
+
+    srv = server.ServerUnderTest(
+        checkout, data_dir, config_path, port, rec.dir / "server.log", python=python
+    )
+    checks: dict[str, bool] = {}
+    detail: dict[str, Any] = {}
+    try:
+        srv.start(wait_seconds=120)
+        client = MarcusMCPClient(srv.url, timeout=120)
+        if not client.connect():
+            raise RuntimeError(
+                f"could not connect to the server under test: {client.last_error}"
+            )
+        seeded = seed.seed_project(
+            client,
+            name="runner-livecheck",
+            description="the toolbox's own live check",
+            tasks=TASKS,
+            project_root=rec.dir / "project-root",
+            objective="Say where you are; the board is the only channel.",
+        )
+        rec.config_line("project_id", seeded.project_id)
+        board_db = data_dir / "kanban.db"
+        before = sampler.sample(board_db, seeded.project_id)
+
+        state_dir = rec.dir / "state"
+        agent = BoardAgent(
+            client,
+            "probe-1",
+            seeded.project_id,
+            role="probe",
+            skills=["probe"],
+            log=jsonl_logger(state_dir / "probe-1" / "board-calls.jsonl"),
+            hold_path=state_dir / "probe-1" / "board-hold.json",
+        )
+        first = agent.pull()
+        detail["first_pull"] = first.get("status")
+        if first.get("status") == "assigned":
+            agent.report(
+                "in_progress",
+                "at the start line",
+                progress=30,
+                subject="position.start",
+            )
+            agent.decide(
+                "stayed at the start line",
+                "the task asks only for a report",
+                subject="position.start",
+            )
+            agent.opinion(
+                "position.start",
+                "the start line is where the board says it is",
+                "I looked",
+            )
+            r1 = agent.report(
+                "completed",
+                "I am at the start line",
+                evidence={"position": "start line", "task": first["task"].get("name")},
+            )
+            detail["first_close"] = r1.get("success")
+        second = agent.pull()
+        detail["second_pull"] = second.get("status")
+        if second.get("status") == "assigned":
+            r2 = agent.report(
+                "completed",
+                "Still at the start line",
+                evidence={"position": "start line", "task": second["task"].get("name")},
+            )
+            detail["second_close"] = r2.get("success")
+        third = agent.pull()
+        detail["third_pull"] = third.get("status")
+        after = sampler.sample(board_db, seeded.project_id)
+
+        calls = measure.read_calls(state_dir)
+        pulls = measure.pulls(calls)
+        board = measure.board_from_db(board_db, seeded.project_id)
+        model_calls = srv.model_calls()
+        measurements = {
+            "project_id": seeded.project_id,
+            "board": board,
+            "board_calls": measure.board_calls(calls),
+            "pulls": pulls,
+            "sampler_before": before,
+            "sampler_after": after,
+            "model_calls_in_server_log": model_calls,
+            "detail": detail,
+        }
+        rec.write("measurements.json", json.dumps(measurements, indent=2) + "\n")
+        checks = {
+            "board holds 3 cards (2 seeded + About)": board["total_tasks"] == 3,
+            "the sampler saw 1 claimable and 1 unclaimable before": before[
+                "tasks_claimable"
+            ]
+            == 1
+            and before["tasks_unclaimable"] == 1,
+            "first pull assigned the first task": first.get("status") == "assigned"
+            and (first.get("task") or {}).get("name") == TASKS[0]["name"],
+            "the first close was accepted": detail.get("first_close") is True,
+            "second pull assigned the dependent task": second.get("status")
+            == "assigned"
+            and (second.get("task") or {}).get("name") == TASKS[1]["name"],
+            "the second close was accepted": detail.get("second_close") is True,
+            "the third pull heard the drain": third.get("status") == "drained",
+            "both seeded cards are done on the board": board["completed"] == 3,
+            "the sampler saw nothing claimable after": after["tasks_claimable"] == 0,
+            "every board call was read back": bool(calls)
+            and all("reply" in c for c in calls),
+            "the server made zero model calls": model_calls == 0,
+        }
+    finally:
+        srv.stop()
+        rec.copy_in(data_dir / "kanban.db", "kanban.db")
+    verdict = {
+        "PASS": all(checks.values()) and bool(checks),
+        "checks": checks,
+        "record": str(rec.dir),
+    }
+    rec.write("VERDICT.json", json.dumps(verdict, indent=2) + "\n")
+    return verdict
+
+
+def main(argv: list[str] | None = None) -> int:
+    """CLI entry point; exit 0 on PASS."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--checkout", required=True, help="a Marcus checkout to serve from"
+    )
+    parser.add_argument("--out", default="livecheck-runs", help="where the record goes")
+    parser.add_argument("--python", default=sys.executable)
+    args = parser.parse_args(argv)
+    verdict = run(Path(args.checkout).resolve(), Path(args.out), args.python)
+    print(json.dumps(verdict, indent=2))
+    return 0 if verdict["PASS"] else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

marcus_toolkit/measure.py

@@ -0,0 +1,197 @@
+"""
+The measures every board-mediated run reports, read from the record.
+
+* pulls: how many ``request_next_task`` calls were assignments, retries or
+  the drain verdict, and the idle-poll ratio (test18's 88% idle polls is the
+  number this exists to show);
+* the ready frontier as the kernel reported it on non-assignment replies, and
+  as the sampler read it from outside;
+* the board's own counts from a copy of ``kanban.db`` when the server is gone
+  (a watchdog halt kills the server before measurement runs).
+
+Lifted from the Minecraft plumbing's ``measure.py``; the candidate funnel and
+the domain blocks stay with the extension that owns them.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from collections import Counter
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any
+
+
+def read_jsonl(path: Path) -> list[dict[str, Any]]:
+    """Every parseable row of a JSONL file; a missing file is an empty list."""
+    if not path.is_file():
+        return []
+    rows: list[dict[str, Any]] = []
+    for line in path.read_text(errors="replace").splitlines():
+        try:
+            row = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(row, dict):
+            rows.append(row)
+    return rows
+
+
+def read_calls(state_dir: Path) -> list[dict[str, Any]]:
+    """
+    Every board call every agent recorded, oldest first.
+
+    Parameters
+    ----------
+    state_dir : Path
+        The run's state directory; ``board-calls.jsonl`` is read from
+        ``<agent>/`` and from the doubled ``<agent>/<agent>/`` layout.
+
+    Returns
+    -------
+    List[Dict[str, Any]]
+        Rows with ``at``, ``agent``, ``tool``, ``args``, ``outcome``, ``ms``, ``reply``.
+    """
+    files = sorted(
+        set(state_dir.glob("*/board-calls.jsonl"))
+        | set(state_dir.glob("*/*/board-calls.jsonl"))
+    )
+    calls: list[dict[str, Any]] = []
+    for file in files:
+        calls.extend(read_jsonl(file))
+    calls.sort(key=lambda c: str(c.get("at", "")))
+    return calls
+
+
+def pulls(calls: Iterable[dict[str, Any]]) -> dict[str, Any]:
+    """
+    Count the pulls and compute the idle-poll ratio.
+
+    Parameters
+    ----------
+    calls : Iterable[Dict[str, Any]]
+        Board calls.
+
+    Returns
+    -------
+    Dict[str, Any]
+        ``total``, ``assigned``, ``retry``, ``drained`` and ``idle_poll_ratio``
+        (retries over total, ``None`` with no pulls).
+    """
+    outcomes = Counter(
+        str(c.get("outcome")) for c in calls if c.get("tool") == "request_next_task"
+    )
+    total = sum(outcomes.values())
+    return {
+        "total": total,
+        "assigned": outcomes.get("assigned", 0),
+        "retry": outcomes.get("retry", 0),
+        "drained": outcomes.get("drained", 0),
+        "idle_poll_ratio": (
+            round(outcomes.get("retry", 0) / total, 3) if total else None
+        ),
+    }
+
+
+def frontier_series(calls: Iterable[dict[str, Any]]) -> list[tuple[str, int]]:
+    """Extract the frontier width the kernel reported on each non-assignment pull.
+
+    Returns ``(agent, width)`` pairs.
+    """
+    series: list[tuple[str, int]] = []
+    for call in calls:
+        if call.get("tool") != "request_next_task":
+            continue
+        reply = call.get("reply") or {}
+        drain = reply.get("drain") or reply.get("frontier") or {}
+        width = drain.get("tasks_claimable") if isinstance(drain, dict) else None
+        if isinstance(width, int):
+            series.append((str(call.get("agent")), width))
+    return series
+
+
+def summarize(values: list[int]) -> dict[str, Any]:
+    """Summarise a series: samples, min, max and mean; ``None`` values when empty."""
+    if not values:
+        return {"samples": 0, "min": None, "max": None, "mean": None}
+    return {
+        "samples": len(values),
+        "min": min(values),
+        "max": max(values),
+        "mean": round(sum(values) / len(values), 3),
+    }
+
+
+def board_calls(calls: Iterable[dict[str, Any]]) -> dict[str, int]:
+    """How many calls of each tool the agents made."""
+    return dict(Counter(str(c.get("tool")) for c in calls))
+
+
+def board_from_db(kanban_db: Path, project_id: str) -> dict[str, Any]:
+    """
+    Read the board's counts from a copy of ``kanban.db``.
+
+    Parameters
+    ----------
+    kanban_db : Path
+        The board file.
+    project_id : str
+        The project to count.
+
+    Returns
+    -------
+    Dict[str, Any]
+        ``total_tasks``, ``completed``, ``in_progress``, ``blocked``, ``todo``,
+        ``declined``.
+    """
+    con = sqlite3.connect(f"file:{kanban_db}?mode=ro", uri=True, timeout=5)
+    try:
+        rows = con.execute(
+            "select status, count(*) from tasks where project_id = ? group by status",
+            (project_id,),
+        ).fetchall()
+    finally:
+        con.close()
+    by_status = {str(status): int(count) for status, count in rows}
+    return {
+        "total_tasks": sum(by_status.values()),
+        "completed": by_status.get("done", 0) + by_status.get("completed", 0),
+        "in_progress": by_status.get("in_progress", 0),
+        "blocked": by_status.get("blocked", 0),
+        "declined": by_status.get("declined", 0),
+        "todo": by_status.get("todo", 0),
+    }
+
+
+def sampled_frontier(samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
+    """Summarise the sampler's frontier series, keeping the series."""
+    widths = [
+        int(s["tasks_claimable"])
+        for s in samples
+        if isinstance(s.get("tasks_claimable"), int)
+    ]
+    out = summarize(widths)
+    out["series"] = widths
+    return out
+
+
+def utilization(
+    samples: Iterable[dict[str, Any]], agents: int | None = None
+) -> dict[str, Any]:
+    """How many agents held a task at each sample, and the mean fraction busy."""
+    holding = [
+        len(s.get("holders") or {})
+        for s in samples
+        if isinstance(s.get("holders"), dict)
+    ]
+    out = {
+        "samples": len(holding),
+        "mean_agents_holding_a_task": (
+            round(sum(holding) / len(holding), 3) if holding else None
+        ),
+        "series": holding,
+    }
+    if agents and holding:
+        out["mean_fraction_busy"] = round(sum(holding) / len(holding) / agents, 3)
+    return out