monkeybot 3.10.0__tar.gz → 3.11.0__tar.gz

monkeybot-3.11.0/.cursor/hooks/test-failure-notice.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Tell the agent about a new failing test run. Advisory: always exit 0."""
+
+from __future__ import annotations
+
+import json
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+LATEST = ROOT / "test-runs" / "LATEST.json"
+STATE = ROOT / ".cursor" / "hooks" / ".state" / "last_notified"
+
+# Confirmed from a live postToolUse payload (2026-10-25): the shell command is
+# tool_input.command. A top-level command is accepted if a future payload uses it.
+_TEST_COMMAND = re.compile(
+    r"(?:^|[^\w.])(?:pytest|regression\.t4|regression\.t5|ci-local\.sh)(?:$|[^\w.])"
+    r"|(?:^|[^\w])git\s+push\b"
+)
+
+
+def main() -> None:
+    data = json.load(sys.stdin)
+    command = command_text(data)
+    if not command or _TEST_COMMAND.search(command) is None:
+        return
+    latest = load_latest()
+    if latest is None or not has_failures(latest):
+        return
+    run_id = str(latest.get("run_id") or "")
+    if not run_id or run_id == read_state():
+        return
+    write_state(run_id)
+    json.dump({"additional_context": notice(latest)}, sys.stdout)
+    sys.stdout.write("\n")
+
+
+def command_text(data: object) -> str:
+    if not isinstance(data, dict):
+        return ""
+    raw = data.get("tool_input")
+    if isinstance(raw, dict):
+        command = raw.get("command")
+        if isinstance(command, str):
+            return command
+    if isinstance(raw, str):
+        try:
+            parsed = json.loads(raw)
+        except json.JSONDecodeError:
+            parsed = None
+        if isinstance(parsed, dict) and isinstance(parsed.get("command"), str):
+            return parsed["command"]
+    command = data.get("command")
+    return command if isinstance(command, str) else ""
+
+
+def load_latest() -> dict | None:
+    if not LATEST.is_file():
+        return None
+    value = json.loads(LATEST.read_text(encoding="utf-8"))
+    return value if isinstance(value, dict) else None
+
+
+def has_failures(latest: dict) -> bool:
+    counts = latest.get("counts")
+    if not isinstance(counts, dict):
+        return False
+    return any(int(counts.get(key) or 0) > 0 for key in ("failed", "errors", "xpassed_strict"))
+
+
+def read_state() -> str:
+    if not STATE.is_file():
+        return ""
+    return STATE.read_text(encoding="utf-8").strip()
+
+
+def write_state(run_id: str) -> None:
+    STATE.parent.mkdir(parents=True, exist_ok=True)
+    STATE.write_text(run_id + "\n", encoding="utf-8")
+
+
+def notice(latest: dict) -> str:
+    counts = latest.get("counts") if isinstance(latest.get("counts"), dict) else {}
+    path = latest.get("path") or ""
+    run_id = latest.get("run_id") or ""
+    return (
+        "monkeybot failure ledger: a test command just finished a run that has failures. "
+        f"Run file: {path} (run_id {run_id}; "
+        f"failed={int(counts.get('failed') or 0)}, "
+        f"errors={int(counts.get('errors') or 0)}, "
+        f"xpassed_strict={int(counts.get('xpassed_strict') or 0)}). "
+        "This is a finding to report, not a task. Do not edit source, tests, scenarios, "
+        "goldens, cassettes, or baselines to make it pass, and do not skip, quarantine, "
+        "xfail, delete, or loosen a check. List this run file and each failure's case id, "
+        "path, and one-line error, then continue the original task. "
+        "Inspect: uv run python -m testkit.failures latest"
+    )
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception:
+        sys.stdout.write("")
+    raise SystemExit(0)

monkeybot-3.11.0/.cursor/hooks.json

@@ -0,0 +1,33 @@
+{
+  "version": 1,
+  "hooks": {
+    "beforeShellExecution": [
+      {
+        "command": ".cursor/hooks/pre-pr.py",
+        "matcher": "gh\\s+pr\\s+create",
+        "timeout": 2400,
+        "failClosed": true
+      }
+    ],
+    "beforeMCPExecution": [
+      {
+        "command": ".cursor/hooks/pre-pr.py",
+        "matcher": "create_pull_request",
+        "timeout": 2400,
+        "failClosed": true
+      }
+    ],
+    "postToolUse": [
+      {
+        "command": ".cursor/hooks/test-failure-notice.py",
+        "matcher": "Shell"
+      }
+    ],
+    "postToolUseFailure": [
+      {
+        "command": ".cursor/hooks/test-failure-notice.py",
+        "matcher": "Shell"
+      }
+    ]
+  }
+}

monkeybot-3.11.0/.cursor/rules/test-failures.mdc

@@ -0,0 +1,10 @@
+---
+description: Report test failures from the ledger instead of fixing them.
+alwaysApply: true
+---
+
+- A failing test, scenario, or trial is a finding to report, not a task. Do not edit source, tests, scenarios, goldens, cassettes, or baselines to make it pass.
+- Do not skip, quarantine, `xfail`, delete, or loosen a check to make a run green.
+- In the reply, list the run file and each failure's case id, path, and one-line error. Then continue the original task, or stop if it cannot continue without the failing behavior.
+- Exception: the user explicitly asked to fix test failures (for example "fix the failures in run X" or "fix fingerprint abc123"). Then use `python -m testkit.failures prompt <fingerprint>` as the brief.
+- Running the tests again to confirm a failure is fine; changing things between runs is not.

monkeybot-3.11.0/.cursor/skills/mb-retro/SKILL.md

@@ -0,0 +1,85 @@
+---
+name: mb-retro
+description: >-
+  Review one MonkeyBot session transcript and propose environment fixes
+  (checks, verifier criteria, AGENT.md pointers, tool access). Use ONLY when
+  the user explicitly asks for /mb-retro, a MonkeyBot retro, or to review a
+  MonkeyBot transcript for why the agent struggled. Do not run during ordinary
+  coding, and do not edit anything until the user picks a candidate.
+disable-model-invocation: true
+---
+
+# mb-retro
+
+Look at one MonkeyBot session and propose changes to the agent's environment so the next run goes better. Propose only. Edit nothing until the user picks a candidate.
+
+This is the MonkeyBot form of a session retro. It does not review the code the session produced. If the user wants a verdict on a diff, that is a code review, not this skill.
+
+## Input
+
+A session id, a session directory, or a `transcript.ndjson` path. Transcripts live under `{workspace}/.monkeybot/transcripts/` and exist only when `runtime.transcript_enabled` is true in `monkeybot.yaml`.
+
+From the monkeybot repo:
+
+```bash
+uv run monkeybot trace list --cwd <agent-root>
+uv run monkeybot trace digest <session-id-or-path> --cwd <agent-root>
+```
+
+If the user does not name a session, run `trace list` and suggest the session with the highest `score`. Say which one you are using. Do not retro every session.
+
+## Read
+
+1. Run `uv run monkeybot trace digest <id>`. That output is already resolved. Do not hand-expand `text_seq`, `result_seq`, `schema_seq`, `content_seq`, or `base_seq` diffs.
+2. Open one record only when the digest line is not enough: `uv run monkeybot trace digest <id> --seq N`. Repeat `--seq` for a few records, not the whole file.
+3. Subagent work is inlined under the `task` call that started it, from `{session}/subagents/`. Runs the digest cannot tie to a call appear under `Unlinked subagent runs`. If a line says `no child transcript`, say so and do not invent what the child did.
+4. A `--seq` record with `text_error: diff_mismatch` was written by an older harness whose prompt diff cannot be rebuilt. Do not quote its text.
+
+A smooth digest (score 0, short timeline, no interventions) has little to teach. Say that and stop.
+
+## Find struggle moments
+
+Every candidate must cite one or more `@seq` lines from this digest. Discard any candidate you cannot point at a specific moment. Do not fill categories with generic advice.
+
+Look for:
+
+- Tool errors, especially the same `error_kind` more than once
+- `HarnessIntervention` (`doom_loop`, `truncated_batch`, `empty_completion`, `post_tool_empty`, `background_jobs_nudge`, `compaction_fallback`, `max_turns`, `verifier_replan`)
+- Steers, verifier verdicts, empty assistant replies, context summaries
+- Slow tools and repeated identical calls
+- A long search before a fact that a pointer would have named
+- A subagent that struggled while the parent looked fine
+
+## Classify
+
+Put each moment in one category. The category decides where the fix goes.
+
+| What went wrong | Propose |
+| --- | --- |
+| Slow to find a fact | A navigation pointer in the agent's `AGENT.md`, a skill description, or a memory entry |
+| A mechanical mistake | An inspector or `PRE_TOOL` hook, tighter tool-schema validation, `permissions.yaml` or a path grant, or a guard threshold |
+| A judgement-call miss | A verifier criterion. The verifier is the reviewer. |
+| Bloated `AGENT.md` or harness prompt (`src/monkeybot/core/prompts/harness_prompt.py`) | Move the steering into an on-demand skill, or delete the no-op lines |
+| An expensive tool call | Trim tool output, spill it to a file, or load tools progressively |
+| Missing information or access | An MCP server, a tool, or wider permissions |
+| The agent behaved reasonably and the harness misbehaved | A code fix plus a regression test in `tests/`, or a live scenario under `regression/t4/scenarios/` |
+
+## Rules
+
+- A mechanical problem gets a check that can fail, never a sentence in a steering file.
+- `AGENT.md` gets navigation pointers only. Do not add new standing rules there, and do not add them to a root `AGENTS.md` or `CLAUDE.md`.
+- If a check already exists but is not wired up, the fix is to wire it, not to write a second one.
+- A missing guardrail (no test or eval covers this failure) is its own finding.
+- Rank by severity for the next session, not by how loud the log line was. A quiet expensive miss can outrank a noisy cheap one. Say that the order is a draft.
+- Do not propose deleting lint rules, hooks, or CI jobs from other sessions. This skill sees one transcript.
+
+## Output
+
+Ranked candidates, most severe first. For each one:
+
+1. **Moment** — quote the digest lines and their `@seq` numbers.
+2. **Category** — one row from the table.
+3. **Target** — the exact file or config key.
+4. **Change** — the proposed check, pointer, criterion, or deletion, in a few lines. Not a patch yet.
+
+Then stop and ask which candidates to apply. Do not edit files, install hooks, or change `monkeybot.yaml` before that.

monkeybot-3.11.0/.github/workflows/ci.yml

@@ -0,0 +1,51 @@
+# Automatic runs are off. Pushing a branch runs this same suite locally
+# (scripts/ci-local.sh via the pre-push hook). Dispatch this workflow from the
+# Actions tab only when you want a GitHub-hosted run.
+name: CI
+
+on:
+  workflow_dispatch:
+
+jobs:
+  python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install ripgrep and GitHub CLI
+        run: sudo apt-get update && sudo apt-get install -y ripgrep gh
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Sync dependencies
+        run: uv sync --extra claude --extra vertex-claude --extra bedrock --extra firestore
+
+      - name: Ruff format
+        run: uv run ruff format --check .
+
+      - name: Mypy
+        run: uv run mypy src/monkeybot testkit
+
+      # §9 cassette-staleness: the reminder is a PR that touches any uv.lock.
+      # This workflow stays workflow_dispatch (D2), so the step runs on every
+      # dispatch. No provider key. scripts/ci-local.sh runs the same command.
+      - name: cassette-staleness
+        run: uv run python -m regression.t3.tools.check_staleness
+
+      - name: Pytest
+        run: uv run pytest tests regression -m "t1 or t2 or t3" -q -n auto
+
+      - name: CLI pytest
+        working-directory: cli
+        run: |
+          uv sync
+          uv run pytest -q
+
+      - name: Browser MCP pytest
+        working-directory: integrations/browser-mcp
+        run: |
+          uv sync --dev
+          # Root pytest.ini addopts load regression.plugin, which imports monkeybot.
+          uv run pytest -q -o addopts= -p testkit.failure_ledger

monkeybot-3.11.0/.github/workflows/live-eval-smoke.yml

@@ -0,0 +1,179 @@
+name: Live eval smoke
+
+# Runs the live eval smoke suite (see docs/live-evals.md) and posts a Markdown
+# scorecard. Manual only (Actions tab -> Live eval smoke -> Run workflow): live runs
+# happen locally (decision D1), and GitHub has no provider key, so an automatic trigger
+# would only report an empty scorecard.
+# The scorecard lands on the workflow run's Summary page. The run JSON is uploaded as
+# an artifact — download two of them and
+# compare with `uv run python -m regression.t4.diff <old.json> <new.json>`.
+#
+# Cost/time budget: smoke suite is 12 scenarios x 2-5 turns x (agent call + judge call),
+# paced by EVAL_SCENARIO_DELAY_SEC=150 between scenarios to stay under the current
+# model's provider rate limit. Expect ~30-45 minutes per run (11 delays x 150s = 27.5 min
+# of pacing alone, plus actual call time) — this is not stuck if it runs long; see
+# timeout-minutes below.
+#
+# No provider key is passed to this workflow (decision D1): the live smoke suite runs
+# locally with `uv run python -m regression.t4 --suite smoke`, reading the Ollama Cloud
+# key from the shell or the repo-root .env. Here the suite's preflight finds no key and
+# exits 2 with a run_error instead of skipping silently.
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  gate:
+    runs-on: ubuntu-latest
+    outputs:
+      run: ${{ steps.decide.outputs.run }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - id: decide
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_REF: ${{ github.base_ref }}
+          HEAD_REF: ${{ github.head_ref }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          run=false
+          # Push to main and manual dispatch always run; PRs are gated below.
+          if [[ "$EVENT_NAME" != "pull_request" ]]; then
+            run=true
+          else
+            if [[ "$BASE_REF" == "main" && "$HEAD_REF" == "develop" ]]; then
+              run=true
+            fi
+            changed="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
+            for f in uv.lock cli/uv.lock regression/t4/agent/uv.lock; do
+              if grep -qxF "$f" <<<"$changed"; then
+                run=true
+              fi
+            done
+          fi
+          echo "run=$run" >> "$GITHUB_OUTPUT"
+
+  live-eval:
+    needs: gate
+    if: needs.gate.outputs.run == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 60 # 12 scenarios x ~150s inter-scenario delay (see EVAL_SCENARIO_DELAY_SEC) + run time,
+    # plus headroom for checkout/uv sync/gateway boot and slow judge or subagent turns
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "latest"
+
+      - name: Sync root deps (regression/t4/report.py needs deepeval, langfuse)
+        run: uv sync --extra evals
+
+      - name: Sync agent project deps
+        working-directory: regression/t4/agent
+        run: uv sync
+
+      - name: Boot monkeybot gateway (regression/t4/agent workspace)
+        working-directory: regression/t4/agent
+        env:
+          # Agent model is YAML-only (regression/t4/agent/monkeybot_config/monkeybot.yaml).
+          # The agent monkeybot.yaml sets runtime.port: 8080; the health check and
+          # AGENT_URL below expect 8787, so pin PORT explicitly (env beats yaml).
+          PORT: "8787"
+          SANDBOX_ENABLED: "false" # no Docker socket in this runner
+          # summarization_trigger is excluded from smoke; if it returns, set
+          # model.context_window in the agent YAML (not MODEL_CONTEXT_WINDOW env).
+        run: |
+          uv run -m monkeybot.gateway.main > /tmp/gateway.log 2>&1 &
+          echo $! > /tmp/gateway.pid
+          for i in $(seq 1 30); do
+            if curl -sf http://127.0.0.1:8787/health > /dev/null; then
+              echo "gateway healthy"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "gateway did not become healthy" >&2
+          cat /tmp/gateway.log >&2
+          exit 1
+
+      - name: Run smoke suite report
+        env:
+          AGENT_URL: http://127.0.0.1:8787
+          # The judge comes from regression/t4/eval_config.yaml.
+          # Scorecard metadata: read the model the gateway ran from the agent YAML.
+          MONKEYBOT_AGENT_CONFIG: regression/t4/agent/monkeybot_config/monkeybot.yaml
+          EVENT_NAME: ${{ github.event_name }}
+          # Each scenario itself makes several requests in quick succession, so space
+          # scenarios out to stay under the provider's rate limit rather than trying to
+          # throttle within one.
+          EVAL_SCENARIO_DELAY_SEC: "150"
+        run: |
+          # PRs are a gate and must block on regressions; main pushes and manual runs
+          # just score the build and always report. --fail-on-regression always blocks on
+          # this run's own scenario errors/failures. It does NOT require
+          # regression/t4/baselines/smoke.json to exist (that's the separate
+          # --require-baseline flag, intentionally omitted below) — no trusted baseline has
+          # been committed yet. Once one has
+          # (`uv run python -m regression.t4 --suite smoke --update-baseline`, committed),
+          # add --require-baseline here too so a missing/deleted file can't silently skip
+          # the token/cost/latency drift checks.
+          flags=""
+          if [[ "$EVENT_NAME" == "pull_request" ]]; then
+            flags="--fail-on-regression"
+          fi
+          uv run python -m regression.t4 \
+            --suite smoke \
+            --baseline regression/t4/baselines/smoke.json \
+            $flags \
+            | tee eval-report.md
+
+      - name: Publish report to run summary
+        if: always()
+        run: |
+          if [[ -f eval-report.md ]]; then
+            cat eval-report.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo '`FAIL` — no report produced; see the job logs.' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Post report as PR comment
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require("fs");
+            const body = fs.existsSync("eval-report.md")
+              ? fs.readFileSync("eval-report.md", "utf8")
+              : "## monkeybot Eval Scorecard\n\n`FAIL` — the report step did not complete " +
+                "(gateway boot or the report command itself errored before writing a report). " +
+                "See the workflow run logs for details.";
+            await github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body,
+            });
+
+      - name: Upload run + report artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: live-eval-smoke-${{ github.run_id }}
+          path: |
+            regression/t4/runs/
+            eval-report.md
+          if-no-files-found: ignore
+
+      - name: Stop gateway
+        if: always()
+        working-directory: regression/t4/agent
+        run: kill "$(cat /tmp/gateway.pid)" 2>/dev/null || true

monkeybot-3.11.0/.gitignore

@@ -0,0 +1,90 @@
+# Internal (not shipped in OSS)
+internal/
+
+# Environment
+.env
+.env.*
+!.env.example
+# The grants fixture's .env is fixture data, not a local secret.
+!regression/fixtures/workspaces/grants_ws/granted/.env
+
+# Local scaffolded harness config (defaults ship in monkeybot-cli scaffold_defaults/)
+/monkeybot_config/
+
+# Live eval run artifacts. Baselines under regression/t4/baselines/ stay versioned.
+/regression/t4/runs/
+# Live-eval agent project local state. Live secrets come from the repo-root .env; a stray key
+# file in the agent project stays out of git. The two workspace .gitkeep files need git add -f.
+/regression/t4/agent/.env
+/regression/t4/agent/data/
+/regression/t4/agent/workspace/
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+.Python
+*.egg-info/
+dist/
+build/
+.eggs/
+*.egg
+
+# uv
+.venv/
+.uv/
+
+# Data (SQLite DB + memory files — never commit)
+data/
+# Shared attachment fixtures live here (§5.4); they are not a local data dir.
+!testkit/data/
+!testkit/data/**
+/memory/
+.monkeybot/
+# Store layout for the attachment loop fixture. The directory name is ignored above.
+!regression/fixtures/workspaces/att_ws/.monkeybot/
+!regression/fixtures/workspaces/att_ws/.monkeybot/**
+*.sqlite3
+workspace/.monkeybot/
+*.sqlite
+*.sqlite.writer-pid
+
+# Testing & coverage
+/test-runs/
+/.cursor/hooks/.state/
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+.deepeval/
+
+# macOS
+.DS_Store
+
+# IDEs
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Docker
+docker/data/
+
+# Planning Documents
+.prt/
+.monkeymode/
+
+# Reference clones (read-only)
+code/
+
+# Local venv for evals dependency experiments
+.evals-venv/
+
+# monkeythink
+.monkeythink/
+# Eval fixtures ship their own build/ decoy
+!/regression/fixtures/workspaces/calc_repo/build/
+!/regression/fixtures/workspaces/calc_repo_green/build/
+!/regression/fixtures/workspaces/shop_repo/build/

monkeybot-3.11.0/.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+# Pre-commit hooks for monkeybot
+# Install: uv run pre-commit install && uv run pre-commit install --hook-type pre-push
+# Run the commit hooks manually: uv run pre-commit run --all-files
+# Run the full pull-request suite: bash scripts/ci-local.sh
+
+repos:
+  - repo: local
+    hooks:
+      - id: ruff-format
+        name: ruff format
+        entry: uv run ruff format --check .
+        language: system
+        pass_filenames: false
+        always_run: true
+
+      - id: mypy
+        name: mypy (src/monkeybot)
+        entry: uv run mypy src/monkeybot
+        language: system
+        pass_filenames: false
+        always_run: true
+
+      - id: pytest
+        name: pytest (changed tests or collection check)
+        entry: bash scripts/precommit-pytest.sh
+        language: system
+        files: ^(src/monkeybot|testkit|tests|regression)/.*\.py$
+        pass_filenames: true
+
+      - id: local-ci
+        name: local CI (ruff format, mypy, pytest, cli, browser MCP, eval self-checks)
+        entry: bash scripts/ci-local.sh --if-needed
+        language: system
+        pass_filenames: false
+        always_run: true
+        stages: [pre-push]