willow-bot 0.1.1__py3-none-any.whl

willow_bot/__init__.py

@@ -0,0 +1,3 @@
+"""willow-bot — GitHub App webhook + orchestrator-feeding steward."""
+
+__version__ = "0.1.1"

willow_bot/cli.py

@@ -0,0 +1,21 @@
+"""Console entrypoints: willow-bot (webhook) and willow-bot-steward."""
+from __future__ import annotations
+
+import os
+import sys
+
+
+def webhook_main() -> None:
+    """Run the FastAPI webhook via uvicorn (reads BOT_HOST / BOT_PORT)."""
+    import uvicorn
+
+    host = os.environ.get("BOT_HOST", "127.0.0.1")
+    port = int(os.environ.get("BOT_PORT", "9000"))
+    # Root-level bot:app kept for systemd compatibility during prove.
+    uvicorn.run("bot:app", host=host, port=port, factory=False)
+
+
+def steward_main(argv: list[str] | None = None) -> int:
+    from willow_bot.steward.tick import main
+
+    return main(argv if argv is not None else sys.argv[1:])

willow_bot/deposits.py

@@ -0,0 +1,137 @@
+"""Propose-only CI / PR deposits (§12 middle row).
+
+Always append a local draft JSONL under ``$WILLOW_HOME/willow-bot/deposits/``.
+Optionally mirror into willow-mcp ``store_put`` when MCP is configured — never seals.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+log = logging.getLogger("willow-bot.deposits")
+
+COLLECTION = "willow_bot_ci_deposits"
+
+
+def _willow_home() -> Path:
+    default = Path.home() / "sean-data-vault" / "willow-operator-box"
+    return Path(os.environ.get("WILLOW_HOME", default))
+
+
+def deposits_dir() -> Path:
+    return _willow_home() / "willow-bot" / "deposits"
+
+
+def deposits_jsonl() -> Path:
+    return deposits_dir() / "ci_outcomes.jsonl"
+
+
+def _now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def ci_outcome_record(
+    *,
+    repo: str,
+    head_sha: str | None,
+    check_run_id: Any,
+    check_name: str | None,
+    conclusion: str | None,
+    status: str | None = None,
+    pr_number: int | None = None,
+    html_url: str | None = None,
+    sender_type: str = "",
+    received_at: str | None = None,
+) -> dict[str, Any]:
+    """Draft claim: how CI went for repo@sha (pass and fail both recorded)."""
+    return {
+        "kind": "ci_outcome",
+        "lane": "draft",
+        "deposited_by": "willows-bot",
+        "actor_type": sender_type or "unknown",
+        "repo": repo,
+        "head_sha": head_sha or "",
+        "check_run_id": check_run_id,
+        "check_name": check_name or "",
+        "conclusion": conclusion or "",
+        "status": status or "",
+        "pr_number": pr_number,
+        "html_url": html_url or "",
+        "received_at": received_at or _now(),
+        "deposited_at": _now(),
+    }
+
+
+def record_id_for(rec: dict[str, Any]) -> str:
+    repo = (rec.get("repo") or "unknown").replace("/", "__")
+    sha = (rec.get("head_sha") or "nosha")[:12]
+    cid = rec.get("check_run_id") or "0"
+    return f"ci-{repo}-{sha}-{cid}"
+
+
+def append_local(rec: dict[str, Any]) -> Path:
+    path = deposits_jsonl()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(rec, separators=(",", ":")) + "\n")
+    return path
+
+
+def deposit_ci_outcome(rec: dict[str, Any]) -> dict[str, Any]:
+    """Write local draft; optionally MCP store_put. Returns status receipt."""
+    path = append_local(rec)
+    out: dict[str, Any] = {
+        "local": str(path),
+        "record_id": record_id_for(rec),
+        "mcp": {"status": "skipped", "detail": "WILLOW_BOT_MCP not enabled"},
+    }
+    if os.environ.get("WILLOW_BOT_MCP", "").strip().lower() not in ("1", "true", "yes"):
+        return out
+    try:
+        from willow_bot.steward import mcp_client
+
+        app_id = os.environ.get("WILLOW_BOT_MCP_APP_ID", "willow").strip() or "willow"
+        result = mcp_client.call(
+            "store_put",
+            {
+                "app_id": app_id,
+                "collection": COLLECTION,
+                "record": rec,
+                "record_id": record_id_for(rec),
+                "deviation": 0,
+            },
+        )
+        out["mcp"] = {"status": "ok", "result": result}
+    except Exception as exc:  # noqa: BLE001 — honest absence, never silent success
+        log.warning("mcp store_put failed: %s", exc)
+        out["mcp"] = {"status": "could-not-run", "detail": str(exc)}
+    return out
+
+
+def deposit_from_check_run_payload(payload: dict) -> dict[str, Any] | None:
+    """Build + deposit from a completed check_run webhook payload."""
+    if payload.get("action") != "completed":
+        return None
+    check = payload.get("check_run") or {}
+    repo = (payload.get("repository") or {}).get("full_name") or ""
+    if not repo:
+        return None
+    prs = check.get("pull_requests") or []
+    pr_number = prs[0].get("number") if prs else None
+    sender_type = str((payload.get("sender") or {}).get("type") or "")
+    rec = ci_outcome_record(
+        repo=repo,
+        head_sha=check.get("head_sha"),
+        check_run_id=check.get("id"),
+        check_name=check.get("name"),
+        conclusion=check.get("conclusion"),
+        status=check.get("status"),
+        pr_number=pr_number,
+        html_url=check.get("html_url"),
+        sender_type=sender_type,
+    )
+    return deposit_ci_outcome(rec)

willow_bot/steward/__init__.py

@@ -0,0 +1 @@
+"""Deterministic PR/CI steward (never Grove 3B Loki)."""

willow_bot/steward/config.py

@@ -0,0 +1,45 @@
+"""Shared env for steward PR watch (deterministic gh/CI — never Grove 3B watcher)."""
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def _truthy(name: str, default: str = "0") -> bool:
+    return os.environ.get(name, default).strip().lower() in ("1", "true", "yes")
+
+
+def willow_home() -> Path:
+    default = Path.home() / "sean-data-vault" / "willow-operator-box"
+    return Path(os.environ.get("WILLOW_HOME", default))
+
+
+def webhook_inbox_dir() -> Path:
+    return willow_home() / "upstream_steward" / "webhook_inbox"
+
+
+def state_path() -> Path:
+    vault = os.environ.get("WILLOW_VAULT_BOX", str(willow_home()))
+    return Path(
+        os.environ.get(
+            "WILLOW_BOT_STEWARD_STATE",
+            os.environ.get("LOKI_PR_WATCH_STATE", f"{vault}/loki_pr_watch_state.json"),
+        )
+    )
+
+
+def watcher_call_enabled() -> bool:
+    """PR watch must not call Ollama/Cerebras. Opt-in is reserved and currently rejected."""
+    return _truthy("WILLOW_BOT_STEWARD_CALL_WATCHER") or _truthy("LOKI_PR_WATCH_CALL_WATCHER")
+
+
+def watcher_url() -> str:
+    """Documented for willow-bot loki.watcher only; steward does not HTTP here."""
+    return os.environ.get("LOKI_WATCHER_URL", "http://localhost:11434/v1").strip()
+
+
+def host_sync_enabled() -> bool:
+    """Optional merge→pull/pip -e. Default on (legacy loki_pr_watch behaviour)."""
+    if "WILLOW_BOT_STEWARD_HOST_SYNC" in os.environ:
+        return _truthy("WILLOW_BOT_STEWARD_HOST_SYNC", "1")
+    return True

willow_bot/steward/fleet.py

@@ -0,0 +1,120 @@
+"""Fleet scope for Loki PR watch — all operator GitHub orgs (+ optional user account)."""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from functools import lru_cache
+from typing import Iterator
+
+# Used only when ``gh api /user/memberships/orgs`` is unavailable.
+_DEFAULT_ORGS = (
+    "almanac-data",
+    "Die-Namic-Systems",
+    "forge-play",
+    "homestead-affairs",
+    "hornbook-knowledge",
+    "terpsi-programs",
+    "willow-memory",
+)
+
+DEFAULT_USER = "rudi193-cmd"
+
+
+def gh_api(path: str):
+    return json.loads(subprocess.check_output(["gh", "api", path], text=True))
+
+
+@lru_cache(maxsize=1)
+def fleet_orgs() -> tuple[str, ...]:
+    """Org logins to scan. Override with LOKI_PR_WATCH_ORGS (comma-separated)."""
+    raw = os.environ.get("LOKI_PR_WATCH_ORGS", "").strip()
+    if raw:
+        return tuple(o.strip() for o in raw.split(",") if o.strip())
+    try:
+        members = gh_api("/user/memberships/orgs?per_page=100")
+        logins = [m["organization"]["login"] for m in members if m.get("organization")]
+        if logins:
+            return tuple(sorted(logins))
+    except (subprocess.CalledProcessError, KeyError, TypeError, json.JSONDecodeError):
+        pass
+    return _DEFAULT_ORGS
+
+
+def fleet_user() -> str:
+    return os.environ.get("LOKI_PR_WATCH_USER", DEFAULT_USER).strip() or DEFAULT_USER
+
+
+def repos_for_org(org: str) -> list[str]:
+    names: list[str] = []
+    page = 1
+    while True:
+        try:
+            batch = gh_api(f"/orgs/{org}/repos?per_page=100&page={page}")
+        except subprocess.CalledProcessError:
+            break
+        if not batch:
+            break
+        names.extend(r["full_name"] for r in batch)
+        if len(batch) < 100:
+            break
+        page += 1
+    return names
+
+
+def repos_for_user(user: str) -> list[str]:
+    names: list[str] = []
+    page = 1
+    while True:
+        try:
+            batch = gh_api(
+                f"/users/{user}/repos?per_page=100&page={page}&type=owner"
+            )
+        except subprocess.CalledProcessError:
+            break
+        if not batch:
+            break
+        names.extend(r["full_name"] for r in batch)
+        if len(batch) < 100:
+            break
+        page += 1
+    return names
+
+
+def fleet_repos() -> list[str]:
+    names: set[str] = set()
+    for org in fleet_orgs():
+        names.update(repos_for_org(org))
+    if os.environ.get("LOKI_PR_WATCH_SKIP_USER", "").strip() not in ("1", "true", "yes"):
+        names.update(repos_for_user(fleet_user()))
+    return sorted(names)
+
+
+def iter_open_pulls() -> Iterator[tuple[str, int, str, str]]:
+    """Yield (full_name, number, title, html_url) for every open PR in fleet scope."""
+    for repo in fleet_repos():
+        try:
+            pulls = gh_api(f"repos/{repo}/pulls?state=open&per_page=50")
+        except subprocess.CalledProcessError:
+            continue
+        for p in pulls:
+            yield (
+                repo,
+                int(p["number"]),
+                str(p.get("title") or ""),
+                str(p.get("html_url") or ""),
+            )
+
+
+def repo_matches_filters(repo: str, num: int, filters: list[str]) -> bool:
+    if not filters:
+        return True
+    key = f"{repo}#{num}"
+    for f in filters:
+        if f == key or f in key:
+            return True
+        if "/" not in f and repo.startswith(f"{f}/"):
+            return True
+        if f == repo or repo.startswith(f):
+            return True
+    return False

willow_bot/steward/heartbeat.py

@@ -0,0 +1,100 @@
+"""Curated willow-mcp tool heartbeat for the steward (prove phase).
+
+Calls platform verbs over MCP; does NOT own commitment dew (Kart
+``CommitmentProactiveHook`` stays until proven). Honest absence when MCP off.
+"""
+from __future__ import annotations
+
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from willow_bot.steward.config import willow_home
+
+
+# Read-only / status verbs — no dew publish, no seal.
+DEFAULT_CURATED: list[tuple[str, dict[str, Any]]] = [
+    ("fleet_health", {}),
+    ("commitment_surface", {}),
+    ("human_required_list", {}),
+    ("diagnostic_summary", {}),
+]
+
+
+def _app_id() -> str:
+    return os.environ.get("WILLOW_BOT_MCP_APP_ID", "willow").strip() or "willow"
+
+
+def _receipts_path() -> Path:
+    return willow_home() / "willow-bot" / "steward_heartbeat.jsonl"
+
+
+def curated_calls() -> list[tuple[str, dict[str, Any]]]:
+    raw = os.environ.get("WILLOW_BOT_STEWARD_TOOLS", "").strip()
+    if not raw:
+        app = _app_id()
+        return [(name, {**args, "app_id": app}) for name, args in DEFAULT_CURATED]
+    # comma-separated tool names; each gets app_id only
+    app = _app_id()
+    return [(n.strip(), {"app_id": app}) for n in raw.split(",") if n.strip()]
+
+
+def run_heartbeat(*, enable_mcp: bool | None = None) -> dict[str, Any]:
+    """Run one curated pass. Returns a receipt dict (also appended to JSONL)."""
+    if enable_mcp is None:
+        enable_mcp = os.environ.get("WILLOW_BOT_MCP", "").strip().lower() in (
+            "1",
+            "true",
+            "yes",
+        )
+
+    receipt: dict[str, Any] = {
+        "event": "steward_heartbeat",
+        "at": datetime.now(timezone.utc).isoformat(),
+        "tools": [],
+    }
+
+    if not enable_mcp:
+        receipt["status"] = "absent"
+        receipt["detail"] = "WILLOW_BOT_MCP not enabled — no tool calls"
+        _append(receipt)
+        print(json.dumps(receipt), flush=True)
+        return receipt
+
+    try:
+        from willow_bot.steward import mcp_client
+    except ImportError as exc:
+        receipt["status"] = "could-not-run"
+        receipt["detail"] = f"mcp package missing: {exc}"
+        _append(receipt)
+        print(json.dumps(receipt), flush=True)
+        return receipt
+
+    receipt["status"] = "ok"
+    for name, args in curated_calls():
+        entry: dict[str, Any] = {"tool": name, "args_keys": sorted(args)}
+        try:
+            result = mcp_client.call(name, args)
+            entry["outcome"] = "ok"
+            # Keep receipt small — summarize
+            if isinstance(result, dict):
+                entry["result_keys"] = sorted(result.keys())[:20]
+            else:
+                entry["result_preview"] = str(result)[:240]
+        except Exception as exc:  # noqa: BLE001
+            entry["outcome"] = "could-not-run"
+            entry["detail"] = str(exc)[:400]
+        receipt["tools"].append(entry)
+
+    _append(receipt)
+    print(json.dumps(receipt), flush=True)
+    return receipt
+
+
+def _append(receipt: dict[str, Any]) -> None:
+    path = _receipts_path()
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(receipt, separators=(",", ":")) + "\n")

willow_bot/steward/inbox.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Merge willow-bot fleet_bridge PR webhook inbox into loki_pr_watch state.
+
+Reads ~/.willow/upstream_steward/webhook_inbox/*.json (WILLOW_HOME), records
+consumed work_ids, emits one JSON line per newly seen pull_request event.
+
+PR watch never calls LOKI_WATCHER_URL / Ollama; Grove Loki (3B) is separate.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+from willow_bot.steward.config import (
+    watcher_call_enabled,
+    watcher_url,
+    webhook_inbox_dir,
+)
+
+_MAX_CONSUMED = 5000
+_MAX_SIGNALS = 100
+
+
+def _load_state(path: Path) -> dict:
+    if path.is_file() and path.read_text().strip():
+        return json.loads(path.read_text())
+    return {"seen": [], "open": [], "merged_synced": []}
+
+
+def _emit(obj: dict) -> None:
+    print(json.dumps(obj), flush=True)
+
+
+def _repo_pr(item: dict) -> str:
+    repo = item.get("repo") or ""
+    num = item.get("number")
+    if repo and num is not None:
+        return f"{repo}#{num}"
+    return ""
+
+
+def ingest(state_path: Path) -> int:
+    if watcher_call_enabled():
+        _emit(
+            {
+                "event": "error",
+                "detail": (
+                    "WILLOW_BOT_STEWARD_CALL_WATCHER / LOKI_PR_WATCH_CALL_WATCHER is set but not "
+                    "supported; willow-bot runs loki.watcher separately "
+                    f"(LOKI_WATCHER_URL={watcher_url()!r} is ignored here)."
+                ),
+            }
+        )
+        return 1
+
+    inbox = webhook_inbox_dir()
+    state = _load_state(state_path)
+    state.setdefault("seen", [])
+    state.setdefault("open", [])
+    state.setdefault("merged_synced", [])
+    consumed: set[str] = set(state.get("inbox_consumed") or [])
+    signals: list[dict] = list(state.get("webhook_signals") or [])
+
+    if not inbox.is_dir():
+        state_path.parent.mkdir(parents=True, exist_ok=True)
+        state["inbox_consumed"] = sorted(consumed)
+        state["webhook_signals"] = signals[-_MAX_SIGNALS:]
+        state_path.write_text(json.dumps(state, indent=2) + "\n")
+        return 0
+
+    for path in sorted(inbox.glob("*.json")):
+        try:
+            item = json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            continue
+        wid = item.get("work_id")
+        if not wid or wid in consumed:
+            continue
+        if item.get("kind") != "pull_request":
+            continue
+
+        consumed.add(wid)
+        key = _repo_pr(item)
+        if not key:
+            continue
+
+        ev = {
+            "event": "webhook_pr",
+            "repo_pr": key,
+            "action": item.get("action"),
+            "title": item.get("title") or "",
+            "url": item.get("html_url") or "",
+            "received_at": item.get("received_at"),
+            "work_id": wid,
+            "merged": item.get("merged"),
+            "pr_state": item.get("state"),
+            "source": item.get("source") or "willow-bot",
+        }
+        _emit(ev)
+        signals.append({k: v for k, v in ev.items() if k != "event"})
+
+    if len(consumed) > _MAX_CONSUMED:
+        consumed = set(sorted(consumed)[-_MAX_CONSUMED:])
+
+    state["inbox_consumed"] = sorted(consumed)
+    state["webhook_signals"] = signals[-_MAX_SIGNALS:]
+    state_path.parent.mkdir(parents=True, exist_ok=True)
+    state_path.write_text(json.dumps(state, indent=2) + "\n")
+    return 0
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print("usage: willow-bot-steward inbox <state.json>", file=sys.stderr)
+        return 2
+    return ingest(Path(sys.argv[1]))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

willow_bot/steward/mcp_client.py

@@ -0,0 +1,118 @@
+"""Thin MCP stdio client for steward prove-phase tool calls.
+
+Re-points the lifecycle pattern from ratatosk.mcp_client (Rule 11) — same env
+forwarding lesson, no dependency on the ratatosk package.
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+import shlex
+import sys
+import threading
+from typing import Any
+
+_mcp_session = None
+_mcp_loop: asyncio.AbstractEventLoop | None = None
+_mcp_stop_event: asyncio.Event | None = None
+_mcp_thread: threading.Thread | None = None
+_mcp_argv: list[str] | None = None
+
+_ENV_PREFIXES = ("WILLOW_", "WILLOW_BOT_", "PG")
+
+
+def default_mcp_argv() -> list[str]:
+    override = os.environ.get("WILLOW_BOT_MCP_COMMAND", "").strip()
+    if override:
+        return shlex.split(override)
+    module = os.environ.get("WILLOW_BOT_MCP_MODULE", "willow_mcp")
+    return [sys.executable, "-m", module]
+
+
+def server_env() -> dict[str, str]:
+    from mcp.client.stdio import get_default_environment
+
+    if os.environ.get("WILLOW_BOT_MCP_INHERIT_ENV", "").strip().lower() in (
+        "1",
+        "true",
+        "yes",
+    ):
+        return dict(os.environ)
+    env = dict(get_default_environment())
+    env.update({k: v for k, v in os.environ.items() if k.startswith(_ENV_PREFIXES)})
+    return env
+
+
+def _call_sync(coro):
+    assert _mcp_loop is not None
+    return asyncio.run_coroutine_threadsafe(coro, _mcp_loop).result(timeout=90)
+
+
+async def _lifecycle(argv: list[str], ready: threading.Event) -> None:
+    global _mcp_session, _mcp_stop_event
+    from mcp import ClientSession
+    from mcp.client.stdio import StdioServerParameters, stdio_client
+
+    stop = asyncio.Event()
+    _mcp_stop_event = stop
+    params = StdioServerParameters(command=argv[0], args=argv[1:], env=server_env())
+    async with stdio_client(params) as (read, write):
+        async with ClientSession(read, write) as session:
+            await session.initialize()
+            _mcp_session = session
+            ready.set()
+            await stop.wait()
+
+
+def start(argv: list[str] | None = None) -> None:
+    global _mcp_loop, _mcp_thread, _mcp_argv
+    if _mcp_session is not None:
+        return
+    argv = argv or default_mcp_argv()
+    _mcp_argv = argv
+    loop = asyncio.new_event_loop()
+    _mcp_loop = loop
+    ready = threading.Event()
+    _mcp_thread = threading.Thread(
+        target=lambda: loop.run_until_complete(_lifecycle(argv, ready)),
+        daemon=True,
+        name="willow-bot-mcp",
+    )
+    _mcp_thread.start()
+    if not ready.wait(timeout=90):
+        raise RuntimeError("MCP server did not initialize within 90s")
+
+
+def shutdown() -> None:
+    global _mcp_session, _mcp_loop, _mcp_thread, _mcp_stop_event
+    if _mcp_stop_event is not None and _mcp_loop is not None:
+        _mcp_loop.call_soon_threadsafe(_mcp_stop_event.set)
+    if _mcp_thread is not None:
+        _mcp_thread.join(timeout=10)
+    _mcp_session = None
+    _mcp_loop = None
+    _mcp_thread = None
+    _mcp_stop_event = None
+
+
+def call(name: str, inputs: dict[str, Any]) -> Any:
+    start()
+    assert _mcp_session is not None
+    result = _call_sync(_mcp_session.call_tool(name, inputs))
+    # SDK 1.x isError / 2.x is_error
+    is_err = getattr(result, "isError", None)
+    if is_err is None:
+        is_err = getattr(result, "is_error", False)
+    texts = []
+    for block in getattr(result, "content", None) or []:
+        t = getattr(block, "text", None)
+        if t is not None:
+            texts.append(t)
+    payload = "\n".join(texts) if texts else str(result)
+    if is_err:
+        raise RuntimeError(payload)
+    try:
+        return json.loads(payload)
+    except (json.JSONDecodeError, TypeError):
+        return payload