pmkit 0.1.1__py3-none-any.whl

pmkit/dogfood/install.py

@@ -0,0 +1,111 @@
+"""Clean-room install runner.
+
+Runs a product's *documented* install/run commands verbatim in a throwaway working
+directory and reports what actually happens. A failed documented step is a gap (a
+doc-vs-reality finding), never a crash — the run continues so later independent steps
+still get exercised. Commands are expected to be self-contained (e.g. `uvx <pkg> ...`,
+`uv run --no-project --with <pkg> ...`), which create their own ephemeral environments.
+
+SECURITY: these command strings are executed verbatim through a shell (`shell=True`) with
+the operator's local privileges — running them IS the point, so they must come from a
+trusted, operator-reviewed source, not directly from untrusted/auto-scraped product docs
+without a human in the loop. To limit blast radius, the run gets a minimal allow-listed
+environment (no inherited API keys/secrets); callers opt specific vars back in via `env`.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import tempfile
+from dataclasses import asdict, dataclass, field
+from typing import Optional
+
+# Only these env vars are passed into the throwaway run; everything else (the operator's
+# API keys and secrets) is withheld from the third-party documented commands.
+_SAFE_ENV_KEYS = {
+    "PATH", "PATHEXT", "SYSTEMROOT", "SYSTEMDRIVE", "COMSPEC", "WINDIR",
+    "TEMP", "TMP", "TMPDIR", "HOME", "HOMEDRIVE", "HOMEPATH", "USERPROFILE",
+    "LANG", "LC_ALL", "LC_CTYPE", "PYTHONUTF8",
+}
+
+
+def _clean_env(extra: Optional[dict]) -> dict:
+    base = {k: v for k, v in os.environ.items() if k.upper() in _SAFE_ENV_KEYS}
+    base.update(extra or {})
+    return base
+
+
+@dataclass
+class StepResult:
+    command: str
+    ok: bool
+    exit_code: Optional[int]
+    output: str
+    gap: bool
+    reason: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class InstallReport:
+    steps: list[StepResult] = field(default_factory=list)
+
+    @property
+    def all_ok(self) -> bool:
+        return bool(self.steps) and all(s.ok for s in self.steps)
+
+    @property
+    def gaps(self) -> list[StepResult]:
+        return [s for s in self.steps if s.gap]
+
+    def to_dict(self) -> dict:
+        return {"all_ok": self.all_ok, "steps": [s.to_dict() for s in self.steps]}
+
+
+def run_documented_install(
+    commands: list[str],
+    *,
+    workdir: Optional[str] = None,
+    timeout: float = 120.0,
+    env: Optional[dict] = None,
+) -> InstallReport:
+    """Run each documented command in a throwaway dir; return a per-step report.
+
+    A non-zero exit, timeout, or spawn failure marks that step as a gap and the run
+    proceeds to the next command rather than raising.
+    """
+    report = InstallReport()
+    cleanup = workdir is None
+    workdir = workdir or tempfile.mkdtemp(prefix="pmkit-dogfood-")
+    run_env = _clean_env(env)
+    try:
+        for cmd in commands:
+            report.steps.append(_run_one(cmd, workdir, timeout, run_env))
+    finally:
+        if cleanup:
+            shutil.rmtree(workdir, ignore_errors=True)
+    return report
+
+
+def _run_one(cmd: str, cwd: str, timeout: float, env: dict) -> StepResult:
+    try:
+        p = subprocess.run(
+            cmd, shell=True, cwd=cwd, env=env,
+            capture_output=True, text=True, timeout=timeout,
+        )
+        output = ((p.stdout or "") + (p.stderr or ""))[-4000:]
+        ok = p.returncode == 0
+        return StepResult(cmd, ok, p.returncode, output, gap=not ok,
+                          reason="" if ok else f"exit {p.returncode}")
+    except subprocess.TimeoutExpired as exc:
+        partial = (exc.stdout or "") + (exc.stderr or "")
+        if isinstance(partial, bytes):
+            partial = partial.decode("utf-8", "replace")
+        return StepResult(cmd, False, None, partial[-4000:], gap=True,
+                          reason=f"timeout after {timeout}s")
+    except Exception as e:  # spawn failure (bad command, missing interpreter, ...)
+        return StepResult(cmd, False, None, str(e), gap=True, reason=f"could not run: {e}")

pmkit/dogfood/mcp.py

@@ -0,0 +1,73 @@
+"""Agent/MCP driver — connects to an MCP server as a real client (FastMCP `Client`).
+
+The call-plan validation is pure and unit-tested. The live handshake lazily imports the
+FastMCP client and is gated on availability (FastMCP is an optional `pmkit[dogfood]` dep,
+not a core one), so the suite runs without it. The driver launches the documented server
+command over stdio and calls the documented tools — exercising the real wire, not the engine.
+
+SECURITY: the server launch command is executed verbatim with the operator's local
+privileges (same trust assumption as the install runner) — it must come from a trusted,
+operator-reviewed source, not an untrusted/auto-scraped doc without a human in the loop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+
+def plan_calls(calls: list[dict]) -> list[dict]:
+    """Validate + normalize tool calls into a plan. Pure."""
+    plan: list[dict] = []
+    for i, c in enumerate(calls):
+        tool = c.get("tool")
+        if not tool:
+            raise ValueError(f"call {i}: missing 'tool'")
+        args = c.get("args", {})
+        if not isinstance(args, dict):
+            raise ValueError(f"call {i}: 'args' must be an object")
+        plan.append({"tool": tool, "args": args})
+    return plan
+
+
+def mcp_client_available() -> bool:
+    try:
+        from fastmcp import Client  # noqa: F401
+    except Exception:
+        return False
+    return True
+
+
+def drive_mcp(server_cmd: list[str], calls: list[dict], *, timeout: float = 30.0) -> list[dict]:
+    """Launch the documented server over stdio and call its tools. Raises if FastMCP absent."""
+    plan = plan_calls(calls)
+    if not mcp_client_available():
+        raise RuntimeError("FastMCP client not available — install pmkit[dogfood] (fastmcp)")
+    return asyncio.run(_drive_async(server_cmd, plan, timeout))
+
+
+async def _drive_async(server_cmd: list[str], plan: list[dict], timeout: float) -> list[dict]:
+    from fastmcp import Client
+    from fastmcp.client.transports import StdioTransport
+
+    obs: list[dict] = []
+    transport = StdioTransport(command=server_cmd[0], args=list(server_cmd[1:]))
+
+    async def _run() -> None:
+        async with Client(transport) as client:
+            for call in plan:
+                try:
+                    res = await client.call_tool(call["tool"], call["args"])
+                    obs.append({"step": f"{call['tool']}({call['args']})", "ok": True,
+                                "observed": str(getattr(res, "data", res))[:500]})
+                except Exception as e:
+                    obs.append({"step": call["tool"], "ok": False,
+                                "observed": f"{type(e).__name__}: {e}"})
+
+    try:
+        await asyncio.wait_for(_run(), timeout)
+    except asyncio.TimeoutError:
+        obs.append({"step": "connect", "ok": False, "observed": f"timeout after {timeout}s"})
+    except Exception as e:
+        obs.append({"step": "connect", "ok": False, "observed": f"{type(e).__name__}: {e}"})
+    return obs

pmkit/dogfood/report.py

@@ -0,0 +1,157 @@
+"""Dogfood findings model, parity check, and report rendering.
+
+Pure functions over the drivers' result shapes. An observation is a dict:
+``{"step": str, "ok": bool, "observed": Any, "claim": str (optional)}``. The report
+normalizes install + UI + MCP observations into per-interface pass/fail findings, adds
+parity findings (UI vs MCP must agree), and renders agent/human-readable markdown.
+"""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, field
+from typing import Any, Optional
+
+
+@dataclass
+class Finding:
+    interface: str          # install | ui | mcp | parity
+    title: str
+    status: str             # pass | fail
+    gap: bool
+    claim: str = ""
+    observed: str = ""
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass
+class DogfoodReport:
+    target: str
+    findings: list[Finding] = field(default_factory=list)
+
+    @property
+    def gaps(self) -> list[Finding]:
+        return [f for f in self.findings if f.gap]
+
+    def passed(self) -> bool:
+        return not self.gaps
+
+    def per_interface(self) -> dict[str, dict[str, int]]:
+        out: dict[str, dict[str, int]] = {}
+        for f in self.findings:
+            d = out.setdefault(f.interface, {"pass": 0, "fail": 0})
+            d["pass" if f.status == "pass" else "fail"] += 1
+        return out
+
+    def to_dict(self) -> dict:
+        return {
+            "target": self.target,
+            "passed": self.passed(),
+            "per_interface": self.per_interface(),
+            "findings": [f.to_dict() for f in self.findings],
+        }
+
+
+def from_install(install_report) -> list[Finding]:
+    out: list[Finding] = []
+    for s in install_report.steps:
+        out.append(Finding(
+            interface="install",
+            title=s.command,
+            status="pass" if s.ok else "fail",
+            gap=s.gap,
+            claim="documented install step succeeds",
+            observed=s.reason or ("ok" if s.ok else (s.output or "")[:200]),
+        ))
+    return out
+
+
+def _observations(interface: str, obs: Optional[list[dict]]) -> list[Finding]:
+    out: list[Finding] = []
+    for ob in obs or []:
+        ok = ob.get("ok", True)
+        out.append(Finding(
+            interface=interface,
+            title=str(ob.get("step", "")),
+            status="pass" if ok else "fail",
+            gap=not ok,
+            claim=str(ob.get("claim", "")),
+            observed=str(ob.get("observed", "")),
+        ))
+    return out
+
+
+def parity_check(ui_state: dict, mcp_state: dict) -> list[Finding]:
+    """Compare the two surfaces' end states on shared keys; divergence is a gap.
+
+    Disjoint non-empty states are a gap too ("not checkable") — they must not read as a
+    clean pass. Two empty states yield nothing (there was no state to compare)."""
+    shared = set(ui_state) & set(mcp_state)
+    if not shared:
+        if ui_state or mcp_state:
+            return [Finding(
+                interface="parity",
+                title="parity not checkable: surfaces share no state keys",
+                status="fail",
+                gap=True,
+                claim="UI and MCP surfaces expose comparable state",
+                observed=f"ui keys={sorted(map(str, ui_state))} mcp keys={sorted(map(str, mcp_state))}",
+            )]
+        return []
+    diverged = [
+        Finding(
+            interface="parity",
+            title=f"surfaces disagree on {k!r}",
+            status="fail",
+            gap=True,
+            claim="UI and MCP surfaces agree (parity)",
+            observed=f"ui={ui_state[k]!r} mcp={mcp_state[k]!r}",
+        )
+        for k in sorted(shared, key=str)
+        if str(ui_state[k]) != str(mcp_state[k])
+    ]
+    if shared and not diverged:
+        return [Finding("parity", "UI/MCP parity holds", "pass", gap=False,
+                        claim="UI and MCP surfaces agree", observed=f"{len(shared)} keys match")]
+    return diverged
+
+
+def build_report(
+    target: str,
+    *,
+    install=None,
+    ui: Optional[list[dict]] = None,
+    mcp: Optional[list[dict]] = None,
+    ui_state: Optional[dict] = None,
+    mcp_state: Optional[dict] = None,
+) -> DogfoodReport:
+    findings: list[Finding] = []
+    if install is not None:
+        findings += from_install(install)
+    findings += _observations("ui", ui)
+    findings += _observations("mcp", mcp)
+    if ui_state is not None and mcp_state is not None:
+        findings += parity_check(ui_state, mcp_state)
+    return DogfoodReport(target, findings)
+
+
+def render_markdown(report: DogfoodReport) -> str:
+    lines = [f"# Dogfood report: {report.target}", ""]
+    lines.append(f"**Result:** {'PASS' if report.passed() else 'GAPS FOUND'}")
+    lines.append("")
+    lines.append("## Per-interface")
+    for iface, c in report.per_interface().items():
+        lines.append(f"- {iface}: {c['pass']} pass, {c['fail']} fail")
+    gaps = report.gaps
+    lines.append("")
+    lines.append(f"## Gaps ({len(gaps)})")
+    if not gaps:
+        lines.append("- none")
+    for g in gaps:
+        lines.append(f"- [{g.interface}] {g.title}")
+        if g.claim:
+            lines.append(f"  - claim: {g.claim}")
+        if g.observed:
+            lines.append(f"  - observed: {g.observed}")
+    return "\n".join(lines)

pmkit/dogfood/sample.py

@@ -0,0 +1,32 @@
+"""Sample-app synthesis.
+
+When a product's documented scenario needs an app to act on (streamlit-mcp serves a
+Streamlit app), pm-dogfood synthesizes a minimal representative one rather than requiring
+the operator to supply it. Kept tiny and deterministic so the same scenario reruns cleanly.
+"""
+
+from __future__ import annotations
+
+SAMPLE_STREAMLIT_APP = '''import streamlit as st
+
+if "saves" not in st.session_state:
+    st.session_state.saves = 0
+
+name = st.text_input("Name", value="world", key="name")
+if st.button("Save", key="save"):
+    st.session_state.saves += 1
+
+st.markdown(f"Hello, {name}!")
+st.markdown(f"saves = {st.session_state.saves}")
+'''
+
+
+def sample_streamlit_source() -> str:
+    return SAMPLE_STREAMLIT_APP
+
+
+def synth_streamlit_app(path: str) -> str:
+    """Write the sample Streamlit app to ``path`` and return the path."""
+    with open(path, "w", encoding="utf-8") as fh:
+        fh.write(SAMPLE_STREAMLIT_APP)
+    return path

pmkit/dogfood/ui.py

@@ -0,0 +1,106 @@
+"""Human/UI driver — drives a rendered app in a real browser via Playwright.
+
+The step-translation (scenario -> action plan) is pure and unit-tested. The live browser
+pass lazily imports Playwright and is gated on availability, so pmkit stays stdlib-only and
+the test suite runs without a browser. Live selectors are best-effort for Streamlit (label
++ role) and are validated by the integration run (U6), not unit tests.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+SUPPORTED_ACTIONS = ("set", "click", "read")
+
+
+def translate_steps(steps: list[dict]) -> list[dict]:
+    """Validate + normalize inferred scenario steps into a UI action plan. Pure."""
+    plan: list[dict] = []
+    for i, s in enumerate(steps):
+        action = s.get("action")
+        if action not in SUPPORTED_ACTIONS:
+            raise ValueError(f"step {i}: unknown action {action!r} (expected {SUPPORTED_ACTIONS})")
+        target = s.get("target")
+        if action in ("set", "click") and not target:
+            raise ValueError(f"step {i}: '{action}' needs a target")
+        if action == "set" and "value" not in s:
+            raise ValueError(f"step {i}: 'set' needs a value")
+        plan.append({"action": action, "target": target, "value": s.get("value")})
+    return plan
+
+
+def playwright_available() -> bool:
+    # Import the sync API, not just the top-level package: the sync API pulls in
+    # greenlet's compiled extension, so a shallow `import playwright` can pass while
+    # the browser runtime cannot actually start (e.g. a missing VC++ runtime DLL on
+    # Windows). Importing sync_playwright makes the gate reflect launchability.
+    try:
+        from playwright.sync_api import sync_playwright  # noqa: F401
+    except Exception:
+        return False
+    return True
+
+
+def drive_ui(url: str, steps: list[dict], *, timeout_ms: int = 10000) -> list[dict]:
+    """Drive the app's rendered UI in a real browser. Raises if Playwright is absent."""
+    plan = translate_steps(steps)
+    if not playwright_available():
+        raise RuntimeError("Playwright not installed — run `pip install 'pmkit[dogfood]'` "
+                           "then `playwright install chromium`")
+    from playwright.sync_api import sync_playwright
+
+    obs: list[dict] = []
+    with sync_playwright() as p:
+        browser = p.chromium.launch()
+        try:
+            page = browser.new_page()
+            try:
+                page.goto(url, timeout=timeout_ms)
+            except Exception as e:
+                # a failed connect is a gap, not a crash (and must not skip browser.close)
+                obs.append({"step": f"goto {url}", "ok": False,
+                            "observed": f"{type(e).__name__}: {e}"})
+                return obs
+            for step in plan:
+                obs.append(_run_step(page, step))
+        finally:
+            browser.close()
+    return obs
+
+
+def _settle(page: Any) -> None:
+    """Let a reactive frontend (e.g. Streamlit) finish its rerun before we read.
+
+    Streamlit reruns asynchronously over a websocket: an interaction returns
+    immediately but the new DOM paints a beat later. Reading too soon captures the
+    pre-rerun page. We wait for network to quiesce (best-effort — the websocket may
+    never fully idle) plus a short paint budget. Cheap and generic across frameworks.
+    """
+    try:
+        page.wait_for_load_state("networkidle", timeout=3000)
+    except Exception:
+        pass
+    page.wait_for_timeout(400)
+
+
+def _run_step(page: Any, step: dict) -> dict:
+    action, target, value = step["action"], step["target"], step["value"]
+    try:
+        if action == "set":
+            loc = page.get_by_label(target)
+            loc.fill(str(value))
+            # Streamlit (and many reactive inputs) only commit a typed value on
+            # Enter/blur, not on a programmatic fill — without this the rerun never
+            # fires and the new value is silently dropped.
+            loc.press("Enter")
+            _settle(page)
+            return {"step": f"set {target}={value}", "ok": True, "observed": "set"}
+        if action == "click":
+            page.get_by_role("button", name=target).click()
+            _settle(page)
+            return {"step": f"click {target}", "ok": True, "observed": "clicked"}
+        # read
+        text = page.get_by_text(target).inner_text() if target else page.inner_text("body")
+        return {"step": f"read {target or 'page'}", "ok": True, "observed": text[:500]}
+    except Exception as e:
+        return {"step": f"{action} {target}", "ok": False, "observed": f"{type(e).__name__}: {e}"}

pmkit/killtest.py

@@ -0,0 +1,31 @@
+"""The kill-test survival rule — the single, tested source of truth.
+
+A blunt "prune on >= N refutes" majority let a vendor-already-shipped idea survive on a
+2-of-4 split (the Dash->MCP case). The fix: the **already-solved** axis is *dispositive* —
+a confident already-solved refutation prunes a candidate regardless of how the other axes
+voted (if the thing already exists, nothing else matters). Otherwise the strict majority
+applies. The pm-run orchestrator calls this via `pmkit backlog killtest --decide` rather
+than re-implementing the rule, so there's no JS/Python drift.
+"""
+
+from __future__ import annotations
+
+SOLVED_AXIS = "already-solved"
+SOLVED_CONFIDENCE = 0.7  # an already-solved refute at/above this is a hard kill
+
+
+def decide_survival(verdicts: list[dict], prune_at: int = 3) -> tuple[bool, str]:
+    """Return (survived, reason) for a candidate given its per-axis kill-test verdicts.
+
+    Each verdict is a dict with at least ``verdict`` ('refute'|'survive') and ``axis``;
+    ``confidence`` (0..1) is optional and defaults to 1.0.
+    """
+    refutes = [v for v in verdicts if v.get("verdict") == "refute"]
+    for v in refutes:
+        if v.get("axis") == SOLVED_AXIS and float(v.get("confidence", 1.0)) >= SOLVED_CONFIDENCE:
+            reason = (v.get("reason") or "").strip()
+            return False, f"pruned: already-solved is dispositive ({reason})"[:200]
+    n = len(refutes)
+    if n >= prune_at:
+        return False, f"pruned: majority refute ({n}/{len(verdicts)})"
+    return True, f"survived ({n} refute(s); need {prune_at}, no dispositive already-solved)"

pmkit/launch/__init__.py

@@ -0,0 +1,15 @@
+"""pm-launch — the funnel's launch/amplify stage (deterministic core).
+
+This package holds the *logistics* layer of launching a shipped product: the launch-state
+ledger and mod-policy cache (``store``), moderator-policy verdicts (``policy``), the
+listen/feedback loop (``listen``), the emit-only launch plan (``plan``), Tier-A collateral
+capture (``collateral``), and draft-starting-point storage (``drafts``).
+
+Hard boundaries, enforced here rather than left to convention:
+- Nothing in this package posts to any channel — ever (the human gate lives in the skill).
+- Drafts are *starting-points*; there is no "final"/"postable" state in the data model.
+- The launch plan is emit-only: it renders an artifact and creates no cron side-effects.
+
+Judgment (which channels, reading prose rules, writing copy, slop critique) lives in the
+``agents/pm-launch-*`` personas; this package is the rule/ledger half.
+"""