agentvow 0.1.0__py3-none-any.whl

agentvow/__init__.py

@@ -0,0 +1,2 @@
+"""Agentvow: check an agent's claims against independently observed repo state."""
+__version__ = "0.1.0"

agentvow/__main__.py

@@ -0,0 +1,3 @@
+from .cli import run
+
+raise SystemExit(run())

agentvow/adapters.py

@@ -0,0 +1,184 @@
+"""Agent-agnostic input: hook payloads and transcripts from different agents.
+
+Verified against: Claude Code session files (used in this project). Everything else is built from vendor documentation or
+tolerant format sniffing and is NOT yet verified against real Copilot/Codex/Cursor/Gemini transcripts: when a transcript
+cannot be understood the caller gets None and reports "nothing was verified" (never a guess).
+"""
+import json
+from pathlib import Path
+
+_TEXT_KEYS = ("text", "output_text", "content", "message", "last_assistant_message", "response")
+_ASSIST = {"assistant", "model", "agent", "ai", "bot", "copilot"}
+_ASSIST_TYPES = {"assistant", "assistant_message", "agent_message", "model", "model_response", "response", "agent_response"}
+
+
+def parse_hook_payload(payload: dict) -> dict:
+    """Normalise camelCase (Copilot CLI/cloud agent), snake_case (VS Code, Claude Code, Codex) and Cursor-style hook inputs."""
+    g = lambda *ks: next((payload[k] for k in ks if payload.get(k)), None)
+    return {
+        "cwd": g("cwd", "workingDirectory", "workspace_root", "workspaceRoot") or (g("workspace_roots") or [None])[0]
+        if isinstance(g("workspace_roots"), list) or g("cwd", "workingDirectory", "workspace_root", "workspaceRoot") else None,
+        "transcript": g("transcript_path", "transcriptPath"),
+        "message": g("last_assistant_message", "lastAssistantMessage", "final_message", "text"),
+        "session": g("session_id", "sessionId", "conversation_id"),
+        "loop": bool(g("stop_hook_active", "stopHookActive")),
+    }
+
+
+def _is_assistant(node: dict) -> bool:
+    role = str(node.get("role", "")).lower()
+    typ = str(node.get("type", "")).lower()
+    return role in _ASSIST or typ in _ASSIST_TYPES
+
+
+MAX_DEPTH = 40
+
+
+def _texts(node, assistant: bool, out: list, depth: int = 0):
+    """Collect text strings under assistant nodes (recursive; tolerant of unknown nesting; depth-bounded)."""
+    if depth > MAX_DEPTH:
+        return
+    if isinstance(node, dict):
+        a = assistant or _is_assistant(node)
+        for k, v in node.items():
+            if isinstance(v, str) and a and k in _TEXT_KEYS and v.strip() and not v.lstrip().startswith(("{", "[")):
+                if str(node.get("type", "")).lower() not in ("tool_use", "tool_result", "function_call", "tool_call", "thinking", "reasoning"):
+                    out.append(v.strip())
+            elif isinstance(v, (dict, list)):
+                _texts(v, a, out, depth + 1)
+    elif isinstance(node, list):
+        for v in node:
+            _texts(v, assistant, out, depth + 1)
+
+
+def _copilot_event_text(obj):
+    """GitHub Copilot CLI session events (~/.copilot/session-state/<id>/events.jsonl): {"type": "assistant.message", "data": {"content": "...",
+    "phase": "final_answer"|"commentary"|None, "toolRequests": [...]}}. Only data.content is the message; tool arguments (which can hold whole
+    files under a `content` key) must never be mistaken for it. -> (text, is_final) or None when this record is not an assistant message."""
+    if isinstance(obj, dict) and obj.get("type") == "assistant.message" and isinstance(obj.get("data"), dict):
+        d = obj["data"]
+        text = d.get("content")
+        return (text.strip() if isinstance(text, str) else ""), d.get("phase") == "final_answer"
+    return None
+
+
+def final_message_info(raw: str):
+    """-> (text or None, is_final, is_copilot). Copilot CLI transcripts are scoped to the CURRENT turn (everything after the latest
+    user.message): when the final answer has not been flushed to the file yet this returns no text, never the previous turn's answer."""
+    last, last_final, parsed_any, copilot = None, None, False, False
+    for line in raw.splitlines():
+        line = line.strip()
+        if not line or line[0] not in "{[" or len(line) > 2_000_000:
+            continue
+        try:
+            obj = json.loads(line)
+        except (ValueError, RecursionError):
+            continue
+        parsed_any = True
+        if isinstance(obj, dict) and obj.get("type") == "user.message":
+            last = last_final = None   # a new turn starts: nothing before this belongs to the answer we are looking for
+            copilot = True
+            continue
+        cop = _copilot_event_text(obj)
+        if cop is not None:
+            copilot = True
+            if cop[0]:
+                last = cop[0]
+                if cop[1]:
+                    last_final = cop[0]
+            continue
+        found: list = []
+        _texts(obj, False, found)
+        if found:
+            last = "\n".join(found).strip() or last
+    if last_final:
+        return last_final, True, copilot
+    if not parsed_any:
+        try:  # a single pretty-printed JSON document
+            obj = json.loads(raw[:5_000_000])
+            found = []
+            _texts(obj, False, found)
+            return (found[-1] if found else None), False, False
+        except (ValueError, RecursionError):
+            return (raw.strip()[-6000:] or None), False, False  # plain text transcript (bounded)
+    return last, False, copilot
+
+
+def final_message_from_text(raw: str):
+    return final_message_info(raw)[0]
+
+
+MAX_TRANSCRIPT_BYTES = 8_000_000   # transcripts can be hundreds of MB; the final answer is at the end, so only the tail is read
+
+
+def read_tail(path: Path, limit: int = MAX_TRANSCRIPT_BYTES) -> str:
+    """The last `limit` bytes of a regular file as text (first partial line dropped). Non-regular files (fifos, devices) are refused."""
+    import os, stat
+    fd = os.open(path, os.O_RDONLY | getattr(os, "O_NONBLOCK", 0))
+    try:
+        st = os.fstat(fd)
+        if not stat.S_ISREG(st.st_mode):
+            raise OSError("not a regular file")
+        cut = st.st_size > limit
+        if cut:
+            os.lseek(fd, st.st_size - limit, os.SEEK_SET)
+        data = b""
+        while len(data) < limit:
+            chunk = os.read(fd, min(1_000_000, limit - len(data)))
+            if not chunk:
+                break
+            data += chunk
+    finally:
+        os.close(fd)
+    text = data.decode("utf-8", errors="replace")
+    return text.split("\n", 1)[1] if cut and "\n" in text else text
+
+
+def final_message_info_file(path: Path):
+    try:
+        return final_message_info(read_tail(path))
+    except OSError:
+        return None, False, False
+
+
+def final_message(path: Path):
+    try:
+        return final_message_from_text(read_tail(path))
+    except OSError:
+        return None
+
+
+def shape_of(node, depth: int = 0):
+    """The STRUCTURE of a JSON value (key names, types, string lengths), never its content: safe to share when debugging an unknown format."""
+    if depth > 6:
+        return "..."
+    if isinstance(node, dict):
+        return {str(k): shape_of(v, depth + 1) for k, v in list(node.items())[:40]}
+    if isinstance(node, list):
+        return [shape_of(node[0], depth + 1), f"... {len(node)} items"] if node else []
+    if isinstance(node, str):
+        return f"str[{len(node)}]"
+    return type(node).__name__
+
+
+def transcript_shapes(path: Path, n: int = 4) -> dict:
+    """Structure of the first and last few records of a JSONL transcript (no content)."""
+    from collections import deque
+    head, tail = [], deque(maxlen=n)
+    try:
+        with path.open(encoding="utf-8", errors="replace") as fh:
+            for i, line in enumerate(fh):
+                line = line[:200_000]
+                if i < n:
+                    head.append(line)
+                else:
+                    tail.append(line)
+    except OSError as e:
+        return {"error": type(e).__name__}
+    shapes = []
+    for line in [*head, *tail]:
+        try:
+            shapes.append(shape_of(json.loads(line)))
+        except (ValueError, RecursionError):
+            shapes.append(f"non-json line, {len(line)} chars")
+    return {"records_sampled": len(shapes), "shapes": shapes}

agentvow/api_diff.py

@@ -0,0 +1,85 @@
+"""Public-API diff between two commits (Python, static): removed names and incompatible signature changes."""
+import ast
+import subprocess
+
+from .reality import safe_prefix
+from pathlib import Path
+
+
+def _show(repo: Path, ref: str, path: str):
+    r = subprocess.run(["git", *safe_prefix(repo), "-C", str(repo), "show", f"{ref}:{path}"], capture_output=True, text=True)
+    return r.stdout if r.returncode == 0 else None
+
+
+def _sig(fn):
+    a = fn.args
+    pos = [x.arg for x in a.posonlyargs + a.args]
+    req = pos[: len(pos) - len(a.defaults)]
+    kw = [x.arg for x in a.kwonlyargs]
+    kwreq = [x.arg for x, d in zip(a.kwonlyargs, a.kw_defaults) if d is None]
+    return {"pos": pos, "req": req, "kw": kw, "kwreq": kwreq, "var": a.vararg is not None, "varkw": a.kwarg is not None}
+
+
+def public_api(source: str, is_init: bool = False) -> dict:
+    api = {}
+    for n in ast.parse(source).body:
+        if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and not n.name.startswith("_"):
+            api[n.name] = _sig(n)
+        elif isinstance(n, ast.ClassDef) and not n.name.startswith("_"):
+            api[n.name] = {"class": True}
+            for m in n.body:
+                if isinstance(m, (ast.FunctionDef, ast.AsyncFunctionDef)) and (not m.name.startswith("_") or m.name == "__init__"):
+                    api[f"{n.name}.{m.name}"] = _sig(m)
+        elif isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name) and not n.target.id.startswith("_"):
+            api[n.target.id] = {"const": True}
+        elif is_init and isinstance(n, ast.ImportFrom):
+            for a in n.names:  # names re-exported from a package __init__ are part of its public API
+                nm = a.asname or a.name
+                if not nm.startswith("_") and nm != "*":
+                    api.setdefault(nm, {"const": True})
+        elif isinstance(n, ast.Assign):
+            for t in n.targets:
+                if isinstance(t, ast.Name) and not t.id.startswith("_"):
+                    api[t.id] = {"const": True}
+    return api
+
+
+def breaking(old: dict, new: dict) -> list:
+    out = []
+    for name, o in old.items():
+        n = new.get(name)
+        if n is None:
+            out.append(f"removed: {name}")
+        elif "pos" in o and "pos" in n:
+            gone = [p for p in o["pos"] + o["kw"] if p not in n["pos"] + n["kw"] and not n["varkw"]]
+            added = [p for p in n["req"] if p not in o["req"]] + [p for p in n["kwreq"] if p not in o["kwreq"]]
+            order = o["pos"] != n["pos"][: len(o["pos"])] and not gone
+            if gone:
+                out.append(f"{name}: parameter(s) removed or renamed: {', '.join(gone)}")
+            if added:
+                out.append(f"{name}: new required parameter(s): {', '.join(added)}")
+            if order:
+                out.append(f"{name}: positional parameter order changed")
+        elif ("pos" in o) != ("pos" in n):
+            out.append(f"{name}: changed kind (function/class/constant)")
+    return out
+
+
+def diff(repo: Path, base: str, files: list):
+    """Return (breaking_changes, gaps). Files that fail to parse are gaps, never silently OK."""
+    found, gaps = [], []
+    for f in files:
+        if not f.endswith(".py"):
+            continue
+        old = _show(repo, base, f)
+        if old is None:
+            continue  # new file: nothing to break
+        try:
+            new_src = (repo / f).read_text(encoding="utf-8") if (repo / f).exists() else None
+            if new_src is None:
+                found.append(f"removed file: {f}")
+                continue
+            found += [f"{f}: {x}" for x in breaking(public_api(old, f.endswith('__init__.py')), public_api(new_src, f.endswith('__init__.py')))]
+        except (SyntaxError, UnicodeDecodeError, RecursionError, MemoryError, ValueError):
+            gaps.append(f"could not parse {f} for API comparison")
+    return found, gaps

agentvow/ci.py

@@ -0,0 +1,59 @@
+"""CI evidence: read the project's own check results for the exact commit through the GitHub CLI (read-only GETs).
+
+Only jobs whose names look like test jobs count towards a "tests pass" claim; lint, security scans, title checks, release
+automation and similar are reported but ignored. A commit with no CI yields no evidence (never a pass).
+"""
+import json
+import re
+import time
+import subprocess
+from pathlib import Path
+
+from .reality import safe_prefix,  Evidence, Snapshot
+
+# v2 (2026-10-07, post-hoc after the RQ4 audit: v1 had 91% precision, 59% recall on 60 blind-judged job names).
+# Token-aware (underscores and digits do not hide a word), recognises python-version matrix names, excludes test.pypi.
+TEST_JOB = re.compile(r"(?<![a-z])(?:tests?(?!\.pypi)|pytest|pytester|unit|unittests?|integration|tox|nox|specs?)(?![a-z])|^(?:python|py)[ -]?3\.\d+|^\U0001F40D", re.I)
+TEST_JOB_V1 = re.compile(r"\b(?:tests?|pytest|unit|unittests?|integration|tox|nox|specs?)\b", re.I)
+_IND = {"framing": "separate", "evidence": "separate", "mechanism": "separate", "authority": "unknown"}  # the PR can change the CI configuration
+
+
+def _gh(path: str):
+    r = subprocess.run(["gh", "api", path], capture_output=True, text=True)
+    return json.loads(r.stdout) if r.returncode == 0 and r.stdout.strip() else None
+
+
+def _slug(repo: Path):
+    r = subprocess.run(["git", *safe_prefix(repo), "-C", str(repo), "remote", "get-url", "origin"], capture_output=True, text=True)
+    m = re.search(r"github\.com[:/]([\w.-]+/[\w.-]+?)(?:\.git)?$", r.stdout.strip())
+    return m.group(1) if m else None
+
+
+def ci_evidence(repo: Path, snap: Snapshot, fetch=_gh, slug=None, sha=None, wait: int = 0, sleep=time.sleep) -> list:
+    """Evidence from CI for HEAD. Not available for uncommitted work (CI never saw it)."""
+    if snap.dirty and not sha:
+        return []
+    sha = sha or snap.commit
+    slug = slug or _slug(repo)
+    if not slug:
+        return []
+    deadline = time.time() + wait
+    while True:
+        data = fetch(f"repos/{slug}/commits/{sha}/check-runs?per_page=100") or {}
+        pending = [r for r in data.get("check_runs", []) if TEST_JOB.search(r.get("name", "")) and r.get("status") != "completed"]
+        if not pending or time.time() >= deadline:   # test jobs still running right after a push: wait (bounded) instead of reporting "no result"
+            break
+        sleep(15)
+    if data.get("total_count", 0) > len(data.get("check_runs", [])):
+        return []  # truncated listing: refuse to draw conclusions from a partial view
+    out = []
+    for run in data.get("check_runs", []):
+        name, status, concl = run.get("name", ""), run.get("status"), run.get("conclusion")
+        if not TEST_JOB.search(name) or status != "completed":
+            continue
+        if concl in ("neutral", "skipped"):
+            continue  # a skipped job proves nothing
+        ok = concl == "success"
+        out.append(Evidence(f"ci_{len(out)}", "ci_check", "github-checks", snap.commit, "", "VALID", _IND,
+                            f"ci:{name} ({concl})", f"{'pass' if ok else 'fail'}: {name} {concl}"))
+    return out

agentvow/claims.py

@@ -0,0 +1,64 @@
+"""Rule-based claim extraction. No LLM: anything not matched is reported as unchecked."""
+import re
+from dataclasses import dataclass
+
+NO_IMPACT = "no_downstream_impact"
+TESTS_PASS = "tests_pass"
+BACKCOMPAT = "backward_compatible"
+
+_PATTERNS = [
+    (BACKCOMPAT, re.compile(r"\b(?:no|without|non)[- ]breaking (?:changes?|api)\b|\bbackward[s]?[- ]compatib|\bno api changes?\b|\bpublic api (?:is )?unchanged\b", re.I)),
+    (NO_IMPACT, re.compile(r"\bno (?:other |further )?(?:downstream |side )?(?:impact|effects?|consumers?|dependents?)\b|\bnothing else (?:depends|is affected)\b|\bdoes(?:n't| not) affect (?:any )?(?:other|downstream)", re.I)),
+    (TESTS_PASS, re.compile(r"\b(?:all )?tests? (?:are )?(?:pass(?:ed|ing)?|green)\b|\b\d+ passed\b|\b(\d[\d,]*)\s*/\s*\1\s+(?:tests?\s+)?(?:pass\w*|green)\b|\b\d[\d,]*\s+(?:[\w-]+\s+){0,2}tests?\b[^.\n]{0,80}\b(?:all )?(?:green|pass\w*|succe\w+)\b|\b(?:ran|run|running)\b[^.\n]{0,40}\b(?:tests?|pytest|test suite|suite)\b[^.\n]{0,60}\b(?:pass\w*|green|succe\w+)\b|\b(?:tests?|pytest|test suite|suite)\b[^.\n]{0,25}[:—-]\s*(?:all )?(?:pass\w*|green|ok|succe\w+)\b|\b(?:verified|confirmed)\b[^.\n]{0,60}\b(?:tests?|pytest|suite)\b[^.\n]{0,40}\b(?:pass\w*|green|succe\w+)\b", re.I)),
+]
+_GENERIC = re.compile(r"\b(?:safe to merge|looks good|fully (?:tested|verified)|ready to ship|backward[- ]compatible)\b", re.I)
+
+
+@dataclass
+class Claim:
+    kind: str  # NO_IMPACT, TESTS_PASS, or "unchecked"
+    text: str
+    count: int | None = None  # number of tests the agent says passed, if stated
+
+
+_RATIO = re.compile(r"\b(\d[\d,]*)\s*/\s*(\d[\d,]*)\s+(?:tests?\s+)?(?:pass\w*|green)\b", re.I)
+_COUNT = re.compile(r"\b(\d[\d,]*)\s+(?:[\w-]+\s+){0,3}?tests?\s+(?:are\s+|were\s+|all\s+)?(?:pass(?:ed|ing)?|green)\b|\b(\d[\d,]*)\s+passed\b|\b(\d[\d,]*)\s+(?:[\w-]+\s+){0,2}tests?\b[^.\n]{0,80}\b(?:all )?(?:green|passing)\b", re.I)
+
+
+_NEGATED = re.compile(r"\b(?:not|never|no longer|n't|unable to|failed to|couldn't|cannot|can't)\b[^.\n]{0,40}\b(?:confirm|verify|verified|run|ran|pass\w*|green)\b|\b(?:could|can) ?not\b", re.I)
+
+
+def extract(transcript: str) -> list[Claim]:
+    claims: list[Claim] = []
+    transcript = transcript[-20000:]  # bound the work done on agent-controlled text (the regexes are not linear-time on adversarial input)
+    for sentence in re.split(r"(?<=[.!?])\s+|\n+", transcript):
+        s = sentence.strip()[:600]
+        if not s:
+            continue
+        if _NEGATED.search(s):
+            claims.append(Claim("unchecked", s))  # a negated statement is not a claim of success; surface it as unchecked
+            continue
+        if re.search(r"\b[1-9]\d*\s+(?:failed|failing|failures?|errors?)\b", s, re.I):
+            claims.append(Claim("unchecked", s))  # an honest failure report is not a claim that tests pass
+            continue
+        matched = False
+        for kind, pat in _PATTERNS:
+            if pat.search(s):
+                r = _RATIO.search(s) if kind == TESTS_PASS else None
+                m = _COUNT.search(s) if kind == TESTS_PASS and not r else None
+                n = int(r.group(2).replace(",", "")) if r else (int(next(g for g in m.groups() if g).replace(",", "")) if m else None)
+                claims.append(Claim(kind, s, n))
+                matched = True
+        if not matched and _GENERIC.search(s):
+            claims.append(Claim("unchecked", s))
+    return claims
+
+
+def sentences(text: str) -> list[str]:
+    return [x.strip() for x in re.split(r"(?<=[.!?])\s+|\n+", text) if x.strip()]
+
+
+def count_unexamined(text: str, found: list) -> int:
+    """Substantive sentences (5+ words) that no extractor pattern looked at."""
+    seen = {c.text for c in found}
+    return sum(1 for x in sentences(text) if len(x.split()) >= 5 and x not in seen)