devin-qa-pack 0.1.0__py3-none-any.whl

devin_qa_pack/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

devin_qa_pack/adapters/__init__.py

@@ -0,0 +1,17 @@
+"""Source adapters: map an agent's transcript format into the audit model.
+
+Every adapter produces a :class:`SourceSession` — the claims an agent
+made plus the tool calls recorded as ground truth, normalized so the
+same verifier runs over Devin stores and foreign transcripts alike.
+
+- :mod:`devin_qa_pack.adapters.devin` — the native store
+  (``sessions.db`` via devin-internals-spec).
+- :mod:`devin_qa_pack.adapters.aider` — ``.aider.chat.history.md``
+  (experimental).
+- :mod:`devin_qa_pack.adapters.claude_code` — Claude Code ``.jsonl``
+  session transcripts (experimental).
+"""
+
+from devin_qa_pack.adapters.base import SourceSession
+
+__all__ = ["SourceSession"]

devin_qa_pack/adapters/aider.py

@@ -0,0 +1,91 @@
+"""Experimental Aider adapter — audits ``.aider.chat.history.md``.
+
+Aider records its chat history as markdown: ``####`` lines are user
+turns, ``>`` lines are commands issued through aider and everything else
+is assistant prose (the claim source).
+
+Evidence is deliberately weak where the format is weak: a recorded
+``/run`` or ``/test`` command becomes an execute call with ``unknown``
+status — the history does not persist exit codes — so test claims
+resolve ``unverifiable``, never a fake ``verified``. Commit claims are
+still checkable: aider auto-commits, so the repo's own ``git log`` is
+ground truth whenever the working directory is on disk.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from devin_qa_pack.adapters.base import SourceSession
+from devin_qa_pack.claims import claims_from_pairs
+from devin_qa_pack.verify import ParsedToolCall
+
+SOURCE = "aider"
+
+_RUN_SLASH = ("/run", "/test")
+
+
+def _shell_command(recorded: str) -> str | None:
+    """Map a ``>`` history line to the shell command it ran, if any.
+
+    ``/run`` and ``/test`` execute a shell command; other slash commands
+    (``/add``, ``/model``…) do not. A bare ``>`` line is treated as a
+    raw command.
+    """
+    for slash in _RUN_SLASH:
+        if recorded == slash or recorded.startswith(slash + " "):
+            return recorded[len(slash):].strip() or None
+    if recorded.startswith("/"):
+        return None
+    return recorded or None
+
+
+def _parse(text: str) -> tuple[list[tuple[str, str]], list[ParsedToolCall]]:
+    pairs: list[tuple[str, str]] = []
+    calls: list[ParsedToolCall] = []
+    for raw in text.splitlines():
+        line = raw.rstrip()
+        if not line.strip():
+            continue
+        if line.startswith("####"):
+            pairs.append(("user", line[4:].strip()))
+            continue
+        if line.startswith("#"):
+            continue  # "# aider chat started at …" and section headers
+        if line.startswith(">"):
+            command = _shell_command(line[1:].strip())
+            if command is not None:
+                n = len(calls) + 1
+                calls.append(
+                    ParsedToolCall(
+                        tool_call_id=f"aider-cmd-{n}",
+                        kind="execute",
+                        status="unknown",  # history does not record exit codes
+                        commands=(command,),
+                        search_text=command.lower().replace("\\", "/"),
+                        http_statuses=(),
+                    )
+                )
+            continue
+        pairs.append(("assistant", line))
+    return pairs, calls
+
+
+def load(path: str | Path, working_directory: str | None = None) -> SourceSession:
+    """Load ``.aider.chat.history.md`` into the normalized audit view.
+
+    ``working_directory`` defaults to the directory containing the
+    history file — aider writes it at the repository root.
+    """
+    p = Path(path).expanduser()
+    pairs, calls = _parse(p.read_text(encoding="utf-8", errors="replace"))
+    session_id = f"aider:{p.name}"
+    return SourceSession(
+        source=SOURCE,
+        session_id=session_id,
+        title=p.name,
+        working_directory=working_directory or str(p.parent),
+        claims=tuple(claims_from_pairs(pairs, session_id)),
+        calls=tuple(calls),
+        raw_call_count=len(calls),
+    )

devin_qa_pack/adapters/base.py

@@ -0,0 +1,26 @@
+"""The normalized session view every adapter produces."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from devin_qa_pack.claims import Claim
+from devin_qa_pack.verify import ParsedToolCall
+
+
+@dataclass(frozen=True)
+class SourceSession:
+    """One session's claims plus its recorded tool calls.
+
+    ``raw_call_count`` is the number of evidence rows *before* decoding —
+    when rows exist but none parse, ground truth is unreadable and claims
+    resolve ``unverifiable`` rather than ``disputed``.
+    """
+
+    source: str  # adapter name: "devin" | "aider" | "claude-code" | ...
+    session_id: str
+    title: str | None
+    working_directory: str
+    claims: tuple[Claim, ...]
+    calls: tuple[ParsedToolCall, ...]
+    raw_call_count: int

devin_qa_pack/adapters/claude_code.py

@@ -0,0 +1,131 @@
+"""Experimental Claude Code adapter — audits session ``.jsonl`` transcripts.
+
+Claude Code writes one JSONL transcript per session (usually under
+``~/.claude/projects/<slug>/``). ``assistant`` entries carry ``text``
+blocks — the claim source — and ``tool_use`` blocks; the matching
+``tool_result`` blocks in ``user`` entries mark calls completed/failed
+and carry the output used as evidence text.
+
+Tool mapping: ``Bash`` → execute, ``Write``/``Edit``/``MultiEdit``/
+``NotebookEdit`` → write, anything else → other (its payload text still
+contributes evidence). Entries carry a ``cwd`` field used as the working
+directory for git/file checks when no override is given.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any, Iterable
+
+from devin_qa_pack.adapters.base import SourceSession
+from devin_qa_pack.claims import claims_from_pairs
+from devin_qa_pack.verify import ParsedToolCall, _http_statuses, _strings
+
+SOURCE = "claude-code"
+
+_EXECUTE_TOOLS = {"bash", "shell", "execute", "run", "terminal"}
+_WRITE_TOOLS = {"write", "edit", "multiedit", "notebookedit"}
+
+
+def _entries(path: Path) -> Iterable[dict[str, Any]]:
+    for raw in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        raw = raw.strip()
+        if not raw:
+            continue
+        try:
+            entry = json.loads(raw)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(entry, dict):
+            yield entry
+
+
+def _blocks(entry: dict[str, Any]) -> list[dict[str, Any]]:
+    msg = entry.get("message")
+    content = msg.get("content") if isinstance(msg, dict) else entry.get("content")
+    if isinstance(content, str):
+        return [{"type": "text", "text": content}]
+    if isinstance(content, list):
+        return [b for b in content if isinstance(b, dict)]
+    return []
+
+
+def _result_text(content: Any) -> str:
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        return "\n".join(
+            str(b.get("text", "")) for b in content
+            if isinstance(b, dict) and isinstance(b.get("text"), str)
+        )
+    return "" if content is None else str(content)
+
+
+def _tool_kind(name: str) -> str:
+    if name in _EXECUTE_TOOLS:
+        return "execute"
+    if name in _WRITE_TOOLS:
+        return "write"
+    return "other"
+
+
+def load(path: str | Path, working_directory: str | None = None) -> SourceSession:
+    p = Path(path).expanduser()
+    pairs: list[tuple[str, str]] = []
+    uses: dict[str, dict[str, Any]] = {}
+    order: list[str] = []
+    status_by_id: dict[str, str] = {}
+    out_by_id: dict[str, str] = {}
+    cwd = working_directory
+
+    for entry in _entries(p):
+        role = entry.get("type") or ""
+        if cwd is None and isinstance(entry.get("cwd"), str):
+            cwd = entry["cwd"]
+        for b in _blocks(entry):
+            btype = b.get("type")
+            if btype == "text":
+                pairs.append((str(role or "assistant"), str(b.get("text", ""))))
+            elif btype == "tool_use":
+                tid = str(b.get("id") or f"call-{len(order) + 1}")
+                if tid not in uses:
+                    order.append(tid)
+                uses[tid] = b
+            elif btype == "tool_result":
+                tid = str(b.get("tool_use_id") or "")
+                out_by_id[tid] = _result_text(b.get("content"))
+                status_by_id[tid] = "failed" if b.get("is_error") else "completed"
+
+    calls: list[ParsedToolCall] = []
+    for tid in order:
+        use = uses[tid]
+        name = str(use.get("name") or "").lower()
+        inp = use.get("input") if isinstance(use.get("input"), dict) else {}
+        command = inp.get("command")
+        commands = (command,) if isinstance(command, str) and command.strip() else ()
+        output = out_by_id.get(tid, "")
+        texts = [*_strings(inp), output, name]
+        calls.append(
+            ParsedToolCall(
+                tool_call_id=tid,
+                kind=_tool_kind(name),
+                status=status_by_id.get(tid, "unknown"),
+                commands=commands,
+                search_text=" ".join(texts).lower().replace("\\", "/"),
+                http_statuses=tuple(sorted(
+                    {*_http_statuses(inp), *_http_statuses(output)}
+                )),
+            )
+        )
+
+    session_id = f"claude-code:{p.name}"
+    return SourceSession(
+        source=SOURCE,
+        session_id=session_id,
+        title=p.name,
+        working_directory=cwd or str(p.parent),
+        claims=tuple(claims_from_pairs(pairs, session_id)),
+        calls=tuple(calls),
+        raw_call_count=len(calls),
+    )

devin_qa_pack/adapters/devin.py

@@ -0,0 +1,28 @@
+"""The Devin adapter — the native ``sessions.db`` source.
+
+Claims come from ``message_nodes`` (agent-role text); ground truth comes
+from ``tool_call_state`` rows decoded via devin-internals-spec parsers.
+"""
+
+from __future__ import annotations
+
+from devin_internals.parsers import SessionsStore
+from devin_internals.parsers.sessions import Session
+
+from devin_qa_pack.adapters.base import SourceSession
+from devin_qa_pack.claims import extract_claims
+from devin_qa_pack.verify import parse_tool_calls
+
+
+def session_view(store: SessionsStore, session: Session) -> SourceSession:
+    nodes = store.message_nodes(session.id)
+    states = store.tool_call_state(session.id)
+    return SourceSession(
+        source="devin",
+        session_id=session.id,
+        title=session.title,
+        working_directory=session.working_directory,
+        claims=tuple(extract_claims(nodes)),
+        calls=tuple(parse_tool_calls(states)),
+        raw_call_count=len(states),
+    )

devin_qa_pack/claims.py

@@ -0,0 +1,226 @@
+"""Extract deliverable claims from a session's ``message_nodes``.
+
+A *claim* is an assertion the agent made about delivered work: tests
+passed, a commit exists, a file was created/changed, work was pushed.
+Only agent-role messages are scanned — a user prompt quoting the same
+words is not a delivery claim.
+
+``chat_message`` payloads are marked *unstable* in devin-internals-spec
+SCHEMA.md, so decoding is defensive: JSON dicts are checked for the
+usual role/text/content keys (including content-block lists); anything
+that is not JSON at all is scanned as plain text.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Iterable
+
+from devin_internals.parsers.sessions import MessageNode
+
+TESTS = "tests"
+COMMIT = "commit"
+FILE = "file"
+PUSH = "push"
+HTTP = "http"
+URL = "url"
+
+CLAIM_KINDS = (TESTS, COMMIT, FILE, PUSH, HTTP, URL)
+
+_RUNNER_TOKENS = (
+    "cargo test",
+    "go test",
+    "npm test",
+    "pytest",
+    "unittest",
+    "vitest",
+    "jest",
+    "mocha",
+    "tox",
+    "nose",
+)
+
+_TESTISH = r"(?:tests?|test suite|pytest|unittest|jest|vitest|mocha|tox|nose)"
+_TESTS_RE = re.compile(
+    rf"\b{_TESTISH}\b[^.\n]*?\b(?:pass(?:ed|es|ing)?|green|succeed(?:ed|ing)?|ok)\b"
+    rf"|\b\d+\s+(?:{_TESTISH}\s+)?passed\b",
+    re.IGNORECASE,
+)
+_COMMIT_WORD_RE = re.compile(r"\bcommit(?:s|ted|ting)?\b", re.IGNORECASE)
+_HEX_RE = re.compile(r"\b[0-9a-f]{7,40}\b")
+_PUSH_RE = re.compile(
+    r"\bgit\s+push\b"
+    r"|\bpush(?:ed|ing)?\b[^.\n]*\b(?:origin|upstream|remote|main|master|branch)\b",
+    re.IGNORECASE,
+)
+_FILE_VERB_RE = re.compile(
+    r"\b(?:created?|wrote|written|writing|added|adding|updated?|updating|"
+    r"modified|changed|edited|generated|saved|deleted|removed|renamed|moved)\b",
+    re.IGNORECASE,
+)
+_PATH_TOKEN_RE = re.compile(
+    r"(?<![\w/.-])(?:[A-Za-z]:[\\/])?[A-Za-z0-9_-]+(?:[\\/][\w.@-]+)*"
+    r"\.[A-Za-z0-9]{2,10}\b"
+)
+
+# "the API returned 200" / "endpoint responded with 404" / "HTTP 500" /
+# "status code: 200" — the captured group is the claimed status code.
+_URL_RE = re.compile(r"https://[\w.-]+(?::\d+)?(?:/[\w./?%&=~#+-]*)?")
+
+_HTTP_CLAIM_RES = (
+    re.compile(
+        r"\b(?:returned|responded|replied|answered|came\s+back)\b"
+        r"[^.\n]*?\b([1-5]\d{2})\b",
+        re.IGNORECASE,
+    ),
+    re.compile(r"\bHTTP\s+(?:status\s*(?:code)?\s*)?([1-5]\d{2})\b",
+               re.IGNORECASE),
+    re.compile(
+        r"\bstatus\s*(?:code)?\s*(?:of|was|is|:|=)\s*([1-5]\d{2})\b",
+        re.IGNORECASE,
+    ),
+)
+
+_SKIP_ROLES = {"user", "human"}
+_MAX_EXCERPT = 160
+
+
+@dataclass(frozen=True)
+class Claim:
+    """One deliverable claim found in an agent message.
+
+    ``detail`` carries the checkable part: the test-runner name (or
+    ``"tests"`` when generic), the commit hash, the claimed file path, or
+    ``"push"``. ``excerpt`` is the source line, truncated — evidence, not
+    full row content.
+    """
+
+    kind: str
+    detail: str
+    session_id: str
+    node_id: int
+    excerpt: str
+
+
+def _decode_message(chat_message: str) -> tuple[str | None, str]:
+    """Return ``(role, text)``. Non-JSON payloads become ``(None, raw)``."""
+    try:
+        data = json.loads(chat_message)
+    except (json.JSONDecodeError, TypeError):
+        return None, chat_message
+    if isinstance(data, str):
+        return None, data
+    if not isinstance(data, dict):
+        return None, ""
+    role = data.get("role") or data.get("author") or data.get("sender")
+    return (str(role).lower() if role is not None else None), _text_of(data)
+
+
+def _text_of(data: dict[str, Any]) -> str:
+    for key in ("text", "content", "message", "body", "markdown"):
+        value = data.get(key)
+        if isinstance(value, str):
+            return value
+        if isinstance(value, list):
+            parts = [
+                block.get("text", "")
+                for block in value
+                if isinstance(block, dict)
+            ]
+            text = "\n".join(p for p in parts if p)
+            if text:
+                return text
+    return ""
+
+
+def _excerpt(line: str) -> str:
+    line = " ".join(line.split())
+    return line if len(line) <= _MAX_EXCERPT else line[: _MAX_EXCERPT - 1] + "…"
+
+
+def _normalize_path(token: str) -> str:
+    return token.replace("\\", "/").rstrip(".,;:!?)]}>")
+
+
+def _runner_in(line: str) -> str | None:
+    low = line.lower()
+    for token in _RUNNER_TOKENS:  # multi-word runners first
+        if re.search(rf"\b{re.escape(token)}\b", low):
+            return token
+    return None
+
+
+def _claims_in_line(line: str) -> Iterable[tuple[str, str]]:
+    if _TESTS_RE.search(line):
+        yield TESTS, _runner_in(line) or TESTS
+    if _COMMIT_WORD_RE.search(line):
+        for m in _HEX_RE.finditer(line):
+            yield COMMIT, m.group(0).lower()
+    if _PUSH_RE.search(line):
+        yield PUSH, PUSH
+    if _FILE_VERB_RE.search(line):
+        for m in _PATH_TOKEN_RE.finditer(line):
+            yield FILE, _normalize_path(m.group(0))
+    for rx in _HTTP_CLAIM_RES:
+        for m in rx.finditer(line):
+            yield HTTP, m.group(1)
+    if re.search(
+        r"\b(?:deployed|live|hosted|available|published|running)\b", line, re.I
+    ):
+        for m in _URL_RE.finditer(line):
+            yield URL, m.group(0).rstrip(".,);'\"")
+
+
+def _collect(
+    session_id: str,
+    node_id: int,
+    text: str,
+    claims: list[Claim],
+    seen: set[tuple[str, str]],
+) -> None:
+    for line in text.splitlines():
+        for kind, detail in _claims_in_line(line):
+            if not detail or (kind, detail) in seen:
+                continue
+            seen.add((kind, detail))
+            claims.append(
+                Claim(
+                    kind=kind,
+                    detail=detail,
+                    session_id=session_id,
+                    node_id=node_id,
+                    excerpt=_excerpt(line),
+                )
+            )
+
+
+def extract_claims(nodes: Iterable[MessageNode]) -> list[Claim]:
+    """All claims across ``nodes``, deduplicated by ``(kind, detail)``."""
+    claims: list[Claim] = []
+    seen: set[tuple[str, str]] = set()
+    for node in nodes:
+        role, text = _decode_message(node.chat_message)
+        if role in _SKIP_ROLES:
+            continue
+        _collect(node.session_id, node.node_id, text, claims, seen)
+    return claims
+
+
+def claims_from_pairs(
+    pairs: Iterable[tuple[str | None, str]],
+    session_id: str,
+) -> list[Claim]:
+    """Claims over pre-decoded ``(role, text)`` pairs — the adapter input.
+
+    ``node_id`` is synthesized per pair so excerpts remain traceable back
+    to their position in the transcript.
+    """
+    claims: list[Claim] = []
+    seen: set[tuple[str, str]] = set()
+    for node_id, (role, text) in enumerate(pairs, 1):
+        if role in _SKIP_ROLES:
+            continue
+        _collect(session_id, node_id, text, claims, seen)
+    return claims