devin-qa-pack 0.1.2__tar.gz → 0.2.0__tar.gz

{devin_qa_pack-0.1.2 → devin_qa_pack-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: devin-qa-pack
-Version: 0.1.2
+Version: 0.2.0
 Summary: QA audit for Devin sessions: verifies deliverable claims (tests, commits, files, pushes) against tool_call_state ground truth.
 Author: Icaro0310
 License: MIT
@@ -173,6 +173,30 @@ evidence and the source excerpt for every checked claim.
 `$XDG_DATA_HOME/devin/cli/sessions.db` on Linux (default
 `~/.local/share/devin/cli/sessions.db`). Always read-only.
 
+### Surfaces (`--surface`)
+
+`audit`/`report` are surface-agnostic. Without an explicit
+`--sessions-db` the default `--surface auto` discovers **every**
+supported agent store — Devin CLI plus Copilot CLI
+(`$COPILOT_DIR`/`$COPILOT_HOME`/`~/.copilot` session-state), Codex CLI
+(`$CODEX_HOME`/`~/.codex` state db + rollout JSONL) and VS Code chat
+(`User/workspaceStorage|globalStorage` chat sessions, incl.
+`.vscode-server`). A surface whose store is absent simply contributes
+nothing — discovery never errors on a missing store.
+
+```bash
+devin-qa-pack audit --all                      # every detected surface
+devin-qa-pack audit --surface copilot-cli --all
+devin-qa-pack audit --surface auto --session vscode-chat:abc123
+```
+
+In provider mode session ids are namespaced `surface:native_id`
+(e.g. `codex-cli:019ae…`); `--session` accepts a bare native prefix or a
+`surface:`-scoped ref. Passing `--sessions-db` *alone* keeps the Devin
+fast path (bare ids, no other surfaces) — pass `--surface` to combine
+a pinned store with the rest. The `qa_audit` MCP tool takes the same
+`surface` argument.
+
 No Devin installed? Try it on a synthetic fixture:
 
 ```bash

{devin_qa_pack-0.1.2 → devin_qa_pack-0.2.0}/README.md

@@ -158,6 +158,30 @@ evidence and the source excerpt for every checked claim.
 `$XDG_DATA_HOME/devin/cli/sessions.db` on Linux (default
 `~/.local/share/devin/cli/sessions.db`). Always read-only.
 
+### Surfaces (`--surface`)
+
+`audit`/`report` are surface-agnostic. Without an explicit
+`--sessions-db` the default `--surface auto` discovers **every**
+supported agent store — Devin CLI plus Copilot CLI
+(`$COPILOT_DIR`/`$COPILOT_HOME`/`~/.copilot` session-state), Codex CLI
+(`$CODEX_HOME`/`~/.codex` state db + rollout JSONL) and VS Code chat
+(`User/workspaceStorage|globalStorage` chat sessions, incl.
+`.vscode-server`). A surface whose store is absent simply contributes
+nothing — discovery never errors on a missing store.
+
+```bash
+devin-qa-pack audit --all                      # every detected surface
+devin-qa-pack audit --surface copilot-cli --all
+devin-qa-pack audit --surface auto --session vscode-chat:abc123
+```
+
+In provider mode session ids are namespaced `surface:native_id`
+(e.g. `codex-cli:019ae…`); `--session` accepts a bare native prefix or a
+`surface:`-scoped ref. Passing `--sessions-db` *alone* keeps the Devin
+fast path (bare ids, no other surfaces) — pass `--surface` to combine
+a pinned store with the rest. The `qa_audit` MCP tool takes the same
+`surface` argument.
+
 No Devin installed? Try it on a synthetic fixture:
 
 ```bash

{devin_qa_pack-0.1.2 → devin_qa_pack-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "devin-qa-pack"
-version = "0.1.2"
+version = "0.2.0"
 description = "QA audit for Devin sessions: verifies deliverable claims (tests, commits, files, pushes) against tool_call_state ground truth."
 readme = "README.md"
 license = { text = "MIT" }

devin_qa_pack-0.2.0/src/devin_qa_pack/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.0"

devin_qa_pack-0.2.0/src/devin_qa_pack/adapters/base.py

@@ -0,0 +1,46 @@
+"""The normalized session view every adapter produces."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from devin_qa_pack.claims import Claim, extract_claims
+from devin_qa_pack.evidence import SessionEvidence
+from devin_qa_pack.verify import ParsedToolCall, parse_tool_calls
+
+
+@dataclass(frozen=True)
+class SourceSession:
+    """One session's claims plus its recorded tool calls.
+
+    ``raw_call_count`` is the number of evidence rows *before* decoding —
+    when rows exist but none parse, ground truth is unreadable and claims
+    resolve ``unverifiable`` rather than ``disputed``.
+    """
+
+    source: str  # adapter name: "devin-cli" | "aider" | "claude-code" | ...
+    session_id: str
+    title: str | None
+    working_directory: str
+    claims: tuple[Claim, ...]
+    calls: tuple[ParsedToolCall, ...]
+    raw_call_count: int
+
+
+def source_view(
+    evidence: SessionEvidence,
+    session_id: str | None = None,
+    title: str | None = None,
+) -> SourceSession:
+    """Bridge :class:`SessionEvidence` into the audit-side view: claims
+    extracted from ``messages``, payloads parsed from ``tool_calls``."""
+    sid = session_id or evidence.native_id
+    return SourceSession(
+        source=evidence.surface,
+        session_id=sid,
+        title=title,
+        working_directory=evidence.working_dir or "",
+        claims=tuple(extract_claims(evidence.messages, sid)),
+        calls=tuple(parse_tool_calls(evidence.tool_calls)),
+        raw_call_count=len(evidence.tool_calls),
+    )

devin_qa_pack-0.2.0/src/devin_qa_pack/adapters/devin.py

@@ -0,0 +1,23 @@
+"""The Devin adapter — the native ``sessions.db`` source.
+
+Claims come from ``message_nodes`` (agent-role text); ground truth comes
+from ``tool_call_state`` rows decoded via devin-internals-spec parsers —
+both through the devin-cli evidence provider.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+
+from devin_internals.parsers import SessionsStore
+from devin_internals.parsers.sessions import Session
+
+from devin_qa_pack.adapters.base import SourceSession, source_view
+from devin_qa_pack.evidence.devin_cli import session_evidence
+
+
+def session_view(store: SessionsStore, session: Session) -> SourceSession:
+    """The store-backed view: bare session id + title (the legacy path —
+    provider audits namespace ids as ``devin-cli:<id>`` instead)."""
+    view = source_view(session_evidence(store, session), session.id)
+    return replace(view, title=session.title)

{devin_qa_pack-0.1.2 → devin_qa_pack-0.2.0}/src/devin_qa_pack/claims.py

@@ -1,25 +1,22 @@
-"""Extract deliverable claims from a session's ``message_nodes``.
+"""Extract deliverable claims from a session's evidence messages.
 
 A *claim* is an assertion the agent made about delivered work: tests
 passed, a commit exists, a file was created/changed, work was pushed.
 Only agent-role messages are scanned — a user prompt quoting the same
 words is not a delivery claim.
 
-``chat_message`` payloads are marked *unstable* in devin-internals-spec
-SCHEMA.md, so decoding is defensive: JSON dicts are checked for the
-usual role/text/content keys (including content-block lists); anything
-that is not JSON at all is scanned as plain text.
+The input is the canonical :class:`EvidenceMessage` — store-specific
+payload decoding (``chat_message`` JSON for the Devin store) happens in
+the evidence providers, so this module only ever sees plain role+text.
 """
 
 from __future__ import annotations
 
-import json
 import re
 from collections.abc import Iterable
 from dataclasses import dataclass
-from typing import Any
 
-from devin_internals.parsers.sessions import MessageNode
+from devin_qa_pack.evidence import EvidenceMessage, normalize_role
 
 TESTS = "tests"
 COMMIT = "commit"
@@ -101,41 +98,10 @@ class Claim:
     kind: str
     detail: str
     session_id: str
-    node_id: int
+    node_id: int | str
     excerpt: str
 
 
-def _decode_message(chat_message: str) -> tuple[str | None, str]:
-    """Return ``(role, text)``. Non-JSON payloads become ``(None, raw)``."""
-    try:
-        data = json.loads(chat_message)
-    except (json.JSONDecodeError, TypeError):
-        return None, chat_message
-    if isinstance(data, str):
-        return None, data
-    if not isinstance(data, dict):
-        return None, ""
-    role = data.get("role") or data.get("author") or data.get("sender")
-    return (str(role).lower() if role is not None else None), _text_of(data)
-
-
-def _text_of(data: dict[str, Any]) -> str:
-    for key in ("text", "content", "message", "body", "markdown"):
-        value = data.get(key)
-        if isinstance(value, str):
-            return value
-        if isinstance(value, list):
-            parts = [
-                block.get("text", "")
-                for block in value
-                if isinstance(block, dict)
-            ]
-            text = "\n".join(p for p in parts if p)
-            if text:
-                return text
-    return ""
-
-
 def _excerpt(line: str) -> str:
     line = " ".join(line.split())
     return line if len(line) <= _MAX_EXCERPT else line[: _MAX_EXCERPT - 1] + "…"
@@ -176,7 +142,7 @@ def _claims_in_line(line: str) -> Iterable[tuple[str, str]]:
 
 def _collect(
     session_id: str,
-    node_id: int,
+    node_id: int | str,
     text: str,
     claims: list[Claim],
     seen: set[tuple[str, str]],
@@ -197,15 +163,28 @@ def _collect(
             )
 
 
-def extract_claims(nodes: Iterable[MessageNode]) -> list[Claim]:
-    """All claims across ``nodes``, deduplicated by ``(kind, detail)``."""
+def extract_claims(
+    messages: Iterable[EvidenceMessage], session_id: str = ""
+) -> list[Claim]:
+    """All claims across ``messages``, deduplicated by ``(kind, detail)``.
+
+    ``Claim.node_id`` cites the surface's real node id
+    (``EvidenceMessage.source_id``, e.g. the devin-cli
+    ``message_nodes.node_id``) when the provider supplies one; otherwise
+    the message's positional index — same convention as
+    :func:`claims_from_pairs`.
+    """
     claims: list[Claim] = []
     seen: set[tuple[str, str]] = set()
-    for node in nodes:
-        role, text = _decode_message(node.chat_message)
-        if role in _SKIP_ROLES:
+    for position, message in enumerate(messages, 1):
+        if message.role in _SKIP_ROLES:
             continue
-        _collect(node.session_id, node.node_id, text, claims, seen)
+        node_id = (
+            message.source_id
+            if message.source_id is not None
+            else position
+        )
+        _collect(session_id, node_id, message.text, claims, seen)
     return claims
 
 
@@ -213,15 +192,13 @@ def claims_from_pairs(
     pairs: Iterable[tuple[str | None, str]],
     session_id: str,
 ) -> list[Claim]:
-    """Claims over pre-decoded ``(role, text)`` pairs — the adapter input.
-
-    ``node_id`` is synthesized per pair so excerpts remain traceable back
-    to their position in the transcript.
-    """
-    claims: list[Claim] = []
-    seen: set[tuple[str, str]] = set()
-    for node_id, (role, text) in enumerate(pairs, 1):
-        if role in _SKIP_ROLES:
-            continue
-        _collect(session_id, node_id, text, claims, seen)
-    return claims
+    """Claims over pre-decoded ``(role, text)`` pairs — the adapter input."""
+    return extract_claims(
+        (
+            EvidenceMessage(
+                role=normalize_role(role), text=text, ts_ms=None
+            )
+            for role, text in pairs
+        ),
+        session_id,
+    )

{devin_qa_pack-0.1.2 → devin_qa_pack-0.2.0}/src/devin_qa_pack/cli.py

@@ -1,11 +1,14 @@
 """Thin CLI wrapper — all logic lives in the library modules.
 
-``devin-qa-pack audit`` reads a sessions.db (explicit or auto-detected),
-verifies deliverable claims against tool_call_state ground truth and
-prints a verdict per session. Read-only. Exit codes: 0 = every audited
-session PASS · 1 = audit ran, some session PARTIAL/UNVERIFIED · 2 = the
-audit could not run (missing/unknown store, unknown session, or no
-session matched the scope — e.g. an empty db or ``--within``).
+``devin-qa-pack audit`` reads session evidence through surface
+providers (``--surface``, default ``auto`` = every detected store:
+devin-cli sessions.db, copilot-cli, codex-cli, vscode-chat), verifies
+deliverable claims against tool-call ground truth and prints a verdict
+per session. An explicit ``--sessions-db`` alone keeps the Devin fast
+path (bare ids, exactly the pre-surface behavior). Read-only. Exit
+codes: 0 = every audited session PASS · 1 = audit ran, some session
+PARTIAL/UNVERIFIED · 2 = the audit could not run (missing/unknown
+store, unknown session, or no session matched the scope).
 
 ``devin-qa-pack session-end`` is the SessionEnd hook variant: it audits
 only the session that just ended, writes the verdict to a JSON side
@@ -30,6 +33,13 @@ from devin_internals.parsers import SessionsStore
 from devin_internals.parsers.sessions import Session
 from devin_internals.schema import SchemaError
 
+from devin_qa_pack import __version__
+from devin_qa_pack.evidence import (
+    Surface,
+    collect_evidences,
+    match_evidence,
+)
+from devin_qa_pack.evidence.devin_cli import session_evidence
 from devin_qa_pack.html_report import render_html
 from devin_qa_pack.intent import (
     FLAGGED,
@@ -37,14 +47,17 @@ from devin_qa_pack.intent import (
     audit_intent,
     intent_dict,
     render_intent,
+    workspace_dir_names,
 )
 from devin_qa_pack.paths import default_sessions_db
 from devin_qa_pack.report import (
     PASS,
     audit_all,
+    audit_evidence,
     audit_session,
     audit_source,
     audits_payload,
+    filter_evidences,
     render_text,
 )
 from devin_qa_pack.session_end import run_session_end, summary_line
@@ -56,12 +69,26 @@ def _build_parser() -> argparse.ArgumentParser:
         description="Audit Devin sessions: verify deliverable claims "
         "(tests, commits, files, pushes) against tool_call_state ground truth.",
     )
+    parser.add_argument(
+        "--version", action="version", version=f"%(prog)s {__version__}"
+    )
     sub = parser.add_subparsers(dest="command")
     audit = sub.add_parser("audit", help="verify claims per session")
     audit.add_argument(
         "--sessions-db",
         metavar="PATH",
-        help="path to sessions.db (default: auto-detect Devin data dir)",
+        help="path to sessions.db (default: auto-detect Devin data dir); "
+        "on its own this stays the Devin fast path — pass --surface to "
+        "combine it with other surfaces",
+    )
+    audit.add_argument(
+        "--surface",
+        choices=Surface.CHOICES,
+        default=None,
+        metavar="S",
+        help="evidence surface: devin-cli | copilot-cli | codex-cli | "
+        "vscode-chat | auto (default: auto — every detected store; "
+        "provider-mode ids are namespaced as surface:native_id)",
     )
     scope = audit.add_mutually_exclusive_group(required=True)
     scope.add_argument("--session", metavar="ID", help="audit one session "
@@ -108,6 +135,15 @@ def _build_parser() -> argparse.ArgumentParser:
         metavar="PATH",
         help="path to sessions.db (default: auto-detect Devin data dir)",
     )
+    report.add_argument(
+        "--surface",
+        choices=Surface.CHOICES,
+        default=None,
+        metavar="S",
+        help="evidence surface to report on (default: auto — every "
+        "detected store; explicit --sessions-db alone keeps the Devin "
+        "fast path)",
+    )
     report.add_argument(
         "--out", metavar="PATH", default="qa-report.html",
         help="HTML output path (default: ./qa-report.html)")
@@ -223,6 +259,60 @@ def _audit_scope(store: SessionsStore, args: argparse.Namespace):
     return audit_all(store, limit=getattr(args, "limit", None), **kw)
 
 
+def _provider_mode(args: argparse.Namespace) -> bool:
+    """True when audit goes through evidence providers rather than the
+    legacy sessions.db path: an explicit ``--surface``, or auto mode
+    without a pinned ``--sessions-db``."""
+    return getattr(args, "surface", None) is not None or (
+        not getattr(args, "sessions_db", None)
+    )
+
+
+def _provider_audits(args: argparse.Namespace, kw: dict):
+    """``(audits, label)`` for provider mode; ``(None, None)`` after a
+    printed error."""
+    surface = args.surface or Surface.AUTO
+    evidences, warnings = collect_evidences(
+        surface, sessions_db=getattr(args, "sessions_db", None)
+    )
+    for warning in warnings:
+        print(f"warning: {warning}", file=sys.stderr)
+    session_arg = getattr(args, "session", None)
+    if session_arg:
+        found = match_evidence(evidences, session_arg)
+        if isinstance(found, list):
+            # ref (workspace-hash-bearing canonical id or ephemeral key)
+            # is selectable; surface:native_id alone can repeat.
+            refs = ", ".join(
+                e.ref or f"{e.surface}:{e.native_id}" for e in found
+            )
+            print(
+                f"error: session '{session_arg}' is ambiguous: {refs}",
+                file=sys.stderr,
+            )
+            return None, None
+        if found is None:
+            print(
+                f"error: unknown session '{session_arg}'", file=sys.stderr
+            )
+            return None, None
+        evidences = [found]
+    else:
+        evidences = filter_evidences(
+            evidences,
+            within=getattr(args, "within", None),
+            limit=getattr(args, "limit", None),
+        )
+    db_arg = getattr(args, "sessions_db", None)
+    label = (
+        str(Path(db_arg).expanduser())
+        if db_arg
+        else ("surfaces: auto" if surface == Surface.AUTO
+              else f"surface: {surface}")
+    )
+    return [audit_evidence(e, **kw) for e in evidences], label
+
+
 def _load_transcript(path_arg: str, fmt: str | None, cwd: str | None):
     """Resolve ``--transcript`` into a SourceSession via the adapters."""
     from devin_qa_pack.adapters import aider, claude_code
@@ -286,6 +376,12 @@ def _cmd_audit(args: argparse.Namespace) -> int:
             )
         ]
         db = Path(args.transcript).expanduser()
+    elif _provider_mode(args):
+        kw = {"online": args.online,
+              "allow_domains": tuple(args.allow_domain)}
+        audits, db = _provider_audits(args, kw)
+        if audits is None:
+            return 2
     else:
         db, store = _open_store(args.sessions_db)
         if store is None:
@@ -315,13 +411,20 @@ def _cmd_audit(args: argparse.Namespace) -> int:
 
 
 def _cmd_report(args: argparse.Namespace) -> int:
-    db, store = _open_store(args.sessions_db)
-    if store is None:
-        return 2
-    with store:
-        audits = _audit_scope(store, args)
+    if _provider_mode(args):
+        kw = {"online": args.online,
+              "allow_domains": tuple(args.allow_domain)}
+        audits, db = _provider_audits(args, kw)
         if audits is None:
             return 2
+    else:
+        db, store = _open_store(args.sessions_db)
+        if store is None:
+            return 2
+        with store:
+            audits = _audit_scope(store, args)
+            if audits is None:
+                return 2
     out = Path(args.out).expanduser()
     try:
         out.write_text(render_html(audits, db), encoding="utf-8")
@@ -342,9 +445,8 @@ def _cmd_intent(args: argparse.Namespace) -> int:
         if session is None:
             return 2
         audit = audit_intent(
-            session,
-            store.message_nodes(session.id),
-            store.tool_call_state(session.id),
+            session_evidence(store, session),
+            repo_names=workspace_dir_names(session),
         )
     if args.json:
         print(json.dumps(intent_dict(audit), indent=2))