qe-harness 0.1.0__tar.gz

qe_harness-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Shan Konduru
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

qe_harness-0.1.0/PKG-INFO

@@ -0,0 +1,143 @@
+Metadata-Version: 2.4
+Name: qe-harness
+Version: 0.1.0
+Summary: Black-box, five-stage validation of conversation-to-action AI assistants: intent, orchestration, MCP tool calls, backend state and user response
+Author: Shan Konduru
+License-Expression: MIT
+Keywords: testing,qa,llm,agents,mcp,evaluation,chatbot
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Quality Assurance
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: mcp>=2.3
+Requires-Dist: pydantic>=2.7
+Requires-Dist: pyyaml>=6
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: websockets>=12
+Provides-Extra: azure
+Requires-Dist: openai>=1.50; extra == "azure"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=2.4; extra == "anthropic"
+Provides-Extra: judge
+Requires-Dist: openai>=1.50; extra == "judge"
+Requires-Dist: anthropic>=2.4; extra == "judge"
+Dynamic: license-file
+
+# qe-harness
+
+Black-box, five-stage validation of conversation-to-action AI assistants.
+
+A chat-only test sees the final reply. `qe-harness` tests every turn at five stages, from the outside:
+
+**intent → orchestration → MCP tool calls → backend state → user response**
+
+It never imports the application under test. It signs in as test users and chats over the app's API, reads the
+app's per-turn traces, records every tool call independently through its own MCP proxy, and compares read-only
+snapshots of the business data before and after each turn. So it catches what a polite reply hides: a write the
+customer never approved, an amount no tool returned, a reply that says "done" when nothing was saved.
+
+## Install
+
+```bash
+pip install qe-harness            # core
+pip install "qe-harness[judge]"   # plus the optional LLM grader (Azure OpenAI or Anthropic)
+```
+
+Python 3.11 or newer.
+
+## Quick start
+
+```bash
+qe init          # writes qe.yaml (sandbox connection) and qe_catalog.yaml (test cases) in this folder
+# edit both files, then provide the two secrets the config names:
+#   QE_SANDBOX_TOKEN  sandbox control token
+#   QE_TEST_PASSWORD  test users' password
+# (environment variables, or a .env file next to qe.yaml)
+qe doctor        # connection checks: all green means testing can start
+qe run --open    # run every suite, evaluate each turn, open the HTML report
+```
+
+`qe` finds its configuration from `--config PATH`, then `$QE_CONFIG`, then `qe.yaml` in the working directory.
+Relative paths in it start from the folder holding the file (or from `base_dir:` if set).
+
+## What the app's sandbox must provide
+
+| Need | Endpoint or setting |
+|---|---|
+| Sign-in as test users | `POST /api/login` `{"username", "password"}`, session cookie |
+| Conversations | `POST /api/conversations` → `{"id"}`; WebSocket `/ws?conversation=<id>` with messages `{"type": "user", "text"}` and `{"type": "confirm", "key", "approved"}`; events `ready`, `awaiting_confirmation`, `assistant_message`, `error`, `turn_completed` |
+| Per-turn traces | `GET /api/conversations/<id>/traces`: intent (`primary_intent`, `intents`), plan steps, tool calls with arguments and outcomes |
+| File uploads (optional) | `POST /api/conversations/<id>/documents?filename=…` |
+| Control | `GET /sandbox/status` (versions, clock, toggles) and `POST /sandbox/reset`, with header `x-sandbox-token` |
+| Independent tool record | Start the MCP tool server through `python -m qe_harness.mcp_proxy --log-dir <dir> -- <server command>` |
+| Business data | A read-only SQLite replica the harness can snapshot |
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| `qe init [--dir D] [--force]` | Starter `qe.yaml` and `qe_catalog.yaml` |
+| `qe doctor [--quick]` | Network, control API, clock, versions, proxy toggle, reset, read-back, sign-in, round trip, trace link, tool capture |
+| `qe status` | Sandbox versions, clock and toggles |
+| `qe cases [--suite S]` | List catalog cases |
+| `qe run [--suite S…] [--case ID] [--repeats N] [--judge] [--open]` | Reset the sandbox, run the suites, evaluate every turn, write evidence and the HTML report |
+| `qe runs` | Recent runs |
+| `qe show RUN CASE` | Evidence for one case: message, approvals, intent, plan, trace and proxy tool calls, data changes, reply, verdicts |
+| `qe report RUN [--open]` | Rebuild the self-contained HTML report |
+| `qe coverage [RUN] [--trend]` | Intent coverage for a run, or per intent across runs |
+| `qe mcp serve` | Run the harness as an MCP server for AI assistants |
+
+## What is checked per turn
+
+| Stage | Checks |
+|---|---|
+| Intent | The router's primary intent is one of the expected intents |
+| Orchestration | Writes only after approval; no unexpected approval prompts; no repeated identical calls |
+| MCP tool call | Required calls succeeded with the expected arguments; forbidden calls absent; **trace and proxy agree** |
+| Backend state | Only the expected tables changed; SQL checks; **declined → nothing changed; data changed → a write succeeded** |
+| User response | Required facts present, forbidden phrases absent, optional LLM grader for consistency with what the tools did |
+
+Each failure records its first failing stage, so developers know where to look. With `--repeats N`, every case gets
+a pass rate with a 95% Wilson range and unstable cases are reported as flaky.
+
+## Intent coverage
+
+List the app's intent families under `intents:` in `qe.yaml`. Each report shows, per family, the cases that test it,
+their pass rate and a status (covered, thin, failing, not tested), plus a families × runs grid of how coverage
+changes over time. `qe coverage --trend` shows the same in the terminal.
+
+## In AI assistants (MCP)
+
+```bash
+claude mcp add qe-harness -- qe mcp serve
+```
+
+For Cursor or VS Code, add a stdio server with command `qe` and args `["mcp", "serve"]`.
+
+Tools: `sandbox_status`, `sandbox_doctor`, `list_cases`, `get_case`, `run_tests` (changes state; capped per call),
+`get_run_status`, `list_runs`, `get_failure` (masked evidence, marked as untrusted data), `get_intent_coverage`,
+`get_intent_trend`. Prompt: `triage_run`.
+
+## LLM grader (optional)
+
+`qe run --judge` asks a model whether each reply is consistent with what the tools actually did. It uses its own
+settings, separate from the app's: `QE_JUDGE_PROVIDER` = `AZURE` (with `QE_JUDGE_AZURE_API_KEY`,
+`QE_JUDGE_AZURE_ENDPOINT`, `QE_JUDGE_AZURE_DEPLOYMENT`) or `ANTHROPIC` (with `ANTHROPIC_API_KEY`, optional
+`QE_JUDGE_MODEL`).
+
+## Status
+
+Alpha. Not yet included: fault injection, load and soak testing, simulated customers, robustness variant
+generation and a release gate.
+
+## License
+
+MIT

qe_harness-0.1.0/README.md

@@ -0,0 +1,110 @@
+# qe-harness
+
+Black-box, five-stage validation of conversation-to-action AI assistants.
+
+A chat-only test sees the final reply. `qe-harness` tests every turn at five stages, from the outside:
+
+**intent → orchestration → MCP tool calls → backend state → user response**
+
+It never imports the application under test. It signs in as test users and chats over the app's API, reads the
+app's per-turn traces, records every tool call independently through its own MCP proxy, and compares read-only
+snapshots of the business data before and after each turn. So it catches what a polite reply hides: a write the
+customer never approved, an amount no tool returned, a reply that says "done" when nothing was saved.
+
+## Install
+
+```bash
+pip install qe-harness            # core
+pip install "qe-harness[judge]"   # plus the optional LLM grader (Azure OpenAI or Anthropic)
+```
+
+Python 3.11 or newer.
+
+## Quick start
+
+```bash
+qe init          # writes qe.yaml (sandbox connection) and qe_catalog.yaml (test cases) in this folder
+# edit both files, then provide the two secrets the config names:
+#   QE_SANDBOX_TOKEN  sandbox control token
+#   QE_TEST_PASSWORD  test users' password
+# (environment variables, or a .env file next to qe.yaml)
+qe doctor        # connection checks: all green means testing can start
+qe run --open    # run every suite, evaluate each turn, open the HTML report
+```
+
+`qe` finds its configuration from `--config PATH`, then `$QE_CONFIG`, then `qe.yaml` in the working directory.
+Relative paths in it start from the folder holding the file (or from `base_dir:` if set).
+
+## What the app's sandbox must provide
+
+| Need | Endpoint or setting |
+|---|---|
+| Sign-in as test users | `POST /api/login` `{"username", "password"}`, session cookie |
+| Conversations | `POST /api/conversations` → `{"id"}`; WebSocket `/ws?conversation=<id>` with messages `{"type": "user", "text"}` and `{"type": "confirm", "key", "approved"}`; events `ready`, `awaiting_confirmation`, `assistant_message`, `error`, `turn_completed` |
+| Per-turn traces | `GET /api/conversations/<id>/traces`: intent (`primary_intent`, `intents`), plan steps, tool calls with arguments and outcomes |
+| File uploads (optional) | `POST /api/conversations/<id>/documents?filename=…` |
+| Control | `GET /sandbox/status` (versions, clock, toggles) and `POST /sandbox/reset`, with header `x-sandbox-token` |
+| Independent tool record | Start the MCP tool server through `python -m qe_harness.mcp_proxy --log-dir <dir> -- <server command>` |
+| Business data | A read-only SQLite replica the harness can snapshot |
+
+## Commands
+
+| Command | What it does |
+|---|---|
+| `qe init [--dir D] [--force]` | Starter `qe.yaml` and `qe_catalog.yaml` |
+| `qe doctor [--quick]` | Network, control API, clock, versions, proxy toggle, reset, read-back, sign-in, round trip, trace link, tool capture |
+| `qe status` | Sandbox versions, clock and toggles |
+| `qe cases [--suite S]` | List catalog cases |
+| `qe run [--suite S…] [--case ID] [--repeats N] [--judge] [--open]` | Reset the sandbox, run the suites, evaluate every turn, write evidence and the HTML report |
+| `qe runs` | Recent runs |
+| `qe show RUN CASE` | Evidence for one case: message, approvals, intent, plan, trace and proxy tool calls, data changes, reply, verdicts |
+| `qe report RUN [--open]` | Rebuild the self-contained HTML report |
+| `qe coverage [RUN] [--trend]` | Intent coverage for a run, or per intent across runs |
+| `qe mcp serve` | Run the harness as an MCP server for AI assistants |
+
+## What is checked per turn
+
+| Stage | Checks |
+|---|---|
+| Intent | The router's primary intent is one of the expected intents |
+| Orchestration | Writes only after approval; no unexpected approval prompts; no repeated identical calls |
+| MCP tool call | Required calls succeeded with the expected arguments; forbidden calls absent; **trace and proxy agree** |
+| Backend state | Only the expected tables changed; SQL checks; **declined → nothing changed; data changed → a write succeeded** |
+| User response | Required facts present, forbidden phrases absent, optional LLM grader for consistency with what the tools did |
+
+Each failure records its first failing stage, so developers know where to look. With `--repeats N`, every case gets
+a pass rate with a 95% Wilson range and unstable cases are reported as flaky.
+
+## Intent coverage
+
+List the app's intent families under `intents:` in `qe.yaml`. Each report shows, per family, the cases that test it,
+their pass rate and a status (covered, thin, failing, not tested), plus a families × runs grid of how coverage
+changes over time. `qe coverage --trend` shows the same in the terminal.
+
+## In AI assistants (MCP)
+
+```bash
+claude mcp add qe-harness -- qe mcp serve
+```
+
+For Cursor or VS Code, add a stdio server with command `qe` and args `["mcp", "serve"]`.
+
+Tools: `sandbox_status`, `sandbox_doctor`, `list_cases`, `get_case`, `run_tests` (changes state; capped per call),
+`get_run_status`, `list_runs`, `get_failure` (masked evidence, marked as untrusted data), `get_intent_coverage`,
+`get_intent_trend`. Prompt: `triage_run`.
+
+## LLM grader (optional)
+
+`qe run --judge` asks a model whether each reply is consistent with what the tools actually did. It uses its own
+settings, separate from the app's: `QE_JUDGE_PROVIDER` = `AZURE` (with `QE_JUDGE_AZURE_API_KEY`,
+`QE_JUDGE_AZURE_ENDPOINT`, `QE_JUDGE_AZURE_DEPLOYMENT`) or `ANTHROPIC` (with `ANTHROPIC_API_KEY`, optional
+`QE_JUDGE_MODEL`).
+
+## Status
+
+Alpha. Not yet included: fault injection, load and soak testing, simulated customers, robustness variant
+generation and a release gate.
+
+## License
+
+MIT

qe_harness-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+# Package definition for publishing the QE harness on its own as `qe-harness`.
+# Build with: python tools/build_harness.py  (stages qe_harness/ next to this file, then builds sdist + wheel)
+[build-system]
+requires = ["setuptools>=77"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "qe-harness"
+dynamic = ["version"]
+description = "Black-box, five-stage validation of conversation-to-action AI assistants: intent, orchestration, MCP tool calls, backend state and user response"
+readme = "README.md"
+requires-python = ">=3.11"
+license = "MIT"
+license-files = ["LICENSE"]
+authors = [{ name = "Shan Konduru" }]
+keywords = ["testing", "qa", "llm", "agents", "mcp", "evaluation", "chatbot"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Software Development :: Quality Assurance",
+    "Topic :: Software Development :: Testing",
+]
+dependencies = [
+    "mcp>=2.3",
+    "pydantic>=2.7",
+    "pyyaml>=6",
+    "python-dotenv>=1.0",
+    "websockets>=12",
+]
+
+[project.optional-dependencies]
+# The optional LLM grader (qe run --judge) calls Azure OpenAI or Anthropic.
+azure = ["openai>=1.50"]
+anthropic = ["anthropic>=2.4"]
+judge = ["openai>=1.50", "anthropic>=2.4"]
+
+[project.scripts]
+qe = "qe_harness.cli:main"
+
+[tool.setuptools]
+packages = ["qe_harness"]
+
+[tool.setuptools.package-data]
+qe_harness = ["templates/*.yaml"]
+
+[tool.setuptools.dynamic]
+version = { attr = "qe_harness.__version__" }

qe_harness-0.1.0/qe_harness/__init__.py

@@ -0,0 +1,12 @@
+"""QE validation harness (prototype).
+
+Tests a conversation-to-action assistant from the outside, at five stages:
+intent, orchestration, MCP tool calls, backend state and user response.
+
+The harness never imports the application: it signs in and chats over the app's
+public API, reads traces through the sandbox's trace API, captures tool calls
+through its own recording proxy, and reads business data through a read-only
+connection - exactly what a QE partner gets from a client sandbox.
+"""
+
+__version__ = "0.1.0"

qe_harness-0.1.0/qe_harness/__main__.py

@@ -0,0 +1,3 @@
+from qe_harness.cli import main
+
+main()

qe_harness-0.1.0/qe_harness/capture.py

@@ -0,0 +1,101 @@
+"""Evidence capture outside the conversation: proxy tool-call log and read-only state snapshots."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from qe_harness.config import HarnessConfig
+
+
+# ------------------------------------------------------------------ proxy log
+def proxy_calls(config: HarnessConfig, customer_id: str, start: float, end: float) -> list[dict[str, Any]]:
+    """Tool calls the recording proxy saw for one customer in a time window, with their outcome."""
+    requests: dict[Any, dict] = {}
+    responses: dict[Any, dict] = {}
+    if not config.proxy_log_dir.exists():
+        return []
+    for path in config.proxy_log_dir.glob(f"{customer_id}-*.jsonl"):
+        for line in path.read_text(encoding="utf-8").splitlines():
+            try:
+                entry = json.loads(line)
+            except ValueError:
+                continue
+            if not (start - 0.5 <= entry["ts"] <= end + 0.5):
+                continue
+            msg = entry["message"]
+            key = (path.name, msg.get("id"))
+            if entry["direction"] == "request":
+                requests[key] = {"ts": entry["ts"], **msg}
+            else:
+                responses[key] = msg
+    calls = []
+    for key, req in sorted(requests.items(), key=lambda kv: kv[1]["ts"]):
+        params = req.get("params", {})
+        resp = responses.get(key, {})
+        result = resp.get("result", {}) or {}
+        text = "\n".join(c.get("text", "") for c in result.get("content", []) if isinstance(c, dict))
+        calls.append({"name": params.get("name"), "arguments": params.get("arguments") or {},
+                      "is_error": bool(result.get("isError") or result.get("is_error") or "error" in resp),
+                      "result": text[:20000], "ts": req["ts"]})
+    return calls
+
+
+# ------------------------------------------------------------------ snapshots
+def snapshot(config: HarnessConfig) -> dict[str, dict[str, dict[str, Any]]]:
+    """Business data keyed by table and primary key, through a read-only connection."""
+    uri = f"file:{config.sor_path.as_posix()}?mode=ro"
+    con = sqlite3.connect(uri, uri=True)
+    con.row_factory = sqlite3.Row
+    out: dict[str, dict[str, dict[str, Any]]] = {}
+    try:
+        for table in config.sor_tables:
+            ignore = set(config.sor_ignore_columns.get(table, []))
+            try:
+                rows = con.execute(f'SELECT * FROM "{table}"').fetchall()
+            except sqlite3.OperationalError:
+                out[table] = {}
+                continue
+            data = {}
+            for r in rows:
+                d = dict(r)
+                for col in ignore & set(d):  # keep a fingerprint instead of large content
+                    val = d.pop(col)
+                    d[col + "_sha256"] = hashlib.sha256(val if isinstance(val, bytes) else str(val).encode()).hexdigest()[:16]
+                data[str(d.get("id", json.dumps(d, sort_keys=True, default=str)))] = d
+            out[table] = data
+    finally:
+        con.close()
+    return out
+
+
+def diff(before: dict, after: dict) -> dict[str, Any]:
+    """Per-table added / removed / changed rows between two snapshots."""
+    result: dict[str, Any] = {}
+    for table in sorted(set(before) | set(after)):
+        b, a = before.get(table, {}), after.get(table, {})
+        added = [a[k] for k in a.keys() - b.keys()]
+        removed = [b[k] for k in b.keys() - a.keys()]
+        changed = {}
+        for k in a.keys() & b.keys():
+            cols = {c: [b[k].get(c), a[k].get(c)] for c in set(a[k]) | set(b[k]) if b[k].get(c) != a[k].get(c)}
+            if cols:
+                changed[k] = cols
+        if added or removed or changed:
+            result[table] = {"added": added, "removed": removed, "changed": changed}
+    return result
+
+
+def query(config: HarnessConfig, sql: str) -> Any:
+    """One value (or rows) from the read-only replica, for state checks."""
+    con = sqlite3.connect(f"file:{config.sor_path.as_posix()}?mode=ro", uri=True)
+    try:
+        rows = con.execute(sql).fetchall()
+    finally:
+        con.close()
+    if len(rows) == 1 and len(rows[0]) == 1:
+        return rows[0][0]
+    return [list(r) for r in rows]

qe_harness-0.1.0/qe_harness/catalog.py

@@ -0,0 +1,49 @@
+"""Test catalog: suites of ordered cases, each a turn in one conversation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+@dataclass
+class Suite:
+    name: str
+    user: str
+    customer_id: str
+    starting_state: str
+    cases: list[dict[str, Any]]
+
+
+def load_catalog(path: Path) -> list[Suite]:
+    data = yaml.safe_load(Path(path).read_text(encoding="utf-8"))
+    return [Suite(s["suite"], s["user"], s["customer_id"], s.get("starting_state", ""), s["cases"]) for s in data["suites"]]
+
+
+def select(suites: list[Suite], suite_names: list[str] | None = None, case_id: str | None = None) -> list[Suite]:
+    """Choose suites; with case_id, run that case's suite up to and including the case (earlier turns set it up)."""
+    if case_id:
+        for s in suites:
+            ids = [c["id"] for c in s.cases]
+            if case_id in ids:
+                return [Suite(s.name, s.user, s.customer_id, s.starting_state, s.cases[: ids.index(case_id) + 1])]
+        raise KeyError(f"Case {case_id} not found in the catalog")
+    if suite_names:
+        wanted = {n.upper() for n in suite_names}
+        chosen = [s for s in suites if s.name.upper() in wanted]
+        missing = wanted - {s.name.upper() for s in chosen}
+        if missing:
+            raise KeyError(f"Unknown suite(s): {', '.join(sorted(missing))}")
+        return chosen
+    return suites
+
+
+def find_case(suites: list[Suite], case_id: str) -> tuple[Suite, dict[str, Any]]:
+    for s in suites:
+        for c in s.cases:
+            if c["id"] == case_id:
+                return s, c
+    raise KeyError(case_id)