assay-it 0.8.0__tar.gz

assay_it-0.8.0/.gitignore

@@ -0,0 +1,31 @@
+/target
+archive/
+**/*.rs.bk
+Cargo.lock
+.eval
+.DS_Store
+*.log
+junit.xml
+sarif.json
+run.json
+otel.jsonl
+out/
+venv/
+.venv/
+__pycache__/
+*.pyc
+.pytest_cache/
+*.egg-info/
+dist/
+build/
+.env
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# Release & Smoke Test
+.venv_smoke/
+smoke_test_ws/
+verdict_sdk.egg-info/
+assay_python_sdk.egg-info/

assay_it-0.8.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 verdict-dev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

assay_it-0.8.0/PKG-INFO

@@ -0,0 +1,174 @@
+Metadata-Version: 2.4
+Name: assay-it
+Version: 0.8.0
+Summary: High-performance evaluation framework for LLM agents
+Project-URL: Homepage, https://assay.dev
+Project-URL: Repository, https://github.com/Rul1an/assay
+Project-URL: Issues, https://github.com/Rul1an/assay/issues
+Author-email: Assay <hello@assay.dev>
+License: MIT License
+        
+        Copyright (c) 2025 verdict-dev
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+License-File: LICENSE
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Software Development :: Quality Assurance
+Requires-Python: >=3.9
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: dev
+Requires-Dist: black; extra == 'dev'
+Requires-Dist: isort; extra == 'dev'
+Requires-Dist: mypy; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-asyncio; extra == 'dev'
+Provides-Extra: openai
+Requires-Dist: openai>=1.0.0; extra == 'openai'
+Description-Content-Type: text/markdown
+
+# Assay Python SDK
+
+**Record deterministic traces from your Python agents for regression gating.**
+
+## 🚀 Golden Quickstart
+
+The fastest way to regression test your AI agent.
+
+### 1. Installation
+```bash
+pip install assay
+```
+
+### 2. Record (`record.py`)
+Run your agent through the SDK to capture a trace. Pass your tool functions to `tool_executors` so Assay can record their inputs and outputs.
+
+```python
+import os
+import openai
+from assay_sdk import TraceWriter, record_chat_completions_with_tools
+
+# 1. Setup Client & Tools
+client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "mock"))
+TOOLS = [{
+    "type": "function",
+    "function": {
+        "name": "GetWeather",
+        "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}
+    }
+}]
+
+# 2. Define Execution Logic (The "Real" Code)
+def get_weather(args):
+    return {"temp": 22, "location": args.get("location")}
+
+# 3. Record the Loop
+writer = TraceWriter("traces/quickstart.jsonl")
+result = record_chat_completions_with_tools(
+    writer=writer,
+    client=client,
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Weather in Tokyo?"}],
+    tools=TOOLS,
+    tool_executors={"GetWeather": get_weather}, # Link schema -> function
+    episode_id="weather_demo",
+    test_id="weather_check"
+)
+print(f"Agent Final Answer: {result['content']}")
+```
+
+### 3. Configure (`assay.yaml`)
+Tell Assay what to check.
+
+```yaml
+version: 1
+model: "trace"
+tests:
+  - id: weather_check
+    input:
+      prompt: "Weather in Tokyo?" # Matches the recorded prompt
+    expected:
+      type: regex_match
+      pattern: ".*" # Pass if any content returned (baseline check)
+```
+
+### 4. Verify
+Run the regression gate. This replays your trace against the recorded tool outputs to ensure determinism.
+
+```bash
+# Verify strictly (fails if any tool call arg changed even slightly)
+assay ci --config assay.yaml --trace-file traces/quickstart.jsonl --replay-strict --db :memory:
+```
+
+---
+
+## 🌊 Advanced: Streaming support
+Capture streaming responses while maintaining tool call execution.
+
+```python
+from assay_sdk import record_chat_completions_stream_with_tools
+
+# ... setup client & writer ...
+
+result = record_chat_completions_stream_with_tools(
+    writer=writer,
+    # ... args ...
+    stream=True # SDK handles chunk aggregation automatically
+    # tool_executors={...} # Required if tools are used
+)
+```
+*Note: The hybrid wrapper (`record_chat_completions_stream_with_tools`) streams the thinking tokens to the user, executes tools, and then performs a standard follow-up call.*
+
+## 🛡️ Advanced: Privacy & Redaction
+Protect sensitive data (PII, API keys) from ever hitting the trace file.
+
+```python
+from assay_sdk import TraceWriter, make_redactor
+
+# Create a redactor that scrubs keys and regex patterns
+redactor = make_redactor(
+    key_denylist={"authorization", "password", "api_key"},
+    patterns=[r"sk-[a-zA-Z0-9]{20,}"] # Mask OpenAI keys
+)
+
+# Attach to writer - happens automatically on write
+writer = TraceWriter("traces/secure.jsonl", redact_fn=redactor)
+```
+
+## ⚡ Async Support
+Native `async` support for high-throughput applications (FastAPI, etc.) is available via the `assay_sdk.async_openai` submodule. It provides full parity with the sync API, including loop and streaming support.
+
+## ❓ Troubleshooting
+
+### `E_TRACE_EPISODE_MISSING`
+**Cause**: The `test_id` or `episode_id` in your trace doesn't match what `assay ci` expected from its config (or implicit default).
+**Fix**: Ensure your `assay.yaml` test IDs match the `test_id` passed to `record_chat_completions...`.
+
+### "Duplicate prompt in strict replay"
+**Cause**: You ran `record.py` twice without cleaning the trace file, so it contains two identical episodes. `assay ci` in strict mode doesn't know which one to replay.
+**Fix**:
+1. Truncate the file before recording: `trace_path = "traces/my_trace.jsonl"; open(trace_path, 'w').close()`.
+2. Use unique `episode_id`s (e.g. UUIDs) for every run.

assay_it-0.8.0/README.md

@@ -0,0 +1,122 @@
+# Assay Python SDK
+
+**Record deterministic traces from your Python agents for regression gating.**
+
+## 🚀 Golden Quickstart
+
+The fastest way to regression test your AI agent.
+
+### 1. Installation
+```bash
+pip install assay
+```
+
+### 2. Record (`record.py`)
+Run your agent through the SDK to capture a trace. Pass your tool functions to `tool_executors` so Assay can record their inputs and outputs.
+
+```python
+import os
+import openai
+from assay_sdk import TraceWriter, record_chat_completions_with_tools
+
+# 1. Setup Client & Tools
+client = openai.OpenAI(api_key=os.environ.get("OPENAI_API_KEY", "mock"))
+TOOLS = [{
+    "type": "function",
+    "function": {
+        "name": "GetWeather",
+        "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}
+    }
+}]
+
+# 2. Define Execution Logic (The "Real" Code)
+def get_weather(args):
+    return {"temp": 22, "location": args.get("location")}
+
+# 3. Record the Loop
+writer = TraceWriter("traces/quickstart.jsonl")
+result = record_chat_completions_with_tools(
+    writer=writer,
+    client=client,
+    model="gpt-4o",
+    messages=[{"role": "user", "content": "Weather in Tokyo?"}],
+    tools=TOOLS,
+    tool_executors={"GetWeather": get_weather}, # Link schema -> function
+    episode_id="weather_demo",
+    test_id="weather_check"
+)
+print(f"Agent Final Answer: {result['content']}")
+```
+
+### 3. Configure (`assay.yaml`)
+Tell Assay what to check.
+
+```yaml
+version: 1
+model: "trace"
+tests:
+  - id: weather_check
+    input:
+      prompt: "Weather in Tokyo?" # Matches the recorded prompt
+    expected:
+      type: regex_match
+      pattern: ".*" # Pass if any content returned (baseline check)
+```
+
+### 4. Verify
+Run the regression gate. This replays your trace against the recorded tool outputs to ensure determinism.
+
+```bash
+# Verify strictly (fails if any tool call arg changed even slightly)
+assay ci --config assay.yaml --trace-file traces/quickstart.jsonl --replay-strict --db :memory:
+```
+
+---
+
+## 🌊 Advanced: Streaming support
+Capture streaming responses while maintaining tool call execution.
+
+```python
+from assay_sdk import record_chat_completions_stream_with_tools
+
+# ... setup client & writer ...
+
+result = record_chat_completions_stream_with_tools(
+    writer=writer,
+    # ... args ...
+    stream=True # SDK handles chunk aggregation automatically
+    # tool_executors={...} # Required if tools are used
+)
+```
+*Note: The hybrid wrapper (`record_chat_completions_stream_with_tools`) streams the thinking tokens to the user, executes tools, and then performs a standard follow-up call.*
+
+## 🛡️ Advanced: Privacy & Redaction
+Protect sensitive data (PII, API keys) from ever hitting the trace file.
+
+```python
+from assay_sdk import TraceWriter, make_redactor
+
+# Create a redactor that scrubs keys and regex patterns
+redactor = make_redactor(
+    key_denylist={"authorization", "password", "api_key"},
+    patterns=[r"sk-[a-zA-Z0-9]{20,}"] # Mask OpenAI keys
+)
+
+# Attach to writer - happens automatically on write
+writer = TraceWriter("traces/secure.jsonl", redact_fn=redactor)
+```
+
+## ⚡ Async Support
+Native `async` support for high-throughput applications (FastAPI, etc.) is available via the `assay_sdk.async_openai` submodule. It provides full parity with the sync API, including loop and streaming support.
+
+## ❓ Troubleshooting
+
+### `E_TRACE_EPISODE_MISSING`
+**Cause**: The `test_id` or `episode_id` in your trace doesn't match what `assay ci` expected from its config (or implicit default).
+**Fix**: Ensure your `assay.yaml` test IDs match the `test_id` passed to `record_chat_completions...`.
+
+### "Duplicate prompt in strict replay"
+**Cause**: You ran `record.py` twice without cleaning the trace file, so it contains two identical episodes. `assay ci` in strict mode doesn't know which one to replay.
+**Fix**:
+1. Truncate the file before recording: `trace_path = "traces/my_trace.jsonl"; open(trace_path, 'w').close()`.
+2. Use unique `episode_id`s (e.g. UUIDs) for every run.

assay_it-0.8.0/assay/__init__.py

@@ -0,0 +1,24 @@
+"""
+Verdict Python SDK: deterministic trace recording for regression gating.
+"""
+
+from .clock import FrozenClock, SystemClock
+from .openai_instrumentor import (record_chat_completions,
+                                  record_chat_completions_with_tools)
+from .openai_stream_wrapper import (record_chat_completions_stream,
+                                    record_chat_completions_stream_with_tools)
+from .recorder import EpisodeRecorder
+from .redaction import make_redactor
+from .writer import TraceWriter
+
+__all__ = [
+    "TraceWriter",
+    "EpisodeRecorder",
+    "SystemClock",
+    "FrozenClock",
+    "record_chat_completions",
+    "record_chat_completions_with_tools",
+    "make_redactor",
+    "record_chat_completions_stream",
+    "record_chat_completions_stream_with_tools",
+]

assay_it-0.8.0/assay/__main__.py

@@ -0,0 +1,25 @@
+import sys
+
+from .doctor import run_doctor
+
+
+def main():
+    if len(sys.argv) < 2:
+        print("Usage: assay <command>")
+        print("Commands:")
+        print("  doctor   Run health checks")
+        sys.exit(1)
+
+    cmd = sys.argv[1]
+    if cmd == "doctor":
+        run_doctor()
+    elif cmd == "enforce":
+        # Placeholder for future Phase 3.1
+        print("Not implemented yet.")
+    else:
+        print(f"Unknown command: {cmd}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

assay_it-0.8.0/assay/artifacts.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from .result import CompareResult, EvalRun, ResultArtifacts
+
+
+class RunWriter:
+    """Manages persistence of EvalRuns and CompareResults to disk."""
+
+    def __init__(self, workdir: Path) -> None:
+        self.workdir = workdir
+        self.runs_dir = workdir / "runs"
+        self.runs_dir.mkdir(parents=True, exist_ok=True)
+
+    def write_run(self, run: EvalRun) -> ResultArtifacts:
+        """Persists EvalRun to .eval/runs/<run_id>/."""
+        run_dir = self.runs_dir / run.run_id
+        run_dir.mkdir(parents=True, exist_ok=True)
+
+        run_json = run_dir / "run.json"
+        results_jsonl = run_dir / "results.jsonl"
+        junit_xml = run_dir / "junit.xml"
+        summary_md = run_dir / "summary.md"
+        artifacts_json = run_dir / "artifacts.json"
+
+        self._write_json(run_json, run.to_dict())
+
+        with open(results_jsonl, "w", encoding="utf-8") as f:
+            for t in run.tests:
+                f.write(json.dumps(t.to_dict()) + "\n")
+
+        if run.artifacts.junit_xml:
+            with open(junit_xml, "w", encoding="utf-8") as f:
+                f.write(run.to_junit_xml())
+
+        with open(summary_md, "w", encoding="utf-8") as f:
+            f.write(run.to_github_summary())
+
+        index_data = {
+            "run_id": run.run_id,
+            "created_at_ms": run.created_at_ms,
+            "paths": {
+                "run_json": str(run_json.absolute()),
+                "results_jsonl": str(results_jsonl.absolute()),
+                "junit_xml": str(junit_xml.absolute()),
+                "summary_md": str(summary_md.absolute()),
+                "sarif": None,
+                "diff_json": None,
+                "trace_path": str(run.trace_path.absolute()),
+            },
+        }
+        self._write_json(artifacts_json, index_data)
+
+        # Return updated artifacts object reflecting true paths
+        # (Though EvalRun object passed in hasn't changed, callers should know the reliable paths)
+        return ResultArtifacts(
+            run_json=run_json,
+            results_jsonl=results_jsonl,
+            junit_xml=junit_xml,
+            sarif=None,
+            diff_json=None,
+            trace_path=run.trace_path,
+        )
+
+    def write_diff(self, result: CompareResult) -> None:
+        """Persists diff.json to the current run's directory."""
+        if not result.current_run_id:
+            return
+        run_dir = self.runs_dir / result.current_run_id
+        if not run_dir.exists():
+            run_dir.mkdir(parents=True, exist_ok=True)
+
+        # 1. diff.json
+        diff_file = run_dir / "diff.json"
+        self._write_json(diff_file, result.to_dict())
+
+        # 2. Update summary.md with comparison details
+        summary_path = run_dir / "summary.md"
+        with open(summary_path, "w", encoding="utf-8") as f:
+            f.write(result.to_github_summary())
+
+        # 3. Update artifacts.json
+        index_path = run_dir / "artifacts.json"
+        if index_path.exists():
+            with open(index_path, "r", encoding="utf-8") as f:
+                index_data = json.load(f)
+        else:
+            # Should exist, but fallback
+            index_data = {"run_id": result.current_run_id, "paths": {}}
+
+        index_data["paths"]["diff_json"] = str(diff_file.absolute())
+        self._write_json(index_path, index_data)
+
+    def _write_json(self, path: Path, data: Dict[str, Any]) -> None:
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)