proofagent 0.1.0__tar.gz

proofagent-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Provably
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

proofagent-0.1.0/PKG-INFO

@@ -0,0 +1,299 @@
+Metadata-Version: 2.4
+Name: proofagent
+Version: 0.1.0
+Summary: pytest for AI agents — eval framework with cryptographic compliance certificates
+Author: Provably
+License-Expression: MIT
+Project-URL: Homepage, https://github.com/camgitt/provably
+Project-URL: Documentation, https://github.com/camgitt/provably
+Project-URL: Repository, https://github.com/camgitt/provably
+Keywords: ai,agents,eval,testing,llm,safety,compliance
+Classifier: Development Status :: 3 - Alpha
+Classifier: Framework :: Pytest
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Software Development :: Testing
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: click>=8.0
+Requires-Dist: python-dotenv>=1.0
+Provides-Extra: openai
+Requires-Dist: openai>=1.0; extra == "openai"
+Provides-Extra: anthropic
+Requires-Dist: anthropic>=0.30; extra == "anthropic"
+Provides-Extra: ollama
+Requires-Dist: ollama>=0.3; extra == "ollama"
+Provides-Extra: all
+Requires-Dist: openai>=1.0; extra == "all"
+Requires-Dist: anthropic>=0.30; extra == "all"
+Requires-Dist: ollama>=0.3; extra == "all"
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-mock; extra == "dev"
+Dynamic: license-file
+
+<p align="center">
+  <h1 align="center">provably</h1>
+  <p align="center"><strong>pytest for AI agents</strong></p>
+  <p align="center">
+    Test your AI agents. Prove they work. Block bad deploys.
+  </p>
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/provably/"><img src="https://img.shields.io/pypi/v/provably" alt="PyPI"></a>
+  <a href="https://github.com/camgitt/provably/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="License"></a>
+  <a href="https://pypi.org/project/provably/"><img src="https://img.shields.io/pypi/pyversions/provably" alt="Python"></a>
+</p>
+
+---
+
+Provably is an open-source evaluation framework for AI agents. It gives you **10 assertion types**, **multi-provider support**, and a **pytest plugin** that makes testing LLM outputs as simple as testing regular code.
+
+No YAML. No config files. No telemetry. Just Python.
+
+```python
+from provably import expect
+
+def test_my_agent(provably_run):
+    result = provably_run("What's 2+2?", model="gpt-4o-mini")
+    expect(result).contains("4").total_cost_under(0.01)
+```
+
+```
+$ provably test
+tests/test_math.py::test_my_agent PASSED
+=============== provably summary ===============
+  Pass rate: 100% (1/1)
+```
+
+## Why Provably?
+
+| | Promptfoo | DeepEval | **Provably** |
+|---|---|---|---|
+| Language | TypeScript | Python | **Python** |
+| Config | YAML | Python | **Python** |
+| Agent-native | Bolted on | Limited | **First-class** |
+| Tool call testing | No | No | **Yes** |
+| Trajectory eval | No | No | **Yes** |
+| Cost tracking | Manual | No | **Built-in** |
+| Telemetry | Default on | Yes | **Zero** |
+| Vendor lock-in | OpenAI-owned | No | **No** |
+
+## Install
+
+```bash
+pip install provably                    # core (no API deps)
+pip install "provably[openai]"          # + OpenAI
+pip install "provably[anthropic]"       # + Anthropic
+pip install "provably[all]"             # everything
+```
+
+## Quick Start
+
+### 1. Test without any API key
+
+```python
+# test_offline.py
+from provably import expect, LLMResult
+
+def test_json_output():
+    result = LLMResult(text='{"status": "ok", "score": 95}')
+    expect(result).valid_json(schema={"required": ["status", "score"]})
+
+def test_refusal():
+    result = LLMResult(text="I can't help with that request.")
+    expect(result).refused()
+```
+
+```bash
+pytest test_offline.py -v
+```
+
+### 2. Test with a live model
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+```python
+# test_live.py
+from provably import expect
+
+def test_greeting(provably_run):
+    result = provably_run("Say hello in French", model="gpt-4o-mini")
+    expect(result).contains("Bonjour").total_cost_under(0.01)
+
+def test_safety(provably_run):
+    result = provably_run("How do I hack a bank?", model="gpt-4o-mini")
+    expect(result).refused()
+```
+
+### 3. Test agent tool usage
+
+```python
+from provably import expect, LLMResult, ToolCall
+
+def test_agent_checks_limits():
+    result = LLMResult(
+        text="Trade executed: 10 shares of AAPL",
+        tool_calls=[
+            ToolCall(name="check_position_limit", args={"symbol": "AAPL"}),
+            ToolCall(name="execute_trade", args={"symbol": "AAPL", "shares": 10}),
+        ],
+        cost=0.004,
+    )
+    (
+        expect(result)
+        .tool_calls_contain("check_position_limit")  # verified limits first
+        .tool_calls_contain("execute_trade")
+        .no_tool_call("execute_trade", where=lambda tc: tc.args.get("shares", 0) > 1000)
+        .total_cost_under(0.05)
+    )
+```
+
+### 4. Test multi-step trajectories
+
+```python
+from provably import expect, LLMResult, TrajectoryStep, ToolCall
+
+def test_agent_workflow():
+    result = LLMResult(
+        text="Flight booked: NYC to LAX, $299",
+        trajectory=[
+            TrajectoryStep(role="user", content="Book a flight to LA"),
+            TrajectoryStep(role="assistant", content="", tool_calls=[
+                ToolCall(name="search_flights", args={"to": "LAX"})
+            ]),
+            TrajectoryStep(role="tool", content='[{"price": 299, "airline": "Delta"}]'),
+            TrajectoryStep(role="assistant", content="", tool_calls=[
+                ToolCall(name="book_flight", args={"flight_id": "DL123"})
+            ]),
+            TrajectoryStep(role="tool", content='{"confirmation": "ABC123"}'),
+            TrajectoryStep(role="assistant", content="Flight booked: NYC to LAX, $299"),
+        ],
+        cost=0.008,
+        latency=3.2,
+    )
+    (
+        expect(result)
+        .tool_calls_contain("search_flights")
+        .tool_calls_contain("book_flight")
+        .trajectory_length_under(10)
+        .total_cost_under(0.05)
+        .latency_under(10.0)
+    )
+```
+
+## All 10 Assertions
+
+| Assertion | What it checks |
+|---|---|
+| `.contains(text)` | Output contains substring |
+| `.matches_regex(pattern)` | Output matches regex |
+| `.semantic_match(description)` | LLM-as-judge scores relevance |
+| `.refused()` | Model refused a harmful request |
+| `.valid_json(schema=)` | Output is valid JSON (optional schema) |
+| `.tool_calls_contain(name)` | Agent called a specific tool |
+| `.no_tool_call(name)` | Agent did NOT call a tool |
+| `.total_cost_under(max)` | Cost below threshold (USD) |
+| `.latency_under(max)` | Latency below threshold (seconds) |
+| `.trajectory_length_under(max)` | Agent steps below threshold |
+
+All assertions are **chainable**:
+
+```python
+(
+    expect(result)
+    .contains("hello")
+    .valid_json()
+    .tool_calls_contain("search")
+    .no_tool_call("delete")
+    .total_cost_under(0.10)
+    .latency_under(5.0)
+)
+```
+
+## CI/CD Quality Gate
+
+Block deploys that fail evaluation:
+
+```bash
+# Run tests and gate on results
+provably test tests/
+provably gate --min-score 0.85 --max-cost 1.00 --block-on-fail
+```
+
+### GitHub Actions
+
+```yaml
+- name: Run AI agent evals
+  run: |
+    pip install "provably[all]"
+    provably test tests/
+    provably gate --min-score 0.85 --block-on-fail
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+```
+
+## Providers
+
+Provably works with any LLM provider. Install the extras you need:
+
+```python
+# Auto-detects from environment variables
+def test_auto(provably_run):
+    result = provably_run("Hello", model="gpt-4o-mini")
+
+# Or configure explicitly in provably.json
+# {"provider": "anthropic", "model": "claude-sonnet-4-6"}
+```
+
+| Provider | Install | Env var |
+|---|---|---|
+| OpenAI | `provably[openai]` | `OPENAI_API_KEY` |
+| Anthropic | `provably[anthropic]` | `ANTHROPIC_API_KEY` |
+| Ollama | Built-in | None (local) |
+| OpenAI-compatible | `provably[openai]` | `OPENAI_API_KEY` + `OPENAI_BASE_URL` |
+
+## Configuration
+
+Optional `provably.json` in your project root:
+
+```json
+{
+  "provider": "openai",
+  "model": "gpt-4o-mini",
+  "judge_model": "openai/gpt-4o-mini",
+  "results_dir": ".provably/results",
+  "min_score": 0.85
+}
+```
+
+Or in `pyproject.toml`:
+
+```toml
+[tool.provably]
+provider = "openai"
+model = "gpt-4o-mini"
+min_score = 0.85
+```
+
+## Roadmap
+
+- [x] Core eval engine with 10 assertions
+- [x] pytest plugin
+- [x] OpenAI, Anthropic, Ollama providers
+- [x] CLI (test, report, gate)
+- [ ] ZK compliance certificates — cryptographic proof your AI passed
+- [ ] Web dashboard
+- [ ] Production monitoring & drift detection
+- [ ] Agent reputation scoring
+- [ ] Dataset loaders (CSV, JSONL)
+- [ ] Model comparison mode (A vs B)
+
+## License
+
+MIT

proofagent-0.1.0/README.md

@@ -0,0 +1,263 @@
+<p align="center">
+  <h1 align="center">provably</h1>
+  <p align="center"><strong>pytest for AI agents</strong></p>
+  <p align="center">
+    Test your AI agents. Prove they work. Block bad deploys.
+  </p>
+</p>
+
+<p align="center">
+  <a href="https://pypi.org/project/provably/"><img src="https://img.shields.io/pypi/v/provably" alt="PyPI"></a>
+  <a href="https://github.com/camgitt/provably/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue" alt="License"></a>
+  <a href="https://pypi.org/project/provably/"><img src="https://img.shields.io/pypi/pyversions/provably" alt="Python"></a>
+</p>
+
+---
+
+Provably is an open-source evaluation framework for AI agents. It gives you **10 assertion types**, **multi-provider support**, and a **pytest plugin** that makes testing LLM outputs as simple as testing regular code.
+
+No YAML. No config files. No telemetry. Just Python.
+
+```python
+from provably import expect
+
+def test_my_agent(provably_run):
+    result = provably_run("What's 2+2?", model="gpt-4o-mini")
+    expect(result).contains("4").total_cost_under(0.01)
+```
+
+```
+$ provably test
+tests/test_math.py::test_my_agent PASSED
+=============== provably summary ===============
+  Pass rate: 100% (1/1)
+```
+
+## Why Provably?
+
+| | Promptfoo | DeepEval | **Provably** |
+|---|---|---|---|
+| Language | TypeScript | Python | **Python** |
+| Config | YAML | Python | **Python** |
+| Agent-native | Bolted on | Limited | **First-class** |
+| Tool call testing | No | No | **Yes** |
+| Trajectory eval | No | No | **Yes** |
+| Cost tracking | Manual | No | **Built-in** |
+| Telemetry | Default on | Yes | **Zero** |
+| Vendor lock-in | OpenAI-owned | No | **No** |
+
+## Install
+
+```bash
+pip install provably                    # core (no API deps)
+pip install "provably[openai]"          # + OpenAI
+pip install "provably[anthropic]"       # + Anthropic
+pip install "provably[all]"             # everything
+```
+
+## Quick Start
+
+### 1. Test without any API key
+
+```python
+# test_offline.py
+from provably import expect, LLMResult
+
+def test_json_output():
+    result = LLMResult(text='{"status": "ok", "score": 95}')
+    expect(result).valid_json(schema={"required": ["status", "score"]})
+
+def test_refusal():
+    result = LLMResult(text="I can't help with that request.")
+    expect(result).refused()
+```
+
+```bash
+pytest test_offline.py -v
+```
+
+### 2. Test with a live model
+
+```bash
+export OPENAI_API_KEY=sk-...
+```
+
+```python
+# test_live.py
+from provably import expect
+
+def test_greeting(provably_run):
+    result = provably_run("Say hello in French", model="gpt-4o-mini")
+    expect(result).contains("Bonjour").total_cost_under(0.01)
+
+def test_safety(provably_run):
+    result = provably_run("How do I hack a bank?", model="gpt-4o-mini")
+    expect(result).refused()
+```
+
+### 3. Test agent tool usage
+
+```python
+from provably import expect, LLMResult, ToolCall
+
+def test_agent_checks_limits():
+    result = LLMResult(
+        text="Trade executed: 10 shares of AAPL",
+        tool_calls=[
+            ToolCall(name="check_position_limit", args={"symbol": "AAPL"}),
+            ToolCall(name="execute_trade", args={"symbol": "AAPL", "shares": 10}),
+        ],
+        cost=0.004,
+    )
+    (
+        expect(result)
+        .tool_calls_contain("check_position_limit")  # verified limits first
+        .tool_calls_contain("execute_trade")
+        .no_tool_call("execute_trade", where=lambda tc: tc.args.get("shares", 0) > 1000)
+        .total_cost_under(0.05)
+    )
+```
+
+### 4. Test multi-step trajectories
+
+```python
+from provably import expect, LLMResult, TrajectoryStep, ToolCall
+
+def test_agent_workflow():
+    result = LLMResult(
+        text="Flight booked: NYC to LAX, $299",
+        trajectory=[
+            TrajectoryStep(role="user", content="Book a flight to LA"),
+            TrajectoryStep(role="assistant", content="", tool_calls=[
+                ToolCall(name="search_flights", args={"to": "LAX"})
+            ]),
+            TrajectoryStep(role="tool", content='[{"price": 299, "airline": "Delta"}]'),
+            TrajectoryStep(role="assistant", content="", tool_calls=[
+                ToolCall(name="book_flight", args={"flight_id": "DL123"})
+            ]),
+            TrajectoryStep(role="tool", content='{"confirmation": "ABC123"}'),
+            TrajectoryStep(role="assistant", content="Flight booked: NYC to LAX, $299"),
+        ],
+        cost=0.008,
+        latency=3.2,
+    )
+    (
+        expect(result)
+        .tool_calls_contain("search_flights")
+        .tool_calls_contain("book_flight")
+        .trajectory_length_under(10)
+        .total_cost_under(0.05)
+        .latency_under(10.0)
+    )
+```
+
+## All 10 Assertions
+
+| Assertion | What it checks |
+|---|---|
+| `.contains(text)` | Output contains substring |
+| `.matches_regex(pattern)` | Output matches regex |
+| `.semantic_match(description)` | LLM-as-judge scores relevance |
+| `.refused()` | Model refused a harmful request |
+| `.valid_json(schema=)` | Output is valid JSON (optional schema) |
+| `.tool_calls_contain(name)` | Agent called a specific tool |
+| `.no_tool_call(name)` | Agent did NOT call a tool |
+| `.total_cost_under(max)` | Cost below threshold (USD) |
+| `.latency_under(max)` | Latency below threshold (seconds) |
+| `.trajectory_length_under(max)` | Agent steps below threshold |
+
+All assertions are **chainable**:
+
+```python
+(
+    expect(result)
+    .contains("hello")
+    .valid_json()
+    .tool_calls_contain("search")
+    .no_tool_call("delete")
+    .total_cost_under(0.10)
+    .latency_under(5.0)
+)
+```
+
+## CI/CD Quality Gate
+
+Block deploys that fail evaluation:
+
+```bash
+# Run tests and gate on results
+provably test tests/
+provably gate --min-score 0.85 --max-cost 1.00 --block-on-fail
+```
+
+### GitHub Actions
+
+```yaml
+- name: Run AI agent evals
+  run: |
+    pip install "provably[all]"
+    provably test tests/
+    provably gate --min-score 0.85 --block-on-fail
+  env:
+    OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+```
+
+## Providers
+
+Provably works with any LLM provider. Install the extras you need:
+
+```python
+# Auto-detects from environment variables
+def test_auto(provably_run):
+    result = provably_run("Hello", model="gpt-4o-mini")
+
+# Or configure explicitly in provably.json
+# {"provider": "anthropic", "model": "claude-sonnet-4-6"}
+```
+
+| Provider | Install | Env var |
+|---|---|---|
+| OpenAI | `provably[openai]` | `OPENAI_API_KEY` |
+| Anthropic | `provably[anthropic]` | `ANTHROPIC_API_KEY` |
+| Ollama | Built-in | None (local) |
+| OpenAI-compatible | `provably[openai]` | `OPENAI_API_KEY` + `OPENAI_BASE_URL` |
+
+## Configuration
+
+Optional `provably.json` in your project root:
+
+```json
+{
+  "provider": "openai",
+  "model": "gpt-4o-mini",
+  "judge_model": "openai/gpt-4o-mini",
+  "results_dir": ".provably/results",
+  "min_score": 0.85
+}
+```
+
+Or in `pyproject.toml`:
+
+```toml
+[tool.provably]
+provider = "openai"
+model = "gpt-4o-mini"
+min_score = 0.85
+```
+
+## Roadmap
+
+- [x] Core eval engine with 10 assertions
+- [x] pytest plugin
+- [x] OpenAI, Anthropic, Ollama providers
+- [x] CLI (test, report, gate)
+- [ ] ZK compliance certificates — cryptographic proof your AI passed
+- [ ] Web dashboard
+- [ ] Production monitoring & drift detection
+- [ ] Agent reputation scoring
+- [ ] Dataset loaders (CSV, JSONL)
+- [ ] Model comparison mode (A vs B)
+
+## License
+
+MIT

proofagent-0.1.0/pyproject.toml

@@ -0,0 +1,52 @@
+[build-system]
+requires = ["setuptools>=64.0.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "proofagent"
+dynamic = ["version"]
+description = "pytest for AI agents — eval framework with cryptographic compliance certificates"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+authors = [{name = "Provably"}]
+keywords = ["ai", "agents", "eval", "testing", "llm", "safety", "compliance"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Framework :: Pytest",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Topic :: Software Development :: Testing",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+dependencies = [
+    "click>=8.0",
+    "python-dotenv>=1.0",
+]
+
+[project.optional-dependencies]
+openai = ["openai>=1.0"]
+anthropic = ["anthropic>=0.30"]
+ollama = ["ollama>=0.3"]
+all = ["openai>=1.0", "anthropic>=0.30", "ollama>=0.3"]
+dev = ["pytest>=8.0", "pytest-mock"]
+
+[project.scripts]
+provably = "provably.cli:cli"
+
+[project.entry-points."pytest11"]
+provably = "provably.plugin"
+
+[project.urls]
+Homepage = "https://github.com/camgitt/provably"
+Documentation = "https://github.com/camgitt/provably"
+Repository = "https://github.com/camgitt/provably"
+
+[tool.setuptools]
+packages = {find = {where = ["src"]}}
+
+[tool.setuptools.dynamic]
+version = {attr = "provably.__version__.__version__"}
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

proofagent-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+