agentproof-sim 0.1.1__py3-none-any.whl

agentproof_sim-0.1.1.dist-info/METADATA

@@ -0,0 +1,407 @@
+Metadata-Version: 2.5
+Name: agentproof-sim
+Version: 0.1.1
+Summary: Stateful simulation and fault-injection testing for AI agents that take actions through tools.
+Project-URL: Homepage, https://github.com/rohansonawane/agentproof
+Project-URL: Documentation, https://github.com/rohansonawane/agentproof/tree/main/docs
+Project-URL: Issues, https://github.com/rohansonawane/agentproof/issues
+Author-email: Rohan Sonawane <rohansonawane28@gmail.com>
+License-Expression: Apache-2.0
+License-File: LICENSE
+Keywords: agents,fault-injection,reliability,simulation,testing
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.11
+Requires-Dist: click<9,>=8.1
+Requires-Dist: pydantic<3,>=2.12
+Requires-Dist: rich<16,>=13
+Provides-Extra: dev
+Requires-Dist: build<2,>=1.2; extra == 'dev'
+Requires-Dist: httpx2[brotli]<3,>=2.12; extra == 'dev'
+Requires-Dist: langchain<2,>=1.3; extra == 'dev'
+Requires-Dist: langgraph<2,>=1.2; extra == 'dev'
+Requires-Dist: mypy<2,>=1.13; extra == 'dev'
+Requires-Dist: openai-agents<1,>=0.22; extra == 'dev'
+Requires-Dist: pytest-asyncio<2,>=0.24; extra == 'dev'
+Requires-Dist: pytest-cov<8,>=5; extra == 'dev'
+Requires-Dist: pytest<10,>=8; extra == 'dev'
+Requires-Dist: ruff<1,>=0.8; extra == 'dev'
+Requires-Dist: twine<8,>=7; extra == 'dev'
+Provides-Extra: langchain
+Requires-Dist: langchain<2,>=1.3; extra == 'langchain'
+Requires-Dist: langgraph<2,>=1.2; extra == 'langchain'
+Provides-Extra: openai
+Requires-Dist: httpx2[brotli]<3,>=2.12; extra == 'openai'
+Requires-Dist: openai-agents<1,>=0.22; extra == 'openai'
+Provides-Extra: test
+Requires-Dist: pytest-asyncio<2,>=0.24; extra == 'test'
+Requires-Dist: pytest-cov<8,>=5; extra == 'test'
+Requires-Dist: pytest<10,>=8; extra == 'test'
+Description-Content-Type: text/markdown
+
+# AgentProof
+
+[![CI](https://github.com/rohansonawane/agentproof/actions/workflows/ci.yml/badge.svg)](https://github.com/rohansonawane/agentproof/actions/workflows/ci.yml)
+
+AgentProof is a Python testing framework for AI agents that take actions through tools.
+
+It helps you answer a practical release question:
+
+> If a tool call times out, returns bad data, gets retried, or receives stale state, does my agent still protect the real world?
+
+AgentProof creates an isolated simulated world, exposes tools to your agent, injects realistic faults, records the side effects that actually commit, and checks deterministic invariants over the result.
+
+It is useful for agents that can change things:
+
+- refunding payments;
+- booking or canceling appointments;
+- sending emails or support replies;
+- changing CRM records;
+- modifying files or repositories;
+- running internal operations or DevOps workflows.
+
+See [Real-World Use Cases](docs/use-cases.md) for concrete examples and invariants.
+See [Real-Project Evidence](docs/real-project-evidence.md) for pinned public
+agent-related projects that AgentProof has been run against.
+
+AgentProof does not prove that an agent is universally safe or correct. It gives you repeatable tests for dangerous action-level failures before they reach production.
+
+## Why This Exists
+
+Most agent evaluations look at text: Was the answer helpful? Did the model choose the right response?
+
+That is not enough for tool-using agents. The embarrassing failures are often stateful:
+
+- the refund succeeded, but the agent saw a timeout and refunded again;
+- the appointment was booked twice after a retry;
+- the approval record changed after the agent read it;
+- a malformed tool response caused the agent to act on the wrong amount;
+- a duplicate event caused the same email or webhook to be sent twice.
+
+AgentProof tests those failure modes by separating two things that production systems often blur:
+
+- the side effect committed;
+- the agent observed success.
+
+That separation is what lets `timeout_after_commit` catch duplicate refunds, bookings, and similar bugs.
+
+## Install
+
+```bash
+pip install agentproof-sim
+```
+
+The PyPI distribution is `agentproof-sim` because `agentproof` is already occupied by an unrelated package. The Python import and CLI remain stable:
+
+```python
+import agentproof
+```
+
+Optional framework adapters are installed separately:
+
+```bash
+pip install "agentproof-sim[openai]"
+pip install "agentproof-sim[langchain]"
+```
+
+## Five-Minute Quickstart
+
+Save this as `refund_demo.py` and run `python refund_demo.py`.
+
+The agent below is intentionally unsafe: if a refund commits but the response times out, it retries without an idempotency key.
+
+```python
+import asyncio
+from typing import Any
+
+from agentproof import AgentTest, World, invariant
+from agentproof.core.effects import EffectDraft
+from agentproof.mutations import TimeoutAfterCommit
+from agentproof.tools.definition import ToolOutcome
+
+
+async def refund_agent(user_input: str, tools: Any) -> str:
+    del user_input
+    order = await tools.call("get_order", order_id="123")
+    try:
+        await tools.call("refund_order", order_id="123", amount=order["total"])
+    except TimeoutError:
+        # Unsafe on purpose: this retry can duplicate the refund if the first
+        # tool call committed before the timeout reached the agent.
+        await tools.call("refund_order", order_id="123", amount=order["total"])
+    return "Done"
+
+
+suite = AgentTest(
+    agent=refund_agent,
+    adapter="native",
+    mutations=[TimeoutAfterCommit(target="refund_order", severity="high")],
+)
+
+
+@suite.scenario(name="refund_delivered_order")
+async def delivered_refund(world: World) -> None:
+    world.state["orders"] = {"123": {"id": "123", "status": "delivered", "total": 49.0}}
+    world.input("Refund order 123")
+
+    async def get_order(world: World, order_id: str) -> dict[str, Any]:
+        return dict(world.state["orders"][order_id])
+
+    async def refund_order(world: World, order_id: str, amount: float) -> ToolOutcome:
+        refund_id = world.next_id("refund")
+        return ToolOutcome(
+            value={"refund_id": refund_id},
+            effects=[
+                # Effects should describe work that actually committed in the
+                # simulated service. Do not infer them from tool calls alone.
+                EffectDraft(
+                    type="refund.created",
+                    operation="create",
+                    resource=f"order:{order_id}",
+                    data={"refund_id": refund_id, "order_id": order_id, "amount": amount},
+                )
+            ],
+        )
+
+    world.tools.register(
+        name="get_order",
+        description="Return an order.",
+        input_schema={
+            "type": "object",
+            "properties": {"order_id": {"type": "string"}},
+            "required": ["order_id"],
+        },
+        handler=get_order,
+        effect="read",
+    )
+    world.tools.register(
+        name="refund_order",
+        description="Refund an order.",
+        input_schema={
+            "type": "object",
+            "properties": {"order_id": {"type": "string"}, "amount": {"type": "number"}},
+            "required": ["order_id", "amount"],
+        },
+        handler=refund_order,
+        effect="financial",
+        idempotent=False,
+    )
+
+
+@invariant
+def no_double_refunds(world: World) -> None:
+    total = world.effects.sum(type="refund.created", where={"order_id": "123"}, field="amount")
+    assert total <= 49.0, f"refunded ${total:.2f}, expected <= $49.00"
+
+
+async def main() -> None:
+    result = await suite.run(store_artifacts=False)
+    failure = result.failures[0]
+    print(failure.status)
+    print(failure.violated_invariants)
+    print(sum(effect.data["amount"] for effect in failure.effects))
+
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+Expected output includes:
+
+```text
+INVARIANT_FAILURE
+['no_double_refunds']
+98.0
+```
+
+In plain English: the first refund happened, AgentProof hid the success response and returned a timeout, the agent retried, and the invariant caught that the customer received `$98` instead of `$49`.
+
+## How It Works
+
+AgentProof has five core concepts:
+
+- `World`: isolated simulated state, virtual tools, virtual clock, event queue, effect ledger, trace, and installed faults.
+- `Tool`: the action boundary exposed to an agent.
+- `Effect`: an explicit committed side effect such as `refund.created` or `appointment.booked`.
+- `Mutation`: a serializable fault such as `timeout_after_commit`, `missing_field`, or `duplicate_event`.
+- `Invariant`: deterministic Python logic that decides whether the final state is acceptable.
+
+The effect ledger is intentionally explicit. Tool handlers return `ToolOutcome` effects after simulated work commits. AgentProof does not guess side effects merely because a tool was called.
+
+## What AgentProof Can Catch
+
+AgentProof is designed for failures that happen around action boundaries:
+
+- duplicate side effects after retries;
+- partial success hidden behind a timeout;
+- malformed, missing, or stale tool responses;
+- rate limits and transient tool errors;
+- permission denial paths;
+- duplicated user requests;
+- delayed or duplicated scheduled events;
+- regressions where a previously idempotent action becomes unsafe.
+
+The most important pattern is retry-after-commit:
+
+```text
+1. Agent calls refund_order.
+2. The simulated refund commits.
+3. AgentProof raises TimeoutError instead of returning success.
+4. The agent retries.
+5. A second refund commits.
+6. The invariant fails.
+```
+
+## Using It Safely
+
+AgentProof runs your test code. It is not a sandbox for untrusted code.
+
+Use it against virtual tools, fakes, local simulators, or carefully isolated staging services. Do not point AgentProof tests at production refund, email, file deletion, deployment, payment, or account-management APIs.
+
+Recommended safety rules:
+
+- Replace destructive real tools with AgentProof virtual tools.
+- Keep live model tests opt-in.
+- Use least-privilege credentials when a live provider is required.
+- Never commit API keys, tokens, customer data, or production payloads.
+- Review JSON, JUnit, replay, and trace artifacts before sharing them.
+- Treat redaction as a safety net, not a full data-loss-prevention system.
+
+AgentProof redacts obvious secret fields such as `api_key`, `authorization`, `token`, `password`, and `secret` in traces and reports. It cannot know every sensitive business field in your domain. See [SECURITY.md](SECURITY.md) for the full security model.
+
+## Stable Mutations
+
+Stable in this technical preview:
+
+- `tool_timeout`
+- `timeout_after_commit`
+- `tool_error`
+- `tool_latency`
+- `rate_limited`
+- `malformed_response`
+- `missing_field`
+- `duplicate_user_request`
+- `stale_state`
+- `state_changed_after_read`
+- `permission_denied`
+- `delayed_event`
+- `duplicate_event`
+
+Experimental:
+
+- `duplicate_tool_result`, because agent-loop semantics differ by framework.
+- `reorder_tool_results`, currently rejected unless a future controlled parallel result scheduler is available.
+
+## Framework Integrations
+
+The core package has no OpenAI or LangChain dependency.
+
+- Native Python callables are supported by default.
+- OpenAI Agents SDK tools are wrapped as real SDK `FunctionTool` objects.
+- LangChain/LangGraph tools are wrapped as real LangChain `StructuredTool` objects and tested through a compiled LangGraph `StateGraph`/`ToolNode`.
+
+Live LLM tests are opt-in only:
+
+```bash
+# Set OPENAI_API_KEY through your shell, CI secret manager, or local secret manager first.
+AGENTPROOF_RUN_LIVE_TESTS=1 pytest -m live -v
+```
+
+Do not put API keys in source files, shell history, issue comments, or README examples. Prefer temporary environment variables or a secret manager.
+
+## Replay
+
+Deterministic native failures write `repro.json` artifacts containing schema version, seed, mutation spec, initial snapshot, trace, and violated invariants.
+
+```bash
+agentproof replay .agentproof/runs/<run-id>/repro.json
+```
+
+Replay is for deterministic reproduction of saved failures. Live LLM behavior can vary by model, provider, and time, so live results should be treated as smoke tests rather than exact replay guarantees.
+
+## Reports And CI
+
+Use the CLI in CI:
+
+```bash
+agentproof run examples/refund_native/suite.py --fail-on high --json report.json --junit report.xml
+```
+
+The command exits nonzero when a failure at or above the configured severity is reproduced. That lets CI fail when AgentProof finds a high-risk behavior.
+
+JSON and JUnit reports include scenario names, mutation names, statuses, violated invariants, failure messages, and committed effects.
+
+For the local release-hardening gate used by this repository:
+
+```bash
+python scripts/release_hardening.py
+```
+
+External-account checks are deliberately separate:
+
+```bash
+# Set OPENAI_API_KEY outside the command before running the live gate.
+AGENTPROOF_RUN_LIVE_TESTS=1 python scripts/release_hardening.py --live
+python scripts/release_hardening.py --github
+```
+
+## Real-App Smoke Test
+
+This repository includes a repeatable smoke test against a real external LangChain app:
+
+```bash
+python scripts/real_project_booking_smoke.py
+```
+
+That command clones [aniket-work/Lets-Build-Online-Booking-System-Using-AI-Agents](https://github.com/aniket-work/Lets-Build-Online-Booking-System-Using-AI-Agents), checks out a pinned commit, wraps its real `book_appointment` tool, and verifies AgentProof catches duplicate bookings caused by retry-after-timeout.
+
+Expected summary:
+
+```text
+baseline: 1 appointment, PASS
+timeout_after_commit: 2 appointments, INVARIANT_FAILURE
+```
+
+The failure is expected. It means AgentProof detected a real duplicate side effect in a real app integration.
+
+## Production Use
+
+AgentProof can be used in production engineering workflows as a pre-release and regression-testing tool. It should not be used as a runtime safety boundary for live autonomous agents.
+
+Before relying on it for a real app, define virtual or staging versions of every destructive tool, add invariants for every high-risk action, run deterministic tests in CI without API keys, and keep live model checks opt-in.
+
+See [Production Readiness](docs/production-readiness.md) for the full checklist and current limits.
+
+## Development
+
+```bash
+python -m pip install -e ".[dev]"
+ruff format .
+ruff check .
+mypy src/agentproof
+pytest -q
+```
+
+Build and validate a wheel:
+
+```bash
+python -m build
+twine check dist/*
+python -m venv /tmp/agentproof-wheel-smoke
+/tmp/agentproof-wheel-smoke/bin/python -m pip install dist/agentproof_sim-*.whl
+/tmp/agentproof-wheel-smoke/bin/agentproof mutations
+```
+
+## Status
+
+AgentProof is a `0.1.1` technical preview. Deterministic native execution, report generation, replay, adapter-boundary tests, clean wheel install, and CI are automated. Live model validation remains opt-in and depends on user-supplied credentials.
+
+## License
+
+Apache-2.0.

agentproof_sim-0.1.1.dist-info/RECORD

@@ -0,0 +1,46 @@
+agentproof/__init__.py,sha256=LC_lCLpmdWH73mPYgTcYi7maYqjceWO9r7r7FYS6jE4,1048
+agentproof/api.py,sha256=_P23tsd9mleJo19gP2Wdf0GzjdWuf7zwyL_S8wTEyS8,3318
+agentproof/cli.py,sha256=Ljn8bvw91Z0RuT8CdyKcg6kvwi1LFXvXDwjQFkvjAn0,5939
+agentproof/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+agentproof/pytest_plugin.py,sha256=o_9zmdhTdIXus82HVA9e2KkjkjwuACgKKhV5DUMa5Tw,1305
+agentproof/adapters/__init__.py,sha256=A5P5wZmpW67gHLN0-HDFA77rTEUhs9gtqY1mrir--x8,409
+agentproof/adapters/base.py,sha256=B5r8vRB3kGLGuRqONd1q7cwgG5Uwto8h6gPOh_mrLQs,471
+agentproof/adapters/langchain.py,sha256=FQ14dyfkeaSE9t3Tm59T1z_8qJb-Y-6wiVn_op5JZBY,3280
+agentproof/adapters/native.py,sha256=PnfBs4psS0W6sRLtUpnwjo1NDPVIA6WnJQ91Oi8OSME,726
+agentproof/adapters/openai_agents.py,sha256=8Cfb-mRfFM_v1oGRg2A-CLMgdcC7HgbkM3J_y9NKJyA,2140
+agentproof/core/clock.py,sha256=OQ7-cVyJT0AwqfbCJIlr2Bn7l8-icCK9pU_aCaY-I_c,474
+agentproof/core/effects.py,sha256=Nue40ZvVo0b9EXpfGMy-1ZNFVcGxWEWuLbjQc0aYR1Q,3308
+agentproof/core/events.py,sha256=EWAArVaw7qp7hQJKqLQdLUk_cheltROx8OrusqR91x0,2233
+agentproof/core/faults.py,sha256=ROpECY-LjKKYCIGWfDcYwIIFsJjmVtiMwyOKFzyP8vY,10598
+agentproof/core/invariant.py,sha256=6fAWK9l6vDl6j_PchNPxnDjJwGKpt0Tjt2E2W8kS_fA,3649
+agentproof/core/redaction.py,sha256=nT5Z_QWnubEMwWzySkkRZNiChALTfrH_X2OE9XN7e1c,1098
+agentproof/core/result.py,sha256=m283g-ZXudRQgWwoCxl4fft9NrmKvUhnit9sGJ6G1Os,2940
+agentproof/core/runner.py,sha256=auB8aeEfqdUN6fyG0R1in5MMz5VMky8JAjDYX34skLo,8114
+agentproof/core/scenario.py,sha256=b4ZqudHvBRj9LOL5Hd6HAiC6TZ6TeVurXE-vTK7p7Ss,276
+agentproof/core/trace.py,sha256=NJIc0NSyaUYyhsNyVG3Gz3GJHrG1k25KzimUJGnTSwU,1381
+agentproof/core/world.py,sha256=qZB4aeLtjRI2zDUsVJRH87DKWzE5nWAik8TyPmN1kow,4387
+agentproof/invariants/__init__.py,sha256=iTQmgeFZ69Q_5gZA6VamcyLflirDuIjVqP2smE3gsRY,395
+agentproof/invariants/builtin.py,sha256=bSJ1u0oDGALFP9Dg34J6iYBG2_sbzEAE76BVrlxBKyY,3248
+agentproof/invariants/temporal.py,sha256=uCnD3pTFjYAJGEZhC9GZxIxtunqfURX_hcKqLro32tY,125
+agentproof/mutations/__init__.py,sha256=Z0j9bYR2iDSpark5ZO6pYcMue06GHrb_o3rt_HzZlUI,1727
+agentproof/mutations/base.py,sha256=j6HKU11YxR815aywA9ypkVI7yuZxaLqoVc6bGJViOWY,1504
+agentproof/mutations/duplication.py,sha256=cEBp2hdEieeUBQx782LOBxXilE3wGLpLtdah0vdwS0M,440
+agentproof/mutations/ordering.py,sha256=qGZtffA2NxK8B_yecn2viVag_bGnQMptk7JjSU3QztQ,309
+agentproof/mutations/state.py,sha256=CQe0BT6BvAZQc4dN937ui8xQ4suROxnknDuid0goe3A,489
+agentproof/mutations/timing.py,sha256=xmFCwkUlKPk7wZq2nVQKVAoTtjF0UYUqfazw7wxbtC0,257
+agentproof/mutations/tool_faults.py,sha256=aLtA8NdC1NEeUuRcVg2zSuon0-tPAiz4DKMdrQ8gcAs,1682
+agentproof/replay/player.py,sha256=Mw03oD5SFXK4qIjO7ZMctmPPeApCdDjgTD1lVIa5DAs,2772
+agentproof/replay/recorder.py,sha256=8SBO-ClzJBz60RTXevTj3PX5V270b3E2ob7JW58tQhw,958
+agentproof/replay/schema.py,sha256=5q4AYd83DPV4QzICx9e_-T1B1IZ8POfCj11N84-EFyo,647
+agentproof/reporting/__init__.py,sha256=RvxaSbJ17YqyLoj_pPkf-DIY6CO5oXm7Lrq-zoGBjss,328
+agentproof/reporting/console.py,sha256=vI7L27Q1UFdLQddADpcFVAV5gv0rPTFWuKY8r37_rUc,1385
+agentproof/reporting/json_report.py,sha256=R1AM7PUCN82nNrNzHL8YWyQQnhcfituVSGR6_Al-CGs,1802
+agentproof/reporting/junit.py,sha256=uuAZJVduuWCQCiqeA0Y0HqD1axADZX1HTEi60nlHX3w,2332
+agentproof/tools/definition.py,sha256=vYijUF56XUKUiIkxa73O4GxC9Z9ydmG6qalSXPgz8M8,695
+agentproof/tools/invocation.py,sha256=G5-VrG8yVtT9vFjBU0DzytGCpnc568qlJrb7hpWEstc,271
+agentproof/tools/registry.py,sha256=RErwrRQDtx_dsGJGtrbWjoLL1vBBULVjXAO08-I2isE,5107
+agentproof_sim-0.1.1.dist-info/METADATA,sha256=FELEOznUq3qtv4BABAbYymOes8imRRN1TYaP-bfYl0g,15019
+agentproof_sim-0.1.1.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+agentproof_sim-0.1.1.dist-info/entry_points.txt,sha256=Y82NugGARcfr8boKCqgdmvjqAzcP3I2OR2qcJx051Dk,101
+agentproof_sim-0.1.1.dist-info/licenses/LICENSE,sha256=uxNmen3mAnYFCBKwENGBiACZwGBF0d7FxfQzzD-owZM,641
+agentproof_sim-0.1.1.dist-info/RECORD,,

agentproof_sim-0.1.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.0
+Root-Is-Purelib: true
+Tag: py3-none-any

agentproof_sim-0.1.1.dist-info/entry_points.txt

@@ -0,0 +1,5 @@
+[console_scripts]
+agentproof = agentproof.cli:main
+
+[pytest11]
+agentproof = agentproof.pytest_plugin

agentproof_sim-0.1.1.dist-info/licenses/LICENSE

@@ -0,0 +1,18 @@
+Apache License
+Version 2.0, January 2004
+https://www.apache.org/licenses/
+
+Copyright 2026 AgentProof Contributors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+