agentchaos-sdk 0.1.0__tar.gz

agentchaos_sdk-0.1.0/.env.example

@@ -0,0 +1,3 @@
+OPENAI_MODEL=""
+OPENAI_BASE_URL=""
+OPENAI_API_KEY=""

agentchaos_sdk-0.1.0/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,98 @@
+name: Bug Report
+description: Report a bug or unexpected behavior in AgentChaos
+title: "[Bug] "
+labels: ["bug"]
+assignees: []
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for taking the time to report a bug! Please fill out the form below.
+
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Pre-submission Checklist
+      options:
+        - label: I have searched existing issues to ensure this is not a duplicate
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the Bug
+      description: A clear and concise description of what the bug is.
+      placeholder: When I run agentchaos.inject(...), it fails with...
+    validations:
+      required: true
+
+  - type: textarea
+    id: steps-to-reproduce
+    attributes:
+      label: Steps to Reproduce
+      description: Provide detailed steps to reproduce the bug.
+      placeholder: |
+        1. Install agentchaos
+        2. Run the following code: ...
+        3. Observe the error...
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected Behavior
+      description: What did you expect to happen?
+    validations:
+      required: true
+
+  - type: textarea
+    id: actual-behavior
+    attributes:
+      label: Actual Behavior
+      description: What actually happened?
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: AgentChaos Version
+      placeholder: "e.g., 0.1.0"
+    validations:
+      required: true
+
+  - type: dropdown
+    id: os
+    attributes:
+      label: Operating System
+      options:
+        - Linux
+        - macOS
+        - Windows
+    validations:
+      required: true
+
+  - type: input
+    id: python-version
+    attributes:
+      label: Python Version
+      placeholder: "e.g., 3.12.0"
+    validations:
+      required: false
+
+  - type: textarea
+    id: logs
+    attributes:
+      label: Relevant Log Output
+      render: shell
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+    validations:
+      required: false

agentchaos_sdk-0.1.0/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: General Question
+    url: https://github.com/floritange/AgentChaos/discussions
+    about: Ask general questions in GitHub Discussions
+  - name: Fault Reference
+    url: https://github.com/floritange/AgentChaos/blob/main/docs/faults.md
+    about: Check the fault reference for all 65 configurations

agentchaos_sdk-0.1.0/.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,66 @@
+name: Feature Request
+description: Suggest a new feature or enhancement for AgentChaos
+title: "[Feature] "
+labels: ["enhancement"]
+assignees: []
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for helping improve AgentChaos! Please describe your feature request below.
+
+  - type: checkboxes
+    id: duplicate-check
+    attributes:
+      label: Pre-submission Checklist
+      options:
+        - label: I have searched existing issues to ensure this is not a duplicate
+          required: true
+
+  - type: textarea
+    id: problem-statement
+    attributes:
+      label: Problem Statement
+      description: What problem does this feature solve?
+      placeholder: |
+        When evaluating agent robustness, I need to...
+    validations:
+      required: true
+
+  - type: textarea
+    id: proposed-solution
+    attributes:
+      label: Proposed Solution
+      description: Describe your proposed solution.
+    validations:
+      required: true
+
+  - type: dropdown
+    id: category
+    attributes:
+      label: Feature Category
+      options:
+        - New Fault Type
+        - Evaluation Enhancement
+        - Trace Format
+        - Framework Support
+        - CLI Improvement
+        - Documentation
+        - Other
+    validations:
+      required: true
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Alternatives Considered
+    validations:
+      required: false
+
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+    validations:
+      required: false

agentchaos_sdk-0.1.0/.github/secret_scanning.yml

@@ -0,0 +1,4 @@
+paths-ignore:
+  - "tests/**"
+  - "examples/**"
+  - "docs/**"

agentchaos_sdk-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: CI
+
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          allow-prereleases: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync --python ${{ matrix.python-version }}
+
+      - name: Run tests
+        run: uv run pytest tests/ -v --cov=agentchaos --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        if: matrix.python-version == '3.12'
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.xml
+          fail_ci_if_error: false
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Install dependencies
+        run: uv sync
+
+      - name: Run ruff
+        run: uv run ruff check agentchaos/ --output-format=github

agentchaos_sdk-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,39 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+      contents: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip-existing: true
+
+      - name: Upload release assets
+        if: github.event_name == 'release'
+        uses: softprops/action-gh-release@v1
+        with:
+          files: dist/*

agentchaos_sdk-0.1.0/.gitignore

@@ -0,0 +1,38 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.egg-info/
+.venv/
+build/
+dist/
+uv.lock
+
+# Environment
+.env
+**/.env
+config.yaml
+
+# IDE & AI tools
+.vscode/
+.idea/
+.claude/
+.omc/
+CLAUDE.md
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Testing
+.pytest_cache/
+coverage.xml
+htmlcov/
+.deepeval/
+
+# Runtime output
+examples/traces/
+
+# Project-specific: not published
+refs/
+paper/

agentchaos_sdk-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  # Ruff version.
+  rev: v0.11.12
+  hooks:
+    # Run the linter.
+    - id: ruff-check
+      types_or: [ python, pyi ]
+      args: [ --fix ]
+    # Run the formatter.
+    - id: ruff-format
+      types_or: [ python, pyi ]
+- repo: https://github.com/gitleaks/gitleaks
+  rev: v8.24.2
+  hooks:
+    - id: gitleaks

agentchaos_sdk-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

agentchaos_sdk-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025-2026 AgentChaos Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

agentchaos_sdk-0.1.0/PKG-INFO

@@ -0,0 +1,256 @@
+Metadata-Version: 2.4
+Name: agentchaos-sdk
+Version: 0.1.0
+Summary: Evaluate agent system robustness through controlled, runtime, non-intrusive LLM API fault injection.
+Project-URL: Homepage, https://github.com/floritange/AgentChaos
+Project-URL: Documentation, https://floritange.github.io/AgentChaos/
+Project-URL: Repository, https://github.com/floritange/AgentChaos
+Project-URL: Issues, https://github.com/floritange/AgentChaos/issues
+Author: AgentChaos Contributors
+License: MIT
+License-File: LICENSE
+Keywords: agent,chaos,evaluation,fault-injection,llm,robust,testing
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Testing
+Requires-Python: >=3.10
+Requires-Dist: google-adk[extensions]<2.0,>=1.0
+Requires-Dist: httpx<1.0,>=0.24
+Requires-Dist: langchain-openai<2.0,>=0.3
+Requires-Dist: langchain<2.0,>=0.3
+Requires-Dist: loguru<1.0,>=0.7
+Requires-Dist: openai<3.0,>=2.0
+Requires-Dist: python-dotenv<2.0,>=1.0
+Description-Content-Type: text/markdown
+
+# AgentChaos
+**Evaluate agent system robustness through controlled, runtime, non-intrusive LLM API fault injection.**
+
+[![PyPI version](https://img.shields.io/pypi/v/agentchaos-sdk?color=blue)](https://pypi.org/project/agentchaos-sdk/)
+[![Python](https://img.shields.io/pypi/pyversions/agentchaos-sdk.svg)](https://pypi.org/project/agentchaos-sdk/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![CI](https://github.com/floritange/AgentChaos/actions/workflows/ci.yml/badge.svg)](https://github.com/floritange/AgentChaos/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/floritange/AgentChaos/graph/badge.svg?branch=main)](https://codecov.io/gh/floritange/AgentChaos?branch=main)
+[![Tests](https://img.shields.io/badge/tests-passing-brightgreen)]()
+[![Docs](https://img.shields.io/badge/docs-github.io-blue)](https://floritange.github.io/AgentChaos/)
+
+---
+
+## Overview
+
+LLM-based agent systems issue multiple API calls per task, and each call can fail (HTTP 5xx, truncation, empty response, encoding corruption, schema violation). Once a faulty response occurs, it propagates through downstream agents and causes task failure. **AgentChaos** injects controlled faults at the HTTP transport layer — without modifying any agent source code — to evaluate robustness before these failures happen in production.
+
+---
+
+## Quick Start
+
+```bash
+pip install agentchaos-sdk
+```
+
+```python
+import agentchaos
+
+# Inject fault (your agent code needs ZERO changes)
+agentchaos.inject("llm_error_single")
+result = await my_agent(query)         # agent runs normally, unaware
+agentchaos.disable()                   # stop
+agentchaos.save_trace("trace.json")   # save full LLM call trace
+```
+
+```bash
+# examples
+git clone https://github.com/floritange/AgentChaos.git
+cd AgentChaos
+uv sync
+uv run python examples/list_faults.py       # list all 65 faults
+uv run python examples/agent_openai.py      # OpenAI agent: normal vs faulted
+uv run python examples/agent_langchain.py   # LangChain agent
+uv run python examples/agent_adk.py         # Google ADK agent
+uv run python examples/eval_batch.py        # batch evaluation
+```
+
+---
+
+## How It Works
+
+<img src="docs/overview.png"/>
+
+An HTTP-layer injection mechanism patches the HTTP client at runtime to intercept and modify LLM API responses according to the fault configuration, requiring no changes to any agent system.
+
+**Properties:**
+- Works with **any** framework using OpenAI-compatible APIs (OpenAI, LangChain, ADK, AutoGen, CrewAI, LiteLLM)
+- **Zero code changes** — just `inject()` / `disable()` around your existing code
+- Records full **execution trace** (raw input/output, token usage, timing) for every LLM call
+- **65 pre-built fault configurations** covering all real-world failure modes
+
+---
+
+## API
+
+| Function | Description |
+|---|---|
+| `agentchaos.inject(fault)` | Start fault injection + trace (`None` = trace only) |
+| `agentchaos.disable()` | Stop injection and trace |
+| `agentchaos.save_trace(path)` | Save trace to JSON |
+| `agentchaos.eval(agent_fn, query, faults)` | Batch robustness evaluation |
+| `agentchaos.diagnose(text)` | Detect fault type from output |
+| `agentchaos.list_faults()` | List all 65 experiments |
+
+```python
+import agentchaos
+
+# Trace only (no fault)
+agentchaos.inject(None)
+result = await my_agent(query)
+agentchaos.disable()
+agentchaos.save_trace("trace_normal.json")
+
+# Inject fault + trace
+agentchaos.inject("llm_error_single")
+result = await my_agent(query)
+agentchaos.disable()
+agentchaos.save_trace("trace_faulted.json")
+
+# Batch evaluation
+report = await agentchaos.eval(my_agent, query, faults="all")
+print(report.summary())
+```
+
+---
+
+## Trace Format
+
+```json
+{
+  "call_index": 0,
+  "raw_input": {"model": "gpt-5.5", "messages": [...], "tools": [...]},
+  "raw_output": {
+    "content": "The answer is 42.",
+    "tool_calls": [],
+    "finish_reason": "stop",
+    "usage": {"prompt_tokens": 306, "completion_tokens": 54, "total_tokens": 360},
+    "http_status": 200
+  },
+  "injected_output": {
+    "content": "[API ERROR] HTTP 500: Internal Server Error.",
+    "tool_calls": []
+  },
+  "timing": {"llm_latency_ms": 1523.4, "total_ms": 1524.1},
+  "fault_applied": true
+}
+```
+
+> `raw_output` = LLM original response. `injected_output` = what the agent actually receives (only present when `fault_applied: true`).
+
+---
+
+## Fault Taxonomy
+
+We define a fault taxonomy by adapting the classical fault classification from distributed systems (Avizienis et al., 2004) to LLM API responses. The taxonomy covers crash, omission, and value faults on both content and tool call fields.
+
+| Category | Fault Type | Content | Tool Call | Real-world Scenario |
+|---|---|:---:|:---:|---|
+| **Crash** | Error | yes | yes | Server overload, HTTP 5xx, rate limiting |
+| **Crash** | Timeout | yes | yes | Network congestion, backend delay, API latency |
+| **Omission** | Empty | yes | yes | Safety filter, content policy rejection |
+| **Omission** | Truncate | yes | yes | Token limit, TCP interruption, incomplete completion |
+| **Value** | Corrupt | yes | yes | Encoding error, garbled characters |
+| **Value** | Schema | yes | yes | Parsing error, schema mismatch |
+
+From **Crash** to **Value**, faults become progressively harder to detect. Crash faults produce obvious error signals and are typically retried. Value faults look like valid output and propagate silently — making them the most dangerous in practice.
+
+**65 = (6 fault types x 2 targets x 4 strategies) + 8 compound + 9 positional**
+
+Detailed documentation: **[docs/faults.md](docs/faults.md)**
+
+---
+
+## Evaluation Results
+
+### Experimental Setup
+
+<table>
+<tr><th>Agent System</th><th>Architecture</th><th>Benchmarks</th></tr>
+<tr><td><a href="https://openreview.net/forum?id=BAakY1hNKS">AutoGen</a></td><td>Iterative (coder + executor)</td><td rowspan="4"><a href="https://arxiv.org/abs/2107.03374">HumanEval</a>, <a href="https://openreview.net/forum?id=1qvx610Cu7">HumanEval+</a>, <a href="https://arxiv.org/abs/2108.07732">MBPP</a>, <a href="https://openreview.net/forum?id=1qvx610Cu7">MBPP+</a>, <a href="https://openreview.net/forum?id=US2eyuYlvS">MMLU-Pro</a>, <a href="https://aclanthology.org/2024.acl-long.410">MATH-500</a></td></tr>
+<tr><td><a href="https://aclanthology.org/2024.acl-long.72">MAD</a></td><td>Debate (proposer + critic)</td></tr>
+<tr><td><a href="https://aclanthology.org/2024.acl-long.269">MapCoder</a></td><td>Pipeline (planner + coder + debugger)</td></tr>
+<tr><td><a href="https://openreview.net/forum?id=jd0RewGP4w">EvoMAC</a></td><td>Iterative (multi-agent collaboration)</td></tr>
+<tr><td><a href="https://arxiv.org/abs/2510.22775">Mini-SE</a></td><td>Iterative (SWE agent)</td><td><a href="https://arxiv.org/abs/2501.14975">SWE-bench Pro</a></td></tr>
+</table>
+
+**Backbone LLMs**: Claude-Sonnet-4.5, GPT-5.2, DeepSeek-V3.2, Seed-1.8
+
+**Metric**: Δpass@1 = pass@1 (w/o fault) − pass@1 (w/ fault). Higher = more vulnerable.
+
+### RQ1: Overall Robustness Degradation (Claude-Sonnet-4.5)
+
+| System | HumanEval | HumanEval+ | MBPP | MBPP+ | MMLU-Pro | MATH-500 |
+|---|---|---|---|---|---|---|
+| AutoGen | 19.44 | 21.13 | 17.31 | 11.61 | 7.05 | 8.38 |
+| MAD | 24.20 | 24.84 | 24.49 | 15.08 | 20.64 | 20.70 |
+| **MapCoder** | **48.61** | **49.30** | **41.07** | **40.85** | **38.25** | **34.27** |
+| EvoMAC | 18.48 | 18.18 | 16.67 | 14.73 | 13.63 | 15.85 |
+| Mini-SE | — | — | — | — | — | — |
+
+> Mini-SE is evaluated only on SWE-bench Pro (Δpass@1 = 0.87%).
+
+### RQ2: Impact of Fault Configurations
+
+<img src="docs/fig_fault_type_impact.png"/>
+
+<img src="docs/fig_strategy_impact.png"/>
+
+- Content faults cause higher Δpass@1 than tool call faults; only **corrupt** stays below 7%
+- **Persistent** injection causes the highest Δpass@1 — up to **62.39%** (MapCoder)
+- **Pipeline** systems are most position-sensitive — single early fault drops pass@1 by up to **83.87%**
+- **Compound** content faults amplify degradation — up to **86.36%** (MapCoder)
+
+### RQ3: Fault Diagnosis
+
+Existing methods achieve below **53%** accuracy on fault type and below **56%** on fault step. Truncation — the most harmful fault — is identified with only **4.3%** accuracy.
+
+### Key Findings
+
+| # | Finding |
+|---|---|
+| 1 | All systems degrade under fault injection (Δpass@1 up to 50 pp) |
+| 2 | Most severe faults are NOT most harmful — truncation/empty propagate silently |
+| 3 | Most harmful faults are hardest to diagnose (truncation: 4.3% accuracy) |
+| 4 | Architecture determines robustness — ranking consistent across all LLMs |
+| 5 | Persistent injection overrides architectural advantages (up to 62.39%) |
+| 6 | Compound content faults amplify degradation (up to 86.36%) |
+
+---
+
+## Documentation
+
+- **[Fault Reference](docs/faults.md)** — Complete reference for all 65 fault configurations
+- **[Examples](examples/)** — Runnable demos for OpenAI, LangChain, ADK
+
+---
+
+## Citation
+
+If you use AgentChaos in your research, please cite:
+
+```bibtex
+@article{agentchaos2026,
+  title={AgentChaos: Chaos Engineering for Robust Agent Evaluation via LLM API Fault Injection},
+  year={2026}
+}
+```
+
+---
+
+## License
+
+MIT -- see [LICENSE](LICENSE).