toolgauntlet 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. toolgauntlet-0.1.0/.gitignore +12 -0
  2. toolgauntlet-0.1.0/CHANGELOG.md +25 -0
  3. toolgauntlet-0.1.0/LICENSE +21 -0
  4. toolgauntlet-0.1.0/PKG-INFO +264 -0
  5. toolgauntlet-0.1.0/README.md +224 -0
  6. toolgauntlet-0.1.0/brand/agent-chaos/README.md +52 -0
  7. toolgauntlet-0.1.0/docs/README.md +41 -0
  8. toolgauntlet-0.1.0/examples/adapters/README.md +9 -0
  9. toolgauntlet-0.1.0/examples/paid_pack_template/CHANGELOG.md +6 -0
  10. toolgauntlet-0.1.0/examples/paid_pack_template/README.md +5 -0
  11. toolgauntlet-0.1.0/marketing/README.md +28 -0
  12. toolgauntlet-0.1.0/pyproject.toml +87 -0
  13. toolgauntlet-0.1.0/src/agent_chaos/__init__.py +8 -0
  14. toolgauntlet-0.1.0/src/agent_chaos/adapters/__init__.py +14 -0
  15. toolgauntlet-0.1.0/src/agent_chaos/adapters/demo.py +59 -0
  16. toolgauntlet-0.1.0/src/agent_chaos/adapters/helpers.py +105 -0
  17. toolgauntlet-0.1.0/src/agent_chaos/adapters/openai_compatible.py +209 -0
  18. toolgauntlet-0.1.0/src/agent_chaos/adapters/types.py +16 -0
  19. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/__init__.py +5 -0
  20. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/app.py +722 -0
  21. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/audit.py +27 -0
  22. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/cli.py +126 -0
  23. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/config.py +99 -0
  24. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/costing.py +33 -0
  25. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/enforcement.py +149 -0
  26. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/ledger.py +90 -0
  27. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/policy.py +139 -0
  28. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/rate_limit.py +119 -0
  29. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/redaction.py +41 -0
  30. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/types.py +73 -0
  31. toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/utils.py +14 -0
  32. toolgauntlet-0.1.0/src/agent_chaos/cli.py +566 -0
  33. toolgauntlet-0.1.0/src/agent_chaos/config.py +22 -0
  34. toolgauntlet-0.1.0/src/agent_chaos/exceptions.py +18 -0
  35. toolgauntlet-0.1.0/src/agent_chaos/injectors/__init__.py +49 -0
  36. toolgauntlet-0.1.0/src/agent_chaos/injectors/base.py +62 -0
  37. toolgauntlet-0.1.0/src/agent_chaos/injectors/builtins.py +229 -0
  38. toolgauntlet-0.1.0/src/agent_chaos/loader.py +150 -0
  39. toolgauntlet-0.1.0/src/agent_chaos/models.py +407 -0
  40. toolgauntlet-0.1.0/src/agent_chaos/pack_signing.py +104 -0
  41. toolgauntlet-0.1.0/src/agent_chaos/redaction.py +52 -0
  42. toolgauntlet-0.1.0/src/agent_chaos/runner.py +420 -0
  43. toolgauntlet-0.1.0/src/agent_chaos/scoring.py +219 -0
  44. toolgauntlet-0.1.0/src/agent_chaos/site_builder.py +174 -0
  45. toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/fixtures/tools.json +35 -0
  46. toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/schemas/get_order.json +15 -0
  47. toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/schemas/issue_refund.json +14 -0
  48. toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/suite.yaml +62 -0
  49. toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/fixtures/tools.json +38 -0
  50. toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/classify_ticket.json +13 -0
  51. toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/escalate_ticket.json +13 -0
  52. toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/get_ticket.json +14 -0
  53. toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/suite.yaml +49 -0
  54. toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/fixtures/tools.json +50 -0
  55. toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/book_flight.json +14 -0
  56. toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/get_travel_policy.json +13 -0
  57. toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/search_flights.json +23 -0
  58. toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/suite.yaml +54 -0
  59. toolgauntlet-0.1.0/src/agent_chaos/suites.py +97 -0
  60. toolgauntlet-0.1.0/src/agent_chaos/telemetry.py +177 -0
  61. toolgauntlet-0.1.0/src/agent_chaos/utils.py +26 -0
  62. toolgauntlet-0.1.0/src/toolgauntlet/__init__.py +8 -0
  63. toolgauntlet-0.1.0/src/toolgauntlet/adapters/__init__.py +21 -0
  64. toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/__init__.py +5 -0
  65. toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/app.py +5 -0
  66. toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/cli.py +9 -0
  67. toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/config.py +5 -0
  68. toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/types.py +5 -0
  69. toolgauntlet-0.1.0/src/toolgauntlet/cli.py +9 -0
  70. toolgauntlet-0.1.0/src/toolgauntlet/injectors/__init__.py +11 -0
  71. toolgauntlet-0.1.0/src/toolgauntlet/suites.py +17 -0
  72. toolgauntlet-0.1.0/src/toolgauntlet/telemetry.py +5 -0
@@ -0,0 +1,12 @@
1
+ __pycache__/
2
+ *.pyc
3
+ .pytest_cache/
4
+ .venv/
5
+ report.json
6
+ report.md
7
+ *.egg-info/
8
+ dist/
9
+ build/
10
+ .env
11
+ .env.*
12
+ !.env.proxy.example
@@ -0,0 +1,25 @@
1
+ # Changelog
2
+
3
+ All notable changes to ToolGauntlet are tracked here.
4
+
5
+ The format follows Keep a Changelog conventions, and this project uses semantic versioning for the core Python package.
6
+
7
+ ## [0.1.0] - Unreleased
8
+
9
+ ### Added
10
+ - Deterministic chaos suite runner for tool-using agents.
11
+ - Built-in ecommerce refunds, support triage, and travel booking suites.
12
+ - CLI commands for listing suites, running suites, rendering reports, scaffolding suites, and signing/verifying suite packs.
13
+ - No-key `quickstart` command that generates safe baseline reports and regression-gate next steps.
14
+ - JSON, Markdown, and HTML report generation.
15
+ - Regression comparison support for CI gates.
16
+ - OpenAI-compatible, async, LangChain-like, and deterministic demo adapter helpers.
17
+ - Budget-as-Code proxy with policy enforcement, redacted audit logs, usage ledger, rate limits, Prometheus metrics, and optional OpenTelemetry export.
18
+ - Static documentation and marketing site builder.
19
+ - AI-led venture operating plan and staged external launch-readiness monitor.
20
+ - Draft-only AI pilot/support triage and weekly operating/marketing briefs with deterministic fallback.
21
+ - Private, self-updating AI venture dashboard with safe Markdown rendering and workflow evidence links.
22
+ - Configuration-backed release constraints and founder decisions that model output cannot rewrite.
23
+ - Private, deduplicated HiLo Labs website-lead ingestion with contact-isolated same-run AI triage.
24
+ - Release, support, security, deployment, entitlement, and versioning documentation.
25
+ - Collision-free `toolgauntlet` distribution, CLI, and public import, with the prerelease `agent-chaos` commands and `agent_chaos` import retained as compatibility aliases.
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Joseph Nilo
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,264 @@
1
+ Metadata-Version: 2.4
2
+ Name: toolgauntlet
3
+ Version: 0.1.0
4
+ Summary: Deterministic reliability testing for tool-using AI agents
5
+ Project-URL: Homepage, https://labs.hilomedia.com/products/toolgauntlet
6
+ Project-URL: Documentation, https://github.com/josephnilo/agent-chaos/tree/main/docs
7
+ Project-URL: Repository, https://github.com/josephnilo/agent-chaos
8
+ Project-URL: Issues, https://github.com/josephnilo/agent-chaos/issues
9
+ Project-URL: Changelog, https://github.com/josephnilo/agent-chaos/blob/main/CHANGELOG.md
10
+ Author: HiLo Labs
11
+ License: MIT
12
+ License-File: LICENSE
13
+ Keywords: agents,chaos-engineering,evaluation,llm,testing
14
+ Classifier: Development Status :: 3 - Alpha
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3 :: Only
19
+ Classifier: Programming Language :: Python :: 3.10
20
+ Classifier: Programming Language :: Python :: 3.11
21
+ Classifier: Programming Language :: Python :: 3.12
22
+ Requires-Python: >=3.10
23
+ Requires-Dist: fastapi>=0.115
24
+ Requires-Dist: httpx>=0.27
25
+ Requires-Dist: jsonschema>=4.21
26
+ Requires-Dist: prometheus-client>=0.20
27
+ Requires-Dist: pyyaml>=6.0
28
+ Requires-Dist: uvicorn>=0.30
29
+ Provides-Extra: dev
30
+ Requires-Dist: markdown>=3.6; extra == 'dev'
31
+ Requires-Dist: pytest>=8.3; extra == 'dev'
32
+ Provides-Extra: otel
33
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27; extra == 'otel'
34
+ Requires-Dist: opentelemetry-sdk>=1.27; extra == 'otel'
35
+ Provides-Extra: redis
36
+ Requires-Dist: redis>=5.0; extra == 'redis'
37
+ Provides-Extra: site
38
+ Requires-Dist: markdown>=3.6; extra == 'site'
39
+ Description-Content-Type: text/markdown
40
+
41
+ # ToolGauntlet
42
+
43
+ ToolGauntlet by HiLo Labs is a Python package and CLI for deterministic reliability testing of tool-using AI agents.
44
+
45
+ ## Install
46
+
47
+ ```bash
48
+ pip install toolgauntlet
49
+ toolgauntlet --version
50
+ ```
51
+
52
+ The distribution, primary CLI, and public Python import are all `toolgauntlet`.
53
+ The prerelease `agent-chaos` commands and `agent_chaos` import remain available
54
+ as compatibility aliases.
55
+
56
+ ## Quick start (CLI)
57
+
58
+ Generate a complete local baseline with no API key:
59
+
60
+ ```bash
61
+ toolgauntlet quickstart --out-dir toolgauntlet-quickstart
62
+ ```
63
+
64
+ This writes JSON, Markdown, and HTML baseline reports plus a copyable next-step
65
+ regression command. It uses the built-in deterministic adapter with safe logging
66
+ enabled and refuses to overwrite its files unless `--force` is supplied.
67
+
68
+ Build a custom workflow manually:
69
+
70
+ ```bash
71
+ toolgauntlet list-suites
72
+ toolgauntlet init-suite --id my_suite_v1 --out-dir suites
73
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 10 --out report.json
74
+ toolgauntlet report --in report.json --format md --out report.md
75
+ toolgauntlet report --in report.json --format html --out report.html
76
+ ```
77
+
78
+ Use a saved baseline as a release gate:
79
+
80
+ ```bash
81
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 25 --out baseline.json
82
+ toolgauntlet run \
83
+ --suite ecommerce_refunds_v1 \
84
+ --runs 25 \
85
+ --baseline baseline.json \
86
+ --fail-on-regression \
87
+ --regression-tolerance 1.0 \
88
+ --out candidate.json
89
+ ```
90
+
91
+ See the copyable CI workflow in [`docs/ci-gate-example.md`](docs/ci-gate-example.md) and a sample report in [`docs/sample-reports/ecommerce_refunds_v1.md`](docs/sample-reports/ecommerce_refunds_v1.md).
92
+
93
+ Before release, run the same local gate used for MVP validation:
94
+
95
+ ```bash
96
+ scripts/release_gate.sh
97
+ ```
98
+
99
+ Verify the configured external launch stage (live page now; registries as the
100
+ stage advances):
101
+
102
+ ```bash
103
+ python scripts/check_launch_readiness.py
104
+ ```
105
+
106
+ Signed pack workflow:
107
+
108
+ ```bash
109
+ export TOOLGAUNTLET_PACK_KEY=your-shared-signing-key
110
+ toolgauntlet sign-pack --suite suites/my_suite_v1
111
+ toolgauntlet verify-pack --suite suites/my_suite_v1
112
+ toolgauntlet run --suite suites/my_suite_v1 --verify-pack --out report.json
113
+ ```
114
+
115
+ OpenTelemetry traces (optional):
116
+
117
+ ```bash
118
+ pip install "toolgauntlet[otel]"
119
+ # Requires an OTLP/HTTP collector reachable at the default endpoint,
120
+ # or pass --otel-endpoint with your collector URL.
121
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 5 --otel-enabled --otel-metrics-enabled
122
+ ```
123
+
124
+ ## Quick start (Python)
125
+
126
+ ```python
127
+ from toolgauntlet import SuiteConfig, run_suite
128
+ from toolgauntlet.adapters import deterministic_demo_agent
129
+
130
+ report = run_suite(
131
+ agent=deterministic_demo_agent,
132
+ suite_path="ecommerce_refunds_v1",
133
+ config=SuiteConfig(runs=10, concurrency=2),
134
+ )
135
+
136
+ report.save_json("report.json")
137
+ report.save_markdown("report.md")
138
+ ```
139
+
140
+ ## OpenAI-compatible adapter
141
+
142
+ ```python
143
+ from toolgauntlet import SuiteConfig, run_suite
144
+ from toolgauntlet.adapters import OpenAICompatibleAdapterConfig, make_openai_tool_loop_adapter
145
+
146
+ agent = make_openai_tool_loop_adapter(
147
+ OpenAICompatibleAdapterConfig(
148
+ model="gpt-4o-mini",
149
+ base_url="https://api.openai.com",
150
+ )
151
+ )
152
+
153
+ report = run_suite(
154
+ agent=agent,
155
+ suite_path="ecommerce_refunds_v1",
156
+ config=SuiteConfig(runs=5, concurrency=1),
157
+ )
158
+ ```
159
+
160
+ ## Additional adapter helpers
161
+ - `make_async_adapter`: use async agent callables with the synchronous harness interface.
162
+ - `make_langchain_like_adapter`: adapt invoke/ainvoke-style runnables.
163
+
164
+ See examples in [`examples/adapters/`](examples/adapters/).
165
+
166
+ ## Budget-as-Code Proxy
167
+
168
+ Start and validate:
169
+
170
+ ```bash
171
+ toolgauntlet-proxy validate-policy --policy examples/budget_proxy/policy.example.yaml
172
+ toolgauntlet-proxy serve \
173
+ --host 127.0.0.1 \
174
+ --port 8080 \
175
+ --policy examples/budget_proxy/policy.example.yaml \
176
+ --ledger logs/usage-ledger.sqlite
177
+ ```
178
+
179
+ One-command local stack (proxy + redis + optional otel/prometheus):
180
+
181
+ ```bash
182
+ docker compose --env-file .env.proxy -f docker-compose.proxy.yml up --build
183
+ ```
184
+
185
+ See full guide: [`docs/compose-deployment.md`](docs/compose-deployment.md)
186
+
187
+ Build and preview site pages locally:
188
+
189
+ ```bash
190
+ pip install -e ".[dev,site]"
191
+ toolgauntlet build-site --pages-root site/pages/toolgauntlet --out-dir site/dist/toolgauntlet
192
+ python -m http.server 8787 --directory site/dist
193
+ ```
194
+
195
+ Key endpoints:
196
+ - `GET /healthz`
197
+ - `GET /metrics`
198
+ - `GET /v1/models`
199
+ - `POST /v1/chat/completions`
200
+ - `POST /v1/responses`
201
+ - `POST /admin/reload-policy`
202
+
203
+ Common environment variables:
204
+ - `OPENAI_API_KEY` or `BUDGET_UPSTREAM_API_KEY`
205
+ - `BUDGET_PROXY_API_KEY`
206
+ - `BUDGET_POLICY_PATH`
207
+ - `BUDGET_AUDIT_LOG_PATH`
208
+ - `BUDGET_LEDGER_PATH`
209
+ - `BUDGET_PROJECT_HEADER`
210
+ - `BUDGET_PROJECT_API_KEY_HEADER`
211
+ - `BUDGET_METRIC_NAMESPACE`
212
+ - `BUDGET_RATE_LIMIT_BACKEND` (`memory` or `redis`)
213
+ - `BUDGET_REDIS_URL` (required when backend is `redis`)
214
+ - `BUDGET_REDIS_KEY_PREFIX`
215
+ - `BUDGET_OTEL_ENABLED`
216
+ - `BUDGET_OTEL_METRICS_ENABLED`
217
+ - `BUDGET_OTEL_SERVICE_NAME`
218
+ - `BUDGET_OTEL_ENDPOINT`
219
+
220
+ To enable distributed rate limiting across proxy instances, install Redis support:
221
+
222
+ ```bash
223
+ pip install "toolgauntlet[redis]"
224
+ ```
225
+
226
+ ## Included suites
227
+ - `ecommerce_refunds_v1`
228
+ - `support_triage_v1`
229
+ - `travel_booking_v1`
230
+
231
+ ## Documentation
232
+ - docs index: [`docs/README.md`](docs/README.md)
233
+ - API reference: [`docs/api-reference.md`](docs/api-reference.md)
234
+ - proxy operations: [`docs/proxy-operations.md`](docs/proxy-operations.md)
235
+ - compose deployment: [`docs/compose-deployment.md`](docs/compose-deployment.md)
236
+ - site preview: [`docs/site-preview.md`](docs/site-preview.md)
237
+ - CI gate example: [`docs/ci-gate-example.md`](docs/ci-gate-example.md)
238
+ - commercial pack readiness: [`docs/commercial-pack-readiness.md`](docs/commercial-pack-readiness.md)
239
+ - customer pack handoff: [`docs/customer-pack-handoff.md`](docs/customer-pack-handoff.md)
240
+ - launch operations checklist: [`docs/launch-operations-checklist.md`](docs/launch-operations-checklist.md)
241
+ - AI venture operating plan: [`docs/venture-operating-plan.md`](docs/venture-operating-plan.md)
242
+ - AI venture operator: [`docs/venture-operator.md`](docs/venture-operator.md)
243
+ - release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
244
+ - versioning policy: [`docs/versioning-policy.md`](docs/versioning-policy.md)
245
+ - entitlement model: [`docs/entitlement-model.md`](docs/entitlement-model.md)
246
+ - support SLA: [`docs/support-sla.md`](docs/support-sla.md)
247
+
248
+ ## Marketing and launch materials
249
+ - materials index: [`marketing/README.md`](marketing/README.md)
250
+ - website copy: [`site/pages/toolgauntlet/`](site/pages/toolgauntlet)
251
+ - pilot sales path: [`site/pages/toolgauntlet/pilot.md`](site/pages/toolgauntlet/pilot.md) and [`docs/pilot-intake.md`](docs/pilot-intake.md)
252
+
253
+ ## Backlog and roadmap
254
+ - product backlog: [`TODO.md`](TODO.md)
255
+
256
+ ## Release workflows
257
+ GitHub Actions workflows included:
258
+ - `.github/workflows/ci.yml`
259
+ - `.github/workflows/launch-readiness.yml`
260
+ - `.github/workflows/venture-ops.yml`
261
+ - `.github/workflows/lead-intake.yml`
262
+ - `.github/workflows/release.yml`
263
+
264
+ Publishing uses trusted publishing via `pypa/gh-action-pypi-publish`.
@@ -0,0 +1,224 @@
1
+ # ToolGauntlet
2
+
3
+ ToolGauntlet by HiLo Labs is a Python package and CLI for deterministic reliability testing of tool-using AI agents.
4
+
5
+ ## Install
6
+
7
+ ```bash
8
+ pip install toolgauntlet
9
+ toolgauntlet --version
10
+ ```
11
+
12
+ The distribution, primary CLI, and public Python import are all `toolgauntlet`.
13
+ The prerelease `agent-chaos` commands and `agent_chaos` import remain available
14
+ as compatibility aliases.
15
+
16
+ ## Quick start (CLI)
17
+
18
+ Generate a complete local baseline with no API key:
19
+
20
+ ```bash
21
+ toolgauntlet quickstart --out-dir toolgauntlet-quickstart
22
+ ```
23
+
24
+ This writes JSON, Markdown, and HTML baseline reports plus a copyable next-step
25
+ regression command. It uses the built-in deterministic adapter with safe logging
26
+ enabled and refuses to overwrite its files unless `--force` is supplied.
27
+
28
+ Build a custom workflow manually:
29
+
30
+ ```bash
31
+ toolgauntlet list-suites
32
+ toolgauntlet init-suite --id my_suite_v1 --out-dir suites
33
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 10 --out report.json
34
+ toolgauntlet report --in report.json --format md --out report.md
35
+ toolgauntlet report --in report.json --format html --out report.html
36
+ ```
37
+
38
+ Use a saved baseline as a release gate:
39
+
40
+ ```bash
41
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 25 --out baseline.json
42
+ toolgauntlet run \
43
+ --suite ecommerce_refunds_v1 \
44
+ --runs 25 \
45
+ --baseline baseline.json \
46
+ --fail-on-regression \
47
+ --regression-tolerance 1.0 \
48
+ --out candidate.json
49
+ ```
50
+
51
+ See the copyable CI workflow in [`docs/ci-gate-example.md`](docs/ci-gate-example.md) and a sample report in [`docs/sample-reports/ecommerce_refunds_v1.md`](docs/sample-reports/ecommerce_refunds_v1.md).
52
+
53
+ Before release, run the same local gate used for MVP validation:
54
+
55
+ ```bash
56
+ scripts/release_gate.sh
57
+ ```
58
+
59
+ Verify the configured external launch stage (live page now; registries as the
60
+ stage advances):
61
+
62
+ ```bash
63
+ python scripts/check_launch_readiness.py
64
+ ```
65
+
66
+ Signed pack workflow:
67
+
68
+ ```bash
69
+ export TOOLGAUNTLET_PACK_KEY=your-shared-signing-key
70
+ toolgauntlet sign-pack --suite suites/my_suite_v1
71
+ toolgauntlet verify-pack --suite suites/my_suite_v1
72
+ toolgauntlet run --suite suites/my_suite_v1 --verify-pack --out report.json
73
+ ```
74
+
75
+ OpenTelemetry traces (optional):
76
+
77
+ ```bash
78
+ pip install "toolgauntlet[otel]"
79
+ # Requires an OTLP/HTTP collector reachable at the default endpoint,
80
+ # or pass --otel-endpoint with your collector URL.
81
+ toolgauntlet run --suite ecommerce_refunds_v1 --runs 5 --otel-enabled --otel-metrics-enabled
82
+ ```
83
+
84
+ ## Quick start (Python)
85
+
86
+ ```python
87
+ from toolgauntlet import SuiteConfig, run_suite
88
+ from toolgauntlet.adapters import deterministic_demo_agent
89
+
90
+ report = run_suite(
91
+ agent=deterministic_demo_agent,
92
+ suite_path="ecommerce_refunds_v1",
93
+ config=SuiteConfig(runs=10, concurrency=2),
94
+ )
95
+
96
+ report.save_json("report.json")
97
+ report.save_markdown("report.md")
98
+ ```
99
+
100
+ ## OpenAI-compatible adapter
101
+
102
+ ```python
103
+ from toolgauntlet import SuiteConfig, run_suite
104
+ from toolgauntlet.adapters import OpenAICompatibleAdapterConfig, make_openai_tool_loop_adapter
105
+
106
+ agent = make_openai_tool_loop_adapter(
107
+ OpenAICompatibleAdapterConfig(
108
+ model="gpt-4o-mini",
109
+ base_url="https://api.openai.com",
110
+ )
111
+ )
112
+
113
+ report = run_suite(
114
+ agent=agent,
115
+ suite_path="ecommerce_refunds_v1",
116
+ config=SuiteConfig(runs=5, concurrency=1),
117
+ )
118
+ ```
119
+
120
+ ## Additional adapter helpers
121
+ - `make_async_adapter`: use async agent callables with the synchronous harness interface.
122
+ - `make_langchain_like_adapter`: adapt invoke/ainvoke-style runnables.
123
+
124
+ See examples in [`examples/adapters/`](examples/adapters/).
125
+
126
+ ## Budget-as-Code Proxy
127
+
128
+ Start and validate:
129
+
130
+ ```bash
131
+ toolgauntlet-proxy validate-policy --policy examples/budget_proxy/policy.example.yaml
132
+ toolgauntlet-proxy serve \
133
+ --host 127.0.0.1 \
134
+ --port 8080 \
135
+ --policy examples/budget_proxy/policy.example.yaml \
136
+ --ledger logs/usage-ledger.sqlite
137
+ ```
138
+
139
+ One-command local stack (proxy + redis + optional otel/prometheus):
140
+
141
+ ```bash
142
+ docker compose --env-file .env.proxy -f docker-compose.proxy.yml up --build
143
+ ```
144
+
145
+ See full guide: [`docs/compose-deployment.md`](docs/compose-deployment.md)
146
+
147
+ Build and preview site pages locally:
148
+
149
+ ```bash
150
+ pip install -e ".[dev,site]"
151
+ toolgauntlet build-site --pages-root site/pages/toolgauntlet --out-dir site/dist/toolgauntlet
152
+ python -m http.server 8787 --directory site/dist
153
+ ```
154
+
155
+ Key endpoints:
156
+ - `GET /healthz`
157
+ - `GET /metrics`
158
+ - `GET /v1/models`
159
+ - `POST /v1/chat/completions`
160
+ - `POST /v1/responses`
161
+ - `POST /admin/reload-policy`
162
+
163
+ Common environment variables:
164
+ - `OPENAI_API_KEY` or `BUDGET_UPSTREAM_API_KEY`
165
+ - `BUDGET_PROXY_API_KEY`
166
+ - `BUDGET_POLICY_PATH`
167
+ - `BUDGET_AUDIT_LOG_PATH`
168
+ - `BUDGET_LEDGER_PATH`
169
+ - `BUDGET_PROJECT_HEADER`
170
+ - `BUDGET_PROJECT_API_KEY_HEADER`
171
+ - `BUDGET_METRIC_NAMESPACE`
172
+ - `BUDGET_RATE_LIMIT_BACKEND` (`memory` or `redis`)
173
+ - `BUDGET_REDIS_URL` (required when backend is `redis`)
174
+ - `BUDGET_REDIS_KEY_PREFIX`
175
+ - `BUDGET_OTEL_ENABLED`
176
+ - `BUDGET_OTEL_METRICS_ENABLED`
177
+ - `BUDGET_OTEL_SERVICE_NAME`
178
+ - `BUDGET_OTEL_ENDPOINT`
179
+
180
+ To enable distributed rate limiting across proxy instances, install Redis support:
181
+
182
+ ```bash
183
+ pip install "toolgauntlet[redis]"
184
+ ```
185
+
186
+ ## Included suites
187
+ - `ecommerce_refunds_v1`
188
+ - `support_triage_v1`
189
+ - `travel_booking_v1`
190
+
191
+ ## Documentation
192
+ - docs index: [`docs/README.md`](docs/README.md)
193
+ - API reference: [`docs/api-reference.md`](docs/api-reference.md)
194
+ - proxy operations: [`docs/proxy-operations.md`](docs/proxy-operations.md)
195
+ - compose deployment: [`docs/compose-deployment.md`](docs/compose-deployment.md)
196
+ - site preview: [`docs/site-preview.md`](docs/site-preview.md)
197
+ - CI gate example: [`docs/ci-gate-example.md`](docs/ci-gate-example.md)
198
+ - commercial pack readiness: [`docs/commercial-pack-readiness.md`](docs/commercial-pack-readiness.md)
199
+ - customer pack handoff: [`docs/customer-pack-handoff.md`](docs/customer-pack-handoff.md)
200
+ - launch operations checklist: [`docs/launch-operations-checklist.md`](docs/launch-operations-checklist.md)
201
+ - AI venture operating plan: [`docs/venture-operating-plan.md`](docs/venture-operating-plan.md)
202
+ - AI venture operator: [`docs/venture-operator.md`](docs/venture-operator.md)
203
+ - release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
204
+ - versioning policy: [`docs/versioning-policy.md`](docs/versioning-policy.md)
205
+ - entitlement model: [`docs/entitlement-model.md`](docs/entitlement-model.md)
206
+ - support SLA: [`docs/support-sla.md`](docs/support-sla.md)
207
+
208
+ ## Marketing and launch materials
209
+ - materials index: [`marketing/README.md`](marketing/README.md)
210
+ - website copy: [`site/pages/toolgauntlet/`](site/pages/toolgauntlet)
211
+ - pilot sales path: [`site/pages/toolgauntlet/pilot.md`](site/pages/toolgauntlet/pilot.md) and [`docs/pilot-intake.md`](docs/pilot-intake.md)
212
+
213
+ ## Backlog and roadmap
214
+ - product backlog: [`TODO.md`](TODO.md)
215
+
216
+ ## Release workflows
217
+ GitHub Actions workflows included:
218
+ - `.github/workflows/ci.yml`
219
+ - `.github/workflows/launch-readiness.yml`
220
+ - `.github/workflows/venture-ops.yml`
221
+ - `.github/workflows/lead-intake.yml`
222
+ - `.github/workflows/release.yml`
223
+
224
+ Publishing uses trusted publishing via `pypa/gh-action-pypi-publish`.
@@ -0,0 +1,52 @@
1
+ # Agent Chaos Visual Package
2
+
3
+ Agent Chaos uses a controlled-chaos mark: calibration brackets, a fault-injection path, and a fixed center point. The intent is reliability engineering, not generic AI magic.
4
+
5
+ ![Agent Chaos visual package preview](./preview.png)
6
+
7
+ ## Assets
8
+
9
+ - `icon.svg`: primary app/package icon.
10
+ - `icon-dark.svg`: inverse icon for dark placements.
11
+ - `favicon.svg`: simplified 64px browser/package mark.
12
+ - `wordmark.svg`: type-only mark.
13
+ - `lockup-horizontal.svg`: default logo lockup for docs, README, and site headers.
14
+ - `lockup-horizontal-dark.svg`: inverse horizontal lockup.
15
+ - `lockup-stacked.svg`: compact centered layout for slides or square placements.
16
+ - `social-card.svg`: 1200x630 launch/social preview.
17
+ - `palette.css`: brand tokens for web usage.
18
+
19
+ ## Brand Tokens
20
+
21
+ | Token | Hex | Use |
22
+ | --- | --- | --- |
23
+ | `--ac-bg` | `#faf7f0` | warm page background |
24
+ | `--ac-surface` | `#f2ece0` | panels and secondary fills |
25
+ | `--ac-ink` | `#111111` | primary text and strokes |
26
+ | `--ac-muted` | `#6b6258` | captions, secondary text |
27
+ | `--ac-border` | `#d8cdbb` | separators and quiet structure |
28
+ | `--ac-accent` | `#c2532d` | the single brand accent |
29
+ | `--ac-accent-dark` | `#8f321c` | accessible accent text |
30
+
31
+ ## Usage Rules
32
+
33
+ - Use `lockup-horizontal.svg` as the default brand mark.
34
+ - Use `icon.svg` where square recognition matters: package pages, avatars, app cards.
35
+ - Keep the accent scarce. In product UI, use the rust accent for one status/detail per view, not every link.
36
+ - Preserve the warm-neutral background and black strokes; they are what keep the mark grounded.
37
+ - Do not put the icon inside a gradient, blob, or extra rounded-card container.
38
+ - Keep clear space around the icon equal to at least one center-dot diameter.
39
+
40
+ ## Typography Direction
41
+
42
+ The SVGs use system-first text stacks so they remain portable. For custom web work, pair:
43
+
44
+ - Display: `Aptos Display`, `Inter`, or `Söhne`-style grotesk.
45
+ - Interface/body: `Inter`, `Aptos`, or system UI.
46
+ - Technical labels: `SFMono-Regular`, `Menlo`, or `IBM Plex Mono`.
47
+
48
+ Use normal case for the wordmark. Reserve all-caps for the monospace descriptor and track it generously.
49
+
50
+ ## Rationale
51
+
52
+ The Open Design reference favors a warm neutral foundation, a single disciplined accent, useful structure, and typographic restraint. This package follows that: the icon looks like a test harness catching a failure signal, and the wordmark stays plain enough for developer tooling.
@@ -0,0 +1,41 @@
1
+ # ToolGauntlet Documentation
2
+
3
+ This folder is the long-form documentation set for ToolGauntlet.
4
+
5
+ ## Start here
6
+ - [User Guide](user-guide.md)
7
+ - [API Reference](api-reference.md)
8
+ - [Troubleshooting](troubleshooting.md)
9
+
10
+ ## Build and evaluate agents
11
+ - [Adapter Cookbook](adapter-cookbook.md)
12
+ - [CI Gate Example](ci-gate-example.md)
13
+ - [Sample Ecommerce Report](sample-reports/ecommerce_refunds_v1.md)
14
+ - [Suite Authoring](suite-authoring.md)
15
+ - [Injector Reference](injector-reference.md)
16
+ - [Scoring Reference](scoring-reference.md)
17
+
18
+ ## Run the Budget-as-Code Proxy
19
+ - [Proxy Operations Guide](proxy-operations.md)
20
+ - [Security Guide](security-guide.md)
21
+ - [Deployment Guide](deployment-guide.md)
22
+ - [Docker Compose Deployment](compose-deployment.md)
23
+ - [Release Checklist](release-checklist.md)
24
+
25
+ ## Commercialization and Operations
26
+ - [Pilot Intake Checklist](pilot-intake.md)
27
+ - [Commercial Pack Readiness](commercial-pack-readiness.md)
28
+ - [Customer Pack Handoff](customer-pack-handoff.md)
29
+ - [Launch Operations Checklist](launch-operations-checklist.md)
30
+ - [AI Venture Operating Plan](venture-operating-plan.md)
31
+ - [AI Venture Operator](venture-operator.md)
32
+ - [Versioning and Compatibility Policy](versioning-policy.md)
33
+ - [Entitlement and Access Model](entitlement-model.md)
34
+ - [Support and SLA Model](support-sla.md)
35
+
36
+ ## Site and Content
37
+ - [Site Preview and Publishing](site-preview.md)
38
+
39
+ ## Scope notes
40
+ - The docs in this folder describe the features currently implemented in this repository.
41
+ - Forward-looking items are tracked in [`TODO.md`](../TODO.md).
@@ -0,0 +1,9 @@
1
+ # Adapter Examples
2
+
3
+ This folder contains lightweight examples for integrating ToolGauntlet with common adapter shapes.
4
+
5
+ ## Files
6
+ - `async_adapter_example.py`: wraps an async agent callable via `make_async_adapter`.
7
+ - `langchain_like_adapter_example.py`: wraps an invoke-style runnable via `make_langchain_like_adapter`.
8
+
9
+ These examples are intentionally framework-agnostic and can be used as starting templates.
@@ -0,0 +1,6 @@
1
+ # Changelog
2
+
3
+ ## [0.1.0] - Draft
4
+
5
+ ### Added
6
+ - Placeholder paid pack layout for private suite development.
@@ -0,0 +1,5 @@
1
+ # Paid Pack Template
2
+
3
+ This directory documents the expected shape of a paid Chaos Pack without exposing private scenarios.
4
+
5
+ Copy this layout into a private pack repository, replace the placeholder suite content, then sign the completed suite directory with `toolgauntlet sign-pack`.