toolgauntlet 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- toolgauntlet-0.1.0/.gitignore +12 -0
- toolgauntlet-0.1.0/CHANGELOG.md +25 -0
- toolgauntlet-0.1.0/LICENSE +21 -0
- toolgauntlet-0.1.0/PKG-INFO +264 -0
- toolgauntlet-0.1.0/README.md +224 -0
- toolgauntlet-0.1.0/brand/agent-chaos/README.md +52 -0
- toolgauntlet-0.1.0/docs/README.md +41 -0
- toolgauntlet-0.1.0/examples/adapters/README.md +9 -0
- toolgauntlet-0.1.0/examples/paid_pack_template/CHANGELOG.md +6 -0
- toolgauntlet-0.1.0/examples/paid_pack_template/README.md +5 -0
- toolgauntlet-0.1.0/marketing/README.md +28 -0
- toolgauntlet-0.1.0/pyproject.toml +87 -0
- toolgauntlet-0.1.0/src/agent_chaos/__init__.py +8 -0
- toolgauntlet-0.1.0/src/agent_chaos/adapters/__init__.py +14 -0
- toolgauntlet-0.1.0/src/agent_chaos/adapters/demo.py +59 -0
- toolgauntlet-0.1.0/src/agent_chaos/adapters/helpers.py +105 -0
- toolgauntlet-0.1.0/src/agent_chaos/adapters/openai_compatible.py +209 -0
- toolgauntlet-0.1.0/src/agent_chaos/adapters/types.py +16 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/__init__.py +5 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/app.py +722 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/audit.py +27 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/cli.py +126 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/config.py +99 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/costing.py +33 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/enforcement.py +149 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/ledger.py +90 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/policy.py +139 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/rate_limit.py +119 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/redaction.py +41 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/types.py +73 -0
- toolgauntlet-0.1.0/src/agent_chaos/budget_proxy/utils.py +14 -0
- toolgauntlet-0.1.0/src/agent_chaos/cli.py +566 -0
- toolgauntlet-0.1.0/src/agent_chaos/config.py +22 -0
- toolgauntlet-0.1.0/src/agent_chaos/exceptions.py +18 -0
- toolgauntlet-0.1.0/src/agent_chaos/injectors/__init__.py +49 -0
- toolgauntlet-0.1.0/src/agent_chaos/injectors/base.py +62 -0
- toolgauntlet-0.1.0/src/agent_chaos/injectors/builtins.py +229 -0
- toolgauntlet-0.1.0/src/agent_chaos/loader.py +150 -0
- toolgauntlet-0.1.0/src/agent_chaos/models.py +407 -0
- toolgauntlet-0.1.0/src/agent_chaos/pack_signing.py +104 -0
- toolgauntlet-0.1.0/src/agent_chaos/redaction.py +52 -0
- toolgauntlet-0.1.0/src/agent_chaos/runner.py +420 -0
- toolgauntlet-0.1.0/src/agent_chaos/scoring.py +219 -0
- toolgauntlet-0.1.0/src/agent_chaos/site_builder.py +174 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/fixtures/tools.json +35 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/schemas/get_order.json +15 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/schemas/issue_refund.json +14 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/ecommerce_refunds_v1/suite.yaml +62 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/fixtures/tools.json +38 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/classify_ticket.json +13 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/escalate_ticket.json +13 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/schemas/get_ticket.json +14 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/support_triage_v1/suite.yaml +49 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/fixtures/tools.json +50 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/book_flight.json +14 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/get_travel_policy.json +13 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/schemas/search_flights.json +23 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites/travel_booking_v1/suite.yaml +54 -0
- toolgauntlet-0.1.0/src/agent_chaos/suites.py +97 -0
- toolgauntlet-0.1.0/src/agent_chaos/telemetry.py +177 -0
- toolgauntlet-0.1.0/src/agent_chaos/utils.py +26 -0
- toolgauntlet-0.1.0/src/toolgauntlet/__init__.py +8 -0
- toolgauntlet-0.1.0/src/toolgauntlet/adapters/__init__.py +21 -0
- toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/__init__.py +5 -0
- toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/app.py +5 -0
- toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/cli.py +9 -0
- toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/config.py +5 -0
- toolgauntlet-0.1.0/src/toolgauntlet/budget_proxy/types.py +5 -0
- toolgauntlet-0.1.0/src/toolgauntlet/cli.py +9 -0
- toolgauntlet-0.1.0/src/toolgauntlet/injectors/__init__.py +11 -0
- toolgauntlet-0.1.0/src/toolgauntlet/suites.py +17 -0
- toolgauntlet-0.1.0/src/toolgauntlet/telemetry.py +5 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to ToolGauntlet are tracked here.
|
|
4
|
+
|
|
5
|
+
The format follows Keep a Changelog conventions, and this project uses semantic versioning for the core Python package.
|
|
6
|
+
|
|
7
|
+
## [0.1.0] - Unreleased
|
|
8
|
+
|
|
9
|
+
### Added
|
|
10
|
+
- Deterministic chaos suite runner for tool-using agents.
|
|
11
|
+
- Built-in ecommerce refunds, support triage, and travel booking suites.
|
|
12
|
+
- CLI commands for listing suites, running suites, rendering reports, scaffolding suites, and signing/verifying suite packs.
|
|
13
|
+
- No-key `quickstart` command that generates safe baseline reports and regression-gate next steps.
|
|
14
|
+
- JSON, Markdown, and HTML report generation.
|
|
15
|
+
- Regression comparison support for CI gates.
|
|
16
|
+
- OpenAI-compatible, async, LangChain-like, and deterministic demo adapter helpers.
|
|
17
|
+
- Budget-as-Code proxy with policy enforcement, redacted audit logs, usage ledger, rate limits, Prometheus metrics, and optional OpenTelemetry export.
|
|
18
|
+
- Static documentation and marketing site builder.
|
|
19
|
+
- AI-led venture operating plan and staged external launch-readiness monitor.
|
|
20
|
+
- Draft-only AI pilot/support triage and weekly operating/marketing briefs with deterministic fallback.
|
|
21
|
+
- Private, self-updating AI venture dashboard with safe Markdown rendering and workflow evidence links.
|
|
22
|
+
- Configuration-backed release constraints and founder decisions that model output cannot rewrite.
|
|
23
|
+
- Private, deduplicated HiLo Labs website-lead ingestion with contact-isolated same-run AI triage.
|
|
24
|
+
- Release, support, security, deployment, entitlement, and versioning documentation.
|
|
25
|
+
- Collision-free `toolgauntlet` distribution, CLI, and public import, with the prerelease `agent-chaos` commands and `agent_chaos` import retained as compatibility aliases.
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Joseph Nilo
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: toolgauntlet
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic reliability testing for tool-using AI agents
|
|
5
|
+
Project-URL: Homepage, https://labs.hilomedia.com/products/toolgauntlet
|
|
6
|
+
Project-URL: Documentation, https://github.com/josephnilo/agent-chaos/tree/main/docs
|
|
7
|
+
Project-URL: Repository, https://github.com/josephnilo/agent-chaos
|
|
8
|
+
Project-URL: Issues, https://github.com/josephnilo/agent-chaos/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/josephnilo/agent-chaos/blob/main/CHANGELOG.md
|
|
10
|
+
Author: HiLo Labs
|
|
11
|
+
License: MIT
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Keywords: agents,chaos-engineering,evaluation,llm,testing
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Requires-Dist: fastapi>=0.115
|
|
24
|
+
Requires-Dist: httpx>=0.27
|
|
25
|
+
Requires-Dist: jsonschema>=4.21
|
|
26
|
+
Requires-Dist: prometheus-client>=0.20
|
|
27
|
+
Requires-Dist: pyyaml>=6.0
|
|
28
|
+
Requires-Dist: uvicorn>=0.30
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: markdown>=3.6; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest>=8.3; extra == 'dev'
|
|
32
|
+
Provides-Extra: otel
|
|
33
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.27; extra == 'otel'
|
|
34
|
+
Requires-Dist: opentelemetry-sdk>=1.27; extra == 'otel'
|
|
35
|
+
Provides-Extra: redis
|
|
36
|
+
Requires-Dist: redis>=5.0; extra == 'redis'
|
|
37
|
+
Provides-Extra: site
|
|
38
|
+
Requires-Dist: markdown>=3.6; extra == 'site'
|
|
39
|
+
Description-Content-Type: text/markdown
|
|
40
|
+
|
|
41
|
+
# ToolGauntlet
|
|
42
|
+
|
|
43
|
+
ToolGauntlet by HiLo Labs is a Python package and CLI for deterministic reliability testing of tool-using AI agents.
|
|
44
|
+
|
|
45
|
+
## Install
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
pip install toolgauntlet
|
|
49
|
+
toolgauntlet --version
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The distribution, primary CLI, and public Python import are all `toolgauntlet`.
|
|
53
|
+
The prerelease `agent-chaos` commands and `agent_chaos` import remain available
|
|
54
|
+
as compatibility aliases.
|
|
55
|
+
|
|
56
|
+
## Quick start (CLI)
|
|
57
|
+
|
|
58
|
+
Generate a complete local baseline with no API key:
|
|
59
|
+
|
|
60
|
+
```bash
|
|
61
|
+
toolgauntlet quickstart --out-dir toolgauntlet-quickstart
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
This writes JSON, Markdown, and HTML baseline reports plus a copyable next-step
|
|
65
|
+
regression command. It uses the built-in deterministic adapter with safe logging
|
|
66
|
+
enabled and refuses to overwrite its files unless `--force` is supplied.
|
|
67
|
+
|
|
68
|
+
Build a custom workflow manually:
|
|
69
|
+
|
|
70
|
+
```bash
|
|
71
|
+
toolgauntlet list-suites
|
|
72
|
+
toolgauntlet init-suite --id my_suite_v1 --out-dir suites
|
|
73
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 10 --out report.json
|
|
74
|
+
toolgauntlet report --in report.json --format md --out report.md
|
|
75
|
+
toolgauntlet report --in report.json --format html --out report.html
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Use a saved baseline as a release gate:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 25 --out baseline.json
|
|
82
|
+
toolgauntlet run \
|
|
83
|
+
--suite ecommerce_refunds_v1 \
|
|
84
|
+
--runs 25 \
|
|
85
|
+
--baseline baseline.json \
|
|
86
|
+
--fail-on-regression \
|
|
87
|
+
--regression-tolerance 1.0 \
|
|
88
|
+
--out candidate.json
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
See the copyable CI workflow in [`docs/ci-gate-example.md`](docs/ci-gate-example.md) and a sample report in [`docs/sample-reports/ecommerce_refunds_v1.md`](docs/sample-reports/ecommerce_refunds_v1.md).
|
|
92
|
+
|
|
93
|
+
Before release, run the same local gate used for MVP validation:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
scripts/release_gate.sh
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
Verify the configured external launch stage (live page now; registries as the
|
|
100
|
+
stage advances):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
python scripts/check_launch_readiness.py
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
Signed pack workflow:
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
export TOOLGAUNTLET_PACK_KEY=your-shared-signing-key
|
|
110
|
+
toolgauntlet sign-pack --suite suites/my_suite_v1
|
|
111
|
+
toolgauntlet verify-pack --suite suites/my_suite_v1
|
|
112
|
+
toolgauntlet run --suite suites/my_suite_v1 --verify-pack --out report.json
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
OpenTelemetry traces (optional):
|
|
116
|
+
|
|
117
|
+
```bash
|
|
118
|
+
pip install "toolgauntlet[otel]"
|
|
119
|
+
# Requires an OTLP/HTTP collector reachable at the default endpoint,
|
|
120
|
+
# or pass --otel-endpoint with your collector URL.
|
|
121
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 5 --otel-enabled --otel-metrics-enabled
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
## Quick start (Python)
|
|
125
|
+
|
|
126
|
+
```python
|
|
127
|
+
from toolgauntlet import SuiteConfig, run_suite
|
|
128
|
+
from toolgauntlet.adapters import deterministic_demo_agent
|
|
129
|
+
|
|
130
|
+
report = run_suite(
|
|
131
|
+
agent=deterministic_demo_agent,
|
|
132
|
+
suite_path="ecommerce_refunds_v1",
|
|
133
|
+
config=SuiteConfig(runs=10, concurrency=2),
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
report.save_json("report.json")
|
|
137
|
+
report.save_markdown("report.md")
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
## OpenAI-compatible adapter
|
|
141
|
+
|
|
142
|
+
```python
|
|
143
|
+
from toolgauntlet import SuiteConfig, run_suite
|
|
144
|
+
from toolgauntlet.adapters import OpenAICompatibleAdapterConfig, make_openai_tool_loop_adapter
|
|
145
|
+
|
|
146
|
+
agent = make_openai_tool_loop_adapter(
|
|
147
|
+
OpenAICompatibleAdapterConfig(
|
|
148
|
+
model="gpt-4o-mini",
|
|
149
|
+
base_url="https://api.openai.com",
|
|
150
|
+
)
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
report = run_suite(
|
|
154
|
+
agent=agent,
|
|
155
|
+
suite_path="ecommerce_refunds_v1",
|
|
156
|
+
config=SuiteConfig(runs=5, concurrency=1),
|
|
157
|
+
)
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
## Additional adapter helpers
|
|
161
|
+
- `make_async_adapter`: use async agent callables with the synchronous harness interface.
|
|
162
|
+
- `make_langchain_like_adapter`: adapt invoke/ainvoke-style runnables.
|
|
163
|
+
|
|
164
|
+
See examples in [`examples/adapters/`](examples/adapters/).
|
|
165
|
+
|
|
166
|
+
## Budget-as-Code Proxy
|
|
167
|
+
|
|
168
|
+
Start and validate:
|
|
169
|
+
|
|
170
|
+
```bash
|
|
171
|
+
toolgauntlet-proxy validate-policy --policy examples/budget_proxy/policy.example.yaml
|
|
172
|
+
toolgauntlet-proxy serve \
|
|
173
|
+
--host 127.0.0.1 \
|
|
174
|
+
--port 8080 \
|
|
175
|
+
--policy examples/budget_proxy/policy.example.yaml \
|
|
176
|
+
--ledger logs/usage-ledger.sqlite
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
One-command local stack (proxy + redis + optional otel/prometheus):
|
|
180
|
+
|
|
181
|
+
```bash
|
|
182
|
+
docker compose --env-file .env.proxy -f docker-compose.proxy.yml up --build
|
|
183
|
+
```
|
|
184
|
+
|
|
185
|
+
See full guide: [`docs/compose-deployment.md`](docs/compose-deployment.md)
|
|
186
|
+
|
|
187
|
+
Build and preview site pages locally:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
pip install -e ".[dev,site]"
|
|
191
|
+
toolgauntlet build-site --pages-root site/pages/toolgauntlet --out-dir site/dist/toolgauntlet
|
|
192
|
+
python -m http.server 8787 --directory site/dist
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
Key endpoints:
|
|
196
|
+
- `GET /healthz`
|
|
197
|
+
- `GET /metrics`
|
|
198
|
+
- `GET /v1/models`
|
|
199
|
+
- `POST /v1/chat/completions`
|
|
200
|
+
- `POST /v1/responses`
|
|
201
|
+
- `POST /admin/reload-policy`
|
|
202
|
+
|
|
203
|
+
Common environment variables:
|
|
204
|
+
- `OPENAI_API_KEY` or `BUDGET_UPSTREAM_API_KEY`
|
|
205
|
+
- `BUDGET_PROXY_API_KEY`
|
|
206
|
+
- `BUDGET_POLICY_PATH`
|
|
207
|
+
- `BUDGET_AUDIT_LOG_PATH`
|
|
208
|
+
- `BUDGET_LEDGER_PATH`
|
|
209
|
+
- `BUDGET_PROJECT_HEADER`
|
|
210
|
+
- `BUDGET_PROJECT_API_KEY_HEADER`
|
|
211
|
+
- `BUDGET_METRIC_NAMESPACE`
|
|
212
|
+
- `BUDGET_RATE_LIMIT_BACKEND` (`memory` or `redis`)
|
|
213
|
+
- `BUDGET_REDIS_URL` (required when backend is `redis`)
|
|
214
|
+
- `BUDGET_REDIS_KEY_PREFIX`
|
|
215
|
+
- `BUDGET_OTEL_ENABLED`
|
|
216
|
+
- `BUDGET_OTEL_METRICS_ENABLED`
|
|
217
|
+
- `BUDGET_OTEL_SERVICE_NAME`
|
|
218
|
+
- `BUDGET_OTEL_ENDPOINT`
|
|
219
|
+
|
|
220
|
+
To enable distributed rate limiting across proxy instances, install Redis support:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
pip install "toolgauntlet[redis]"
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
## Included suites
|
|
227
|
+
- `ecommerce_refunds_v1`
|
|
228
|
+
- `support_triage_v1`
|
|
229
|
+
- `travel_booking_v1`
|
|
230
|
+
|
|
231
|
+
## Documentation
|
|
232
|
+
- docs index: [`docs/README.md`](docs/README.md)
|
|
233
|
+
- API reference: [`docs/api-reference.md`](docs/api-reference.md)
|
|
234
|
+
- proxy operations: [`docs/proxy-operations.md`](docs/proxy-operations.md)
|
|
235
|
+
- compose deployment: [`docs/compose-deployment.md`](docs/compose-deployment.md)
|
|
236
|
+
- site preview: [`docs/site-preview.md`](docs/site-preview.md)
|
|
237
|
+
- CI gate example: [`docs/ci-gate-example.md`](docs/ci-gate-example.md)
|
|
238
|
+
- commercial pack readiness: [`docs/commercial-pack-readiness.md`](docs/commercial-pack-readiness.md)
|
|
239
|
+
- customer pack handoff: [`docs/customer-pack-handoff.md`](docs/customer-pack-handoff.md)
|
|
240
|
+
- launch operations checklist: [`docs/launch-operations-checklist.md`](docs/launch-operations-checklist.md)
|
|
241
|
+
- AI venture operating plan: [`docs/venture-operating-plan.md`](docs/venture-operating-plan.md)
|
|
242
|
+
- AI venture operator: [`docs/venture-operator.md`](docs/venture-operator.md)
|
|
243
|
+
- release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
|
|
244
|
+
- versioning policy: [`docs/versioning-policy.md`](docs/versioning-policy.md)
|
|
245
|
+
- entitlement model: [`docs/entitlement-model.md`](docs/entitlement-model.md)
|
|
246
|
+
- support SLA: [`docs/support-sla.md`](docs/support-sla.md)
|
|
247
|
+
|
|
248
|
+
## Marketing and launch materials
|
|
249
|
+
- materials index: [`marketing/README.md`](marketing/README.md)
|
|
250
|
+
- website copy: [`site/pages/toolgauntlet/`](site/pages/toolgauntlet)
|
|
251
|
+
- pilot sales path: [`site/pages/toolgauntlet/pilot.md`](site/pages/toolgauntlet/pilot.md) and [`docs/pilot-intake.md`](docs/pilot-intake.md)
|
|
252
|
+
|
|
253
|
+
## Backlog and roadmap
|
|
254
|
+
- product backlog: [`TODO.md`](TODO.md)
|
|
255
|
+
|
|
256
|
+
## Release workflows
|
|
257
|
+
GitHub Actions workflows included:
|
|
258
|
+
- `.github/workflows/ci.yml`
|
|
259
|
+
- `.github/workflows/launch-readiness.yml`
|
|
260
|
+
- `.github/workflows/venture-ops.yml`
|
|
261
|
+
- `.github/workflows/lead-intake.yml`
|
|
262
|
+
- `.github/workflows/release.yml`
|
|
263
|
+
|
|
264
|
+
Publishing uses trusted publishing via `pypa/gh-action-pypi-publish`.
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# ToolGauntlet
|
|
2
|
+
|
|
3
|
+
ToolGauntlet by HiLo Labs is a Python package and CLI for deterministic reliability testing of tool-using AI agents.
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install toolgauntlet
|
|
9
|
+
toolgauntlet --version
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
The distribution, primary CLI, and public Python import are all `toolgauntlet`.
|
|
13
|
+
The prerelease `agent-chaos` commands and `agent_chaos` import remain available
|
|
14
|
+
as compatibility aliases.
|
|
15
|
+
|
|
16
|
+
## Quick start (CLI)
|
|
17
|
+
|
|
18
|
+
Generate a complete local baseline with no API key:
|
|
19
|
+
|
|
20
|
+
```bash
|
|
21
|
+
toolgauntlet quickstart --out-dir toolgauntlet-quickstart
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
This writes JSON, Markdown, and HTML baseline reports plus a copyable next-step
|
|
25
|
+
regression command. It uses the built-in deterministic adapter with safe logging
|
|
26
|
+
enabled and refuses to overwrite its files unless `--force` is supplied.
|
|
27
|
+
|
|
28
|
+
Build a custom workflow manually:
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
toolgauntlet list-suites
|
|
32
|
+
toolgauntlet init-suite --id my_suite_v1 --out-dir suites
|
|
33
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 10 --out report.json
|
|
34
|
+
toolgauntlet report --in report.json --format md --out report.md
|
|
35
|
+
toolgauntlet report --in report.json --format html --out report.html
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Use a saved baseline as a release gate:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 25 --out baseline.json
|
|
42
|
+
toolgauntlet run \
|
|
43
|
+
--suite ecommerce_refunds_v1 \
|
|
44
|
+
--runs 25 \
|
|
45
|
+
--baseline baseline.json \
|
|
46
|
+
--fail-on-regression \
|
|
47
|
+
--regression-tolerance 1.0 \
|
|
48
|
+
--out candidate.json
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
See the copyable CI workflow in [`docs/ci-gate-example.md`](docs/ci-gate-example.md) and a sample report in [`docs/sample-reports/ecommerce_refunds_v1.md`](docs/sample-reports/ecommerce_refunds_v1.md).
|
|
52
|
+
|
|
53
|
+
Before release, run the same local gate used for MVP validation:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
scripts/release_gate.sh
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Verify the configured external launch stage (live page now; registries as the
|
|
60
|
+
stage advances):
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
python scripts/check_launch_readiness.py
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Signed pack workflow:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
export TOOLGAUNTLET_PACK_KEY=your-shared-signing-key
|
|
70
|
+
toolgauntlet sign-pack --suite suites/my_suite_v1
|
|
71
|
+
toolgauntlet verify-pack --suite suites/my_suite_v1
|
|
72
|
+
toolgauntlet run --suite suites/my_suite_v1 --verify-pack --out report.json
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
OpenTelemetry traces (optional):
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install "toolgauntlet[otel]"
|
|
79
|
+
# Requires an OTLP/HTTP collector reachable at the default endpoint,
|
|
80
|
+
# or pass --otel-endpoint with your collector URL.
|
|
81
|
+
toolgauntlet run --suite ecommerce_refunds_v1 --runs 5 --otel-enabled --otel-metrics-enabled
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Quick start (Python)
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from toolgauntlet import SuiteConfig, run_suite
|
|
88
|
+
from toolgauntlet.adapters import deterministic_demo_agent
|
|
89
|
+
|
|
90
|
+
report = run_suite(
|
|
91
|
+
agent=deterministic_demo_agent,
|
|
92
|
+
suite_path="ecommerce_refunds_v1",
|
|
93
|
+
config=SuiteConfig(runs=10, concurrency=2),
|
|
94
|
+
)
|
|
95
|
+
|
|
96
|
+
report.save_json("report.json")
|
|
97
|
+
report.save_markdown("report.md")
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
## OpenAI-compatible adapter
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from toolgauntlet import SuiteConfig, run_suite
|
|
104
|
+
from toolgauntlet.adapters import OpenAICompatibleAdapterConfig, make_openai_tool_loop_adapter
|
|
105
|
+
|
|
106
|
+
agent = make_openai_tool_loop_adapter(
|
|
107
|
+
OpenAICompatibleAdapterConfig(
|
|
108
|
+
model="gpt-4o-mini",
|
|
109
|
+
base_url="https://api.openai.com",
|
|
110
|
+
)
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
report = run_suite(
|
|
114
|
+
agent=agent,
|
|
115
|
+
suite_path="ecommerce_refunds_v1",
|
|
116
|
+
config=SuiteConfig(runs=5, concurrency=1),
|
|
117
|
+
)
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
## Additional adapter helpers
|
|
121
|
+
- `make_async_adapter`: use async agent callables with the synchronous harness interface.
|
|
122
|
+
- `make_langchain_like_adapter`: adapt invoke/ainvoke-style runnables.
|
|
123
|
+
|
|
124
|
+
See examples in [`examples/adapters/`](examples/adapters/).
|
|
125
|
+
|
|
126
|
+
## Budget-as-Code Proxy
|
|
127
|
+
|
|
128
|
+
Start and validate:
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
toolgauntlet-proxy validate-policy --policy examples/budget_proxy/policy.example.yaml
|
|
132
|
+
toolgauntlet-proxy serve \
|
|
133
|
+
--host 127.0.0.1 \
|
|
134
|
+
--port 8080 \
|
|
135
|
+
--policy examples/budget_proxy/policy.example.yaml \
|
|
136
|
+
--ledger logs/usage-ledger.sqlite
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
One-command local stack (proxy + redis + optional otel/prometheus):
|
|
140
|
+
|
|
141
|
+
```bash
|
|
142
|
+
docker compose --env-file .env.proxy -f docker-compose.proxy.yml up --build
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
See full guide: [`docs/compose-deployment.md`](docs/compose-deployment.md)
|
|
146
|
+
|
|
147
|
+
Build and preview site pages locally:
|
|
148
|
+
|
|
149
|
+
```bash
|
|
150
|
+
pip install -e ".[dev,site]"
|
|
151
|
+
toolgauntlet build-site --pages-root site/pages/toolgauntlet --out-dir site/dist/toolgauntlet
|
|
152
|
+
python -m http.server 8787 --directory site/dist
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Key endpoints:
|
|
156
|
+
- `GET /healthz`
|
|
157
|
+
- `GET /metrics`
|
|
158
|
+
- `GET /v1/models`
|
|
159
|
+
- `POST /v1/chat/completions`
|
|
160
|
+
- `POST /v1/responses`
|
|
161
|
+
- `POST /admin/reload-policy`
|
|
162
|
+
|
|
163
|
+
Common environment variables:
|
|
164
|
+
- `OPENAI_API_KEY` or `BUDGET_UPSTREAM_API_KEY`
|
|
165
|
+
- `BUDGET_PROXY_API_KEY`
|
|
166
|
+
- `BUDGET_POLICY_PATH`
|
|
167
|
+
- `BUDGET_AUDIT_LOG_PATH`
|
|
168
|
+
- `BUDGET_LEDGER_PATH`
|
|
169
|
+
- `BUDGET_PROJECT_HEADER`
|
|
170
|
+
- `BUDGET_PROJECT_API_KEY_HEADER`
|
|
171
|
+
- `BUDGET_METRIC_NAMESPACE`
|
|
172
|
+
- `BUDGET_RATE_LIMIT_BACKEND` (`memory` or `redis`)
|
|
173
|
+
- `BUDGET_REDIS_URL` (required when backend is `redis`)
|
|
174
|
+
- `BUDGET_REDIS_KEY_PREFIX`
|
|
175
|
+
- `BUDGET_OTEL_ENABLED`
|
|
176
|
+
- `BUDGET_OTEL_METRICS_ENABLED`
|
|
177
|
+
- `BUDGET_OTEL_SERVICE_NAME`
|
|
178
|
+
- `BUDGET_OTEL_ENDPOINT`
|
|
179
|
+
|
|
180
|
+
To enable distributed rate limiting across proxy instances, install Redis support:
|
|
181
|
+
|
|
182
|
+
```bash
|
|
183
|
+
pip install "toolgauntlet[redis]"
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Included suites
|
|
187
|
+
- `ecommerce_refunds_v1`
|
|
188
|
+
- `support_triage_v1`
|
|
189
|
+
- `travel_booking_v1`
|
|
190
|
+
|
|
191
|
+
## Documentation
|
|
192
|
+
- docs index: [`docs/README.md`](docs/README.md)
|
|
193
|
+
- API reference: [`docs/api-reference.md`](docs/api-reference.md)
|
|
194
|
+
- proxy operations: [`docs/proxy-operations.md`](docs/proxy-operations.md)
|
|
195
|
+
- compose deployment: [`docs/compose-deployment.md`](docs/compose-deployment.md)
|
|
196
|
+
- site preview: [`docs/site-preview.md`](docs/site-preview.md)
|
|
197
|
+
- CI gate example: [`docs/ci-gate-example.md`](docs/ci-gate-example.md)
|
|
198
|
+
- commercial pack readiness: [`docs/commercial-pack-readiness.md`](docs/commercial-pack-readiness.md)
|
|
199
|
+
- customer pack handoff: [`docs/customer-pack-handoff.md`](docs/customer-pack-handoff.md)
|
|
200
|
+
- launch operations checklist: [`docs/launch-operations-checklist.md`](docs/launch-operations-checklist.md)
|
|
201
|
+
- AI venture operating plan: [`docs/venture-operating-plan.md`](docs/venture-operating-plan.md)
|
|
202
|
+
- AI venture operator: [`docs/venture-operator.md`](docs/venture-operator.md)
|
|
203
|
+
- release checklist: [`docs/release-checklist.md`](docs/release-checklist.md)
|
|
204
|
+
- versioning policy: [`docs/versioning-policy.md`](docs/versioning-policy.md)
|
|
205
|
+
- entitlement model: [`docs/entitlement-model.md`](docs/entitlement-model.md)
|
|
206
|
+
- support SLA: [`docs/support-sla.md`](docs/support-sla.md)
|
|
207
|
+
|
|
208
|
+
## Marketing and launch materials
|
|
209
|
+
- materials index: [`marketing/README.md`](marketing/README.md)
|
|
210
|
+
- website copy: [`site/pages/toolgauntlet/`](site/pages/toolgauntlet)
|
|
211
|
+
- pilot sales path: [`site/pages/toolgauntlet/pilot.md`](site/pages/toolgauntlet/pilot.md) and [`docs/pilot-intake.md`](docs/pilot-intake.md)
|
|
212
|
+
|
|
213
|
+
## Backlog and roadmap
|
|
214
|
+
- product backlog: [`TODO.md`](TODO.md)
|
|
215
|
+
|
|
216
|
+
## Release workflows
|
|
217
|
+
GitHub Actions workflows included:
|
|
218
|
+
- `.github/workflows/ci.yml`
|
|
219
|
+
- `.github/workflows/launch-readiness.yml`
|
|
220
|
+
- `.github/workflows/venture-ops.yml`
|
|
221
|
+
- `.github/workflows/lead-intake.yml`
|
|
222
|
+
- `.github/workflows/release.yml`
|
|
223
|
+
|
|
224
|
+
Publishing uses trusted publishing via `pypa/gh-action-pypi-publish`.
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Agent Chaos Visual Package
|
|
2
|
+
|
|
3
|
+
Agent Chaos uses a controlled-chaos mark: calibration brackets, a fault-injection path, and a fixed center point. The intent is reliability engineering, not generic AI magic.
|
|
4
|
+
|
|
5
|
+

|
|
6
|
+
|
|
7
|
+
## Assets
|
|
8
|
+
|
|
9
|
+
- `icon.svg`: primary app/package icon.
|
|
10
|
+
- `icon-dark.svg`: inverse icon for dark placements.
|
|
11
|
+
- `favicon.svg`: simplified 64px browser/package mark.
|
|
12
|
+
- `wordmark.svg`: type-only mark.
|
|
13
|
+
- `lockup-horizontal.svg`: default logo lockup for docs, README, and site headers.
|
|
14
|
+
- `lockup-horizontal-dark.svg`: inverse horizontal lockup.
|
|
15
|
+
- `lockup-stacked.svg`: compact centered layout for slides or square placements.
|
|
16
|
+
- `social-card.svg`: 1200x630 launch/social preview.
|
|
17
|
+
- `palette.css`: brand tokens for web usage.
|
|
18
|
+
|
|
19
|
+
## Brand Tokens
|
|
20
|
+
|
|
21
|
+
| Token | Hex | Use |
|
|
22
|
+
| --- | --- | --- |
|
|
23
|
+
| `--ac-bg` | `#faf7f0` | warm page background |
|
|
24
|
+
| `--ac-surface` | `#f2ece0` | panels and secondary fills |
|
|
25
|
+
| `--ac-ink` | `#111111` | primary text and strokes |
|
|
26
|
+
| `--ac-muted` | `#6b6258` | captions, secondary text |
|
|
27
|
+
| `--ac-border` | `#d8cdbb` | separators and quiet structure |
|
|
28
|
+
| `--ac-accent` | `#c2532d` | the single brand accent |
|
|
29
|
+
| `--ac-accent-dark` | `#8f321c` | accessible accent text |
|
|
30
|
+
|
|
31
|
+
## Usage Rules
|
|
32
|
+
|
|
33
|
+
- Use `lockup-horizontal.svg` as the default brand mark.
|
|
34
|
+
- Use `icon.svg` where square recognition matters: package pages, avatars, app cards.
|
|
35
|
+
- Keep the accent scarce. In product UI, use the rust accent for one status/detail per view, not every link.
|
|
36
|
+
- Preserve the warm-neutral background and black strokes; they are what keep the mark grounded.
|
|
37
|
+
- Do not put the icon inside a gradient, blob, or extra rounded-card container.
|
|
38
|
+
- Keep clear space around the icon equal to at least one center-dot diameter.
|
|
39
|
+
|
|
40
|
+
## Typography Direction
|
|
41
|
+
|
|
42
|
+
The SVGs use system-first text stacks so they remain portable. For custom web work, pair:
|
|
43
|
+
|
|
44
|
+
- Display: `Aptos Display`, `Inter`, or `Söhne`-style grotesk.
|
|
45
|
+
- Interface/body: `Inter`, `Aptos`, or system UI.
|
|
46
|
+
- Technical labels: `SFMono-Regular`, `Menlo`, or `IBM Plex Mono`.
|
|
47
|
+
|
|
48
|
+
Use normal case for the wordmark. Reserve all-caps for the monospace descriptor and track it generously.
|
|
49
|
+
|
|
50
|
+
## Rationale
|
|
51
|
+
|
|
52
|
+
The Open Design reference favors a warm neutral foundation, a single disciplined accent, useful structure, and typographic restraint. This package follows that: the icon looks like a test harness catching a failure signal, and the wordmark stays plain enough for developer tooling.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# ToolGauntlet Documentation
|
|
2
|
+
|
|
3
|
+
This folder is the long-form documentation set for ToolGauntlet.
|
|
4
|
+
|
|
5
|
+
## Start here
|
|
6
|
+
- [User Guide](user-guide.md)
|
|
7
|
+
- [API Reference](api-reference.md)
|
|
8
|
+
- [Troubleshooting](troubleshooting.md)
|
|
9
|
+
|
|
10
|
+
## Build and evaluate agents
|
|
11
|
+
- [Adapter Cookbook](adapter-cookbook.md)
|
|
12
|
+
- [CI Gate Example](ci-gate-example.md)
|
|
13
|
+
- [Sample Ecommerce Report](sample-reports/ecommerce_refunds_v1.md)
|
|
14
|
+
- [Suite Authoring](suite-authoring.md)
|
|
15
|
+
- [Injector Reference](injector-reference.md)
|
|
16
|
+
- [Scoring Reference](scoring-reference.md)
|
|
17
|
+
|
|
18
|
+
## Run the Budget-as-Code Proxy
|
|
19
|
+
- [Proxy Operations Guide](proxy-operations.md)
|
|
20
|
+
- [Security Guide](security-guide.md)
|
|
21
|
+
- [Deployment Guide](deployment-guide.md)
|
|
22
|
+
- [Docker Compose Deployment](compose-deployment.md)
|
|
23
|
+
- [Release Checklist](release-checklist.md)
|
|
24
|
+
|
|
25
|
+
## Commercialization and Operations
|
|
26
|
+
- [Pilot Intake Checklist](pilot-intake.md)
|
|
27
|
+
- [Commercial Pack Readiness](commercial-pack-readiness.md)
|
|
28
|
+
- [Customer Pack Handoff](customer-pack-handoff.md)
|
|
29
|
+
- [Launch Operations Checklist](launch-operations-checklist.md)
|
|
30
|
+
- [AI Venture Operating Plan](venture-operating-plan.md)
|
|
31
|
+
- [AI Venture Operator](venture-operator.md)
|
|
32
|
+
- [Versioning and Compatibility Policy](versioning-policy.md)
|
|
33
|
+
- [Entitlement and Access Model](entitlement-model.md)
|
|
34
|
+
- [Support and SLA Model](support-sla.md)
|
|
35
|
+
|
|
36
|
+
## Site and Content
|
|
37
|
+
- [Site Preview and Publishing](site-preview.md)
|
|
38
|
+
|
|
39
|
+
## Scope notes
|
|
40
|
+
- The docs in this folder describe the features currently implemented in this repository.
|
|
41
|
+
- Forward-looking items are tracked in [`TODO.md`](../TODO.md).
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# Adapter Examples
|
|
2
|
+
|
|
3
|
+
This folder contains lightweight examples for integrating ToolGauntlet with common adapter shapes.
|
|
4
|
+
|
|
5
|
+
## Files
|
|
6
|
+
- `async_adapter_example.py`: wraps an async agent callable via `make_async_adapter`.
|
|
7
|
+
- `langchain_like_adapter_example.py`: wraps an invoke-style runnable via `make_langchain_like_adapter`.
|
|
8
|
+
|
|
9
|
+
These examples are intentionally framework-agnostic and can be used as starting templates.
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# Paid Pack Template
|
|
2
|
+
|
|
3
|
+
This directory documents the expected shape of a paid Chaos Pack without exposing private scenarios.
|
|
4
|
+
|
|
5
|
+
Copy this layout into a private pack repository, replace the placeholder suite content, then sign the completed suite directory with `toolgauntlet sign-pack`.
|