openapi-agent-ready 0.2.1__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openapi_agent_ready-0.4.0/CHANGELOG.md +162 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/PKG-INFO +53 -20
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/README.md +52 -19
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/pyproject.toml +1 -1
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/auditor.py +9 -3
- openapi_agent_ready-0.4.0/src/agent_ready/baseline.py +137 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/cli.py +112 -2
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/rubric.py +54 -12
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/tests/test_audit.py +86 -0
- openapi_agent_ready-0.4.0/tests/test_baseline.py +306 -0
- openapi_agent_ready-0.2.1/CHANGELOG.md +0 -129
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/.github/workflows/ci.yml +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/.github/workflows/publish.yml +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/.gitignore +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/CONTRIBUTING.md +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/LICENSE +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/examples/agent_ready_booking_api.yaml +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/examples/sample_booking_api.yaml +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/openapi.yaml +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/__init__.py +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/mcp_generator.py +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/src/agent_ready/report_generator.py +0 -0
- {openapi_agent_ready-0.2.1 → openapi_agent_ready-0.4.0}/tests/test_mcp_conformance.py +0 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here.
|
|
4
|
+
Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.4.0] - 2026-08-08
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- **Baseline mode** (`--write-baseline`, `--baseline`, `--max-new`). A mature
|
|
12
|
+
API audited for the first time produces hundreds of findings, so a score gate
|
|
13
|
+
is unusable on day one and gets switched off. Recording the current state lets
|
|
14
|
+
CI block only regressions while the backlog stays visible.
|
|
15
|
+
|
|
16
|
+
Findings are matched on category, endpoint, and severity rather than message
|
|
17
|
+
text — messages embed counts and parameter names ("3/3 parameters have no
|
|
18
|
+
description (a, b, c)") that shift when unrelated things change, which would
|
|
19
|
+
report unfixed problems as new. Severity is part of the identity so a warning
|
|
20
|
+
escalating to a fail is caught; message is excluded so rewording a finding in
|
|
21
|
+
a future release doesn't invalidate stored baselines.
|
|
22
|
+
|
|
23
|
+
## [0.3.0] - 2026-07-31
|
|
24
|
+
|
|
25
|
+
### Added
|
|
26
|
+
- **Progress indicator for long audits.** The ambiguity check dominates runtime
|
|
27
|
+
on large specs and previously produced no output while running — a 587-endpoint
|
|
28
|
+
spec looked like the tool had hung. A progress bar now reports on stderr, shown
|
|
29
|
+
only for specs above 150 endpoints and only when stderr is a terminal, so piped
|
|
30
|
+
output and `--format json` stay clean.
|
|
31
|
+
|
|
32
|
+
### Changed
|
|
33
|
+
- **Ambiguity check is ~4.7x faster** (Stripe: 77s to 16s; Asana: 6.4s to 1.6s)
|
|
34
|
+
using `SequenceMatcher`'s `real_quick_ratio` and `quick_ratio` as prefilters.
|
|
35
|
+
Both are documented upper bounds on `ratio()`, so a pair rejected by a cheap
|
|
36
|
+
bound could not have exceeded the threshold anyway. Results are verified
|
|
37
|
+
identical to the unoptimised implementation.
|
|
38
|
+
|
|
39
|
+
Truncating comparison text was also tried and rejected: another ~5x, but it
|
|
40
|
+
changed which pairs were reported. A faster check that disagrees with itself
|
|
41
|
+
is not an optimisation.
|
|
42
|
+
|
|
43
|
+
A subtler trap surfaced here too: `SequenceMatcher.ratio()` is not symmetric,
|
|
44
|
+
because the autojunk heuristic applies to the second sequence only. The first
|
|
45
|
+
version of this optimisation reversed the argument order to reuse a cached
|
|
46
|
+
sequence — it was faster, passed every existing test, and silently reported
|
|
47
|
+
different pairs. Results are now pinned against a reference implementation.
|
|
48
|
+
|
|
49
|
+
## [0.2.1] - 2026-07-31
|
|
50
|
+
|
|
51
|
+
Found by running the published 0.2.0 against Asana's public spec.
|
|
52
|
+
|
|
53
|
+
### Fixed
|
|
54
|
+
- **Generated scaffolds could contain duplicate arguments and fail to parse.**
|
|
55
|
+
OpenAPI identifies a parameter by name *and* location, and an operation-level
|
|
56
|
+
declaration overrides a path-level one. The extractor concatenated both lists
|
|
57
|
+
without deduplicating, so a spec declaring the same parameter at each level —
|
|
58
|
+
Asana's declares `project_gid` that way — produced a function with two
|
|
59
|
+
identically-named arguments, a Python `SyntaxError`. Parameters declared at
|
|
60
|
+
both levels are now deduplicated with operation-level winning; the same name
|
|
61
|
+
in a different location is still treated as distinct.
|
|
62
|
+
- **`--version` reported a stale hardcoded number.** The CLI printed `0.1.0`
|
|
63
|
+
from a published 0.2.0 install. The version is now read from installed package
|
|
64
|
+
metadata, so `pyproject.toml` is the single source of truth and cannot drift.
|
|
65
|
+
|
|
66
|
+
## [0.2.0] - 2026-07-30
|
|
67
|
+
|
|
68
|
+
The generated MCP scaffold is now verified against the real protocol rather than
|
|
69
|
+
merely checked for syntax — which immediately surfaced two bugs that made
|
|
70
|
+
generated servers uncallable, and a third that stopped them starting at all
|
|
71
|
+
against the current SDK.
|
|
72
|
+
|
|
73
|
+
**Anyone on 0.1.0 should upgrade:** scaffolds generated by 0.1.0 fail to import
|
|
74
|
+
against mcp 2.x.
|
|
75
|
+
|
|
76
|
+
### Added
|
|
77
|
+
- **MCP conformance test suite.** Launches the generated scaffold as a
|
|
78
|
+
subprocess, completes a real MCP handshake over stdio using the official
|
|
79
|
+
client, and validates the advertised tool definitions: name format,
|
|
80
|
+
uniqueness, description presence, input-schema validity as JSON Schema,
|
|
81
|
+
required-argument correctness, and side-effect disclosure. Previously the only
|
|
82
|
+
check on generated output was that the file compiled.
|
|
83
|
+
- **Tool Surface & Schema Strictness** category (category 8), derived from
|
|
84
|
+
OpenAI's function-calling guidance: flags catalogues far above the ~20-tool
|
|
85
|
+
ceiling, request bodies incompatible with strict mode
|
|
86
|
+
(`additionalProperties: false`, all properties required), and parameters that
|
|
87
|
+
imply a fixed value set but accept free-form strings where an enum belongs.
|
|
88
|
+
- **`ToolAnnotations` on generated tools** (`readOnlyHint`, `destructiveHint`,
|
|
89
|
+
`idempotentHint`, `openWorldHint`), derived from the HTTP method. Prose in a
|
|
90
|
+
description helps a model reason about side effects; annotations let a client
|
|
91
|
+
*enforce* — auto-approving reads while requiring confirmation for writes.
|
|
92
|
+
- **`[server]` extra**, a clearer alias for `[mcp]`: these are dependencies the
|
|
93
|
+
*generated* server needs at runtime, not dependencies of the auditor.
|
|
94
|
+
- Findings now carry a `scope` ("endpoint" or "spec"). Spec-level findings are
|
|
95
|
+
weighted at 50% of their category so a single whole-API finding isn't averaged
|
|
96
|
+
away against hundreds of endpoint findings. Stripe's tool-surface score was
|
|
97
|
+
99/100 before this fix and 49.6/100 after.
|
|
98
|
+
|
|
99
|
+
### Fixed
|
|
100
|
+
- **Generated scaffolds failed to import on mcp 2.0.** FastMCP was replaced by
|
|
101
|
+
`MCPServer` in the 2.x SDK. The scaffold now imports through a compatibility
|
|
102
|
+
shim supporting both generations, verified against mcp 1.28.1 and 2.0.0.
|
|
103
|
+
- **Request body fields were silently dropped.** The generator read only
|
|
104
|
+
`parameters`, so a `POST` endpoint's payload properties never became tool
|
|
105
|
+
arguments — producing, for example, a create-booking tool with no way to say
|
|
106
|
+
what to book. Syntactically valid and completely uncallable, which is exactly
|
|
107
|
+
the class of bug a compile check cannot catch.
|
|
108
|
+
- **No arguments were marked required.** Every parameter was emitted with a
|
|
109
|
+
`= None` default, so the advertised input schema had an empty `required`
|
|
110
|
+
array. A model could call an endpoint with a path parameter and omit it,
|
|
111
|
+
producing a malformed URL. Path parameters are now always required, and
|
|
112
|
+
`required: true` from the spec is honoured for query, header, and body fields.
|
|
113
|
+
- **Conformance tests silently skipped under mcp 2.0.** The module guard probed
|
|
114
|
+
a 1.x-only import path, so the whole suite skipped against a current install —
|
|
115
|
+
a green build that verified nothing. The guard now probes for either server
|
|
116
|
+
class, and CI fails explicitly if the suite skips rather than runs.
|
|
117
|
+
- **Conformance assertions were pinned to 1.x field names.** mcp 2.0 renamed
|
|
118
|
+
result fields to snake_case (`serverInfo` → `server_info`, `inputSchema` →
|
|
119
|
+
`input_schema`, `readOnlyHint` → `read_only_hint`). Assertions now read fields
|
|
120
|
+
through version-agnostic accessors; the suite passes on both 1.28.1 and 2.0.0.
|
|
121
|
+
- **Failures inside anyio task groups are now readable.** Exception groups are
|
|
122
|
+
flattened to their leaf causes and `BaseException` is caught rather than
|
|
123
|
+
`Exception`, so a `BaseExceptionGroup` no longer escapes the diagnostic
|
|
124
|
+
wrapper and leaves the real error truncated.
|
|
125
|
+
|
|
126
|
+
### Changed
|
|
127
|
+
- **Distribution renamed to `openapi-agent-ready`.** PyPI rejected `agent-ready`
|
|
128
|
+
as too similar to the existing `agentready` package — its similarity check
|
|
129
|
+
strips separators before comparing. Import name (`agent_ready`), CLI command
|
|
130
|
+
(`agent-ready`), and repository name are unchanged.
|
|
131
|
+
- **Generated scaffolds are self-documenting.** The file header states its
|
|
132
|
+
dependencies, how to run it, a paste-ready Claude Desktop config block, and
|
|
133
|
+
three things to fix before shipping — notably that the scaffold sends no
|
|
134
|
+
credentials, so calls against a real API will 401 until auth is wired in.
|
|
135
|
+
- **Generated servers are named after their source API** (`spotify-web-api`
|
|
136
|
+
rather than a shared `generated-api-server` placeholder), so two connected to
|
|
137
|
+
the same client no longer present identical names.
|
|
138
|
+
- Tool listings are fetched once per generated file and cached rather than
|
|
139
|
+
spawning a server subprocess per test, cutting spawns from ~14 to 4.
|
|
140
|
+
|
|
141
|
+
## [0.1.0] - 2026-07-28
|
|
142
|
+
|
|
143
|
+
Initial release.
|
|
144
|
+
|
|
145
|
+
### Added
|
|
146
|
+
- Seven-category AI-readiness rubric for OpenAPI specs
|
|
147
|
+
- CLI with text, markdown, and JSON output formats
|
|
148
|
+
- CI gate support via `--min-score` and `--max-fails` with documented exit codes
|
|
149
|
+
- Remote spec loading over http(s)
|
|
150
|
+
- Parameter `$ref` resolution and path-level shared parameter inheritance
|
|
151
|
+
- MCP server scaffold generation for endpoints that pass the audit
|
|
152
|
+
- Python API (`load_spec`, `run_audit`)
|
|
153
|
+
|
|
154
|
+
### Fixed
|
|
155
|
+
- OpenAPI `default` responses now count as error documentation. Previously only
|
|
156
|
+
4xx/5xx codes were recognised, which failed every endpoint in specs that use
|
|
157
|
+
`default` — including Stripe's entire public API. (Found by auditing Stripe.)
|
|
158
|
+
- Ambiguity findings are capped at the 25 most-similar pairs with a summary
|
|
159
|
+
line, so large catalogues produce a readable report.
|
|
160
|
+
- Usage-guidelines detection is now regex-based; literal matching missed
|
|
161
|
+
phrasings like "use this only when" and "use this only after".
|
|
162
|
+
- CLI exits cleanly when its output is piped to `head` or `less`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: openapi-agent-ready
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: Audit any OpenAPI spec for AI-agent and MCP readiness, and generate an MCP server scaffold.
|
|
5
5
|
Project-URL: Homepage, https://github.com/sfaisal/agent-ready
|
|
6
6
|
Project-URL: Issues, https://github.com/sfaisal/agent-ready/issues
|
|
@@ -75,18 +75,17 @@ agent-ready https://example.com/openapi.yaml
|
|
|
75
75
|
|
|
76
76
|
```
|
|
77
77
|
Generic Booking Platform API (v1.0)
|
|
78
|
-
AI-readiness score:
|
|
79
|
-
Endpoints: 4 Fails:
|
|
78
|
+
AI-readiness score: 54.4/100
|
|
79
|
+
Endpoints: 4 Fails: 6 Warnings: 12
|
|
80
80
|
|
|
81
81
|
Category scores:
|
|
82
|
-
Description Clarity
|
|
83
|
-
Side Effects
|
|
84
|
-
Error Responses
|
|
85
|
-
Auth Clarity
|
|
86
|
-
Ambiguity
|
|
87
|
-
Parameter Explanation
|
|
88
|
-
Usage Guidelines
|
|
89
|
-
Tool Surface 100.0/100
|
|
82
|
+
Description Clarity 37.5/100
|
|
83
|
+
Side Effects 70.0/100
|
|
84
|
+
Error Responses 62.5/100
|
|
85
|
+
Auth Clarity 50.0/100
|
|
86
|
+
Ambiguity 100.0/100
|
|
87
|
+
Parameter Explanation 16.7/100
|
|
88
|
+
Usage Guidelines 50.0/100
|
|
90
89
|
```
|
|
91
90
|
|
|
92
91
|
## Why this exists
|
|
@@ -128,7 +127,7 @@ agent failures in production.
|
|
|
128
127
|
|
|
129
128
|
```bash
|
|
130
129
|
pip install openapi-agent-ready # core
|
|
131
|
-
pip install "openapi-agent-ready[server]"
|
|
130
|
+
pip install "openapi-agent-ready[server]" # + deps to RUN a generated scaffold
|
|
132
131
|
```
|
|
133
132
|
|
|
134
133
|
From source:
|
|
@@ -156,6 +155,10 @@ agent-ready openapi.yaml --format json
|
|
|
156
155
|
|
|
157
156
|
# Generate an MCP server for the endpoints that pass
|
|
158
157
|
agent-ready openapi.yaml --generate-mcp server.py
|
|
158
|
+
|
|
159
|
+
# Generating only writes a file — running it needs the server deps:
|
|
160
|
+
pip install "openapi-agent-ready[server]"
|
|
161
|
+
python server.py
|
|
159
162
|
```
|
|
160
163
|
|
|
161
164
|
### Use it as a CI gate
|
|
@@ -180,9 +183,42 @@ Exit codes:
|
|
|
180
183
|
| Code | Meaning |
|
|
181
184
|
|---|---|
|
|
182
185
|
| `0` | All configured gates passed |
|
|
183
|
-
| `1` | A gate failed (`--min-score` / `--max-fails`) |
|
|
186
|
+
| `1` | A gate failed (`--min-score` / `--max-fails` / `--max-new`) |
|
|
184
187
|
| `2` | The spec could not be loaded or parsed |
|
|
185
188
|
|
|
189
|
+
### Adopting on an existing API
|
|
190
|
+
|
|
191
|
+
A mature API audited for the first time will produce hundreds of findings. No
|
|
192
|
+
team can fix all of them before merging anything, so a score gate is unusable
|
|
193
|
+
on day one and gets switched off within a week.
|
|
194
|
+
|
|
195
|
+
Record the current state instead, and block only what gets worse:
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# once, when you adopt the tool — commit this file
|
|
199
|
+
agent-ready openapi.yaml --write-baseline .agent-ready-baseline.json
|
|
200
|
+
|
|
201
|
+
# in CI, from then on
|
|
202
|
+
agent-ready openapi.yaml --baseline .agent-ready-baseline.json --max-new 0
|
|
203
|
+
```
|
|
204
|
+
|
|
205
|
+
Existing findings stay visible in the report but don't fail the build. A pull
|
|
206
|
+
request that introduces a new one does, and the output names exactly which
|
|
207
|
+
findings it added:
|
|
208
|
+
|
|
209
|
+
```
|
|
210
|
+
Compared against .agent-ready-baseline.json: 5 new, 0 fixed, 11 unchanged
|
|
211
|
+
NEW FAIL GET /invoices — Description is a bare label (12 chars) ...
|
|
212
|
+
NEW FAIL GET /invoices — No error responses documented ...
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
Findings are matched on category, endpoint, and severity — not on message text,
|
|
216
|
+
which contains counts and parameter names that shift when unrelated things
|
|
217
|
+
change. A warning escalating to a fail counts as new; rewording a finding in a
|
|
218
|
+
future release does not invalidate your baseline.
|
|
219
|
+
|
|
220
|
+
As the team fixes the backlog, regenerate the baseline to lock in the progress.
|
|
221
|
+
|
|
186
222
|
## Real-world results
|
|
187
223
|
|
|
188
224
|
Running it against Stripe's production spec (587 endpoints), as of v0.1.0.
|
|
@@ -191,10 +227,8 @@ numbers:
|
|
|
191
227
|
|
|
192
228
|
```
|
|
193
229
|
Stripe API (v2026-06-24.dahlia)
|
|
194
|
-
AI-readiness score:
|
|
195
|
-
Endpoints: 587 Fails: 393 Warnings: 2133
|
|
230
|
+
AI-readiness score: 62.9/100
|
|
196
231
|
|
|
197
|
-
Category scores:
|
|
198
232
|
Description Clarity 68.3/100
|
|
199
233
|
Side Effects 64.7/100
|
|
200
234
|
Error Responses 100.0/100
|
|
@@ -202,7 +236,6 @@ Category scores:
|
|
|
202
236
|
Ambiguity 50.0/100
|
|
203
237
|
Parameter Explanation 43.0/100
|
|
204
238
|
Usage Guidelines 51.2/100
|
|
205
|
-
Tool Surface 49.6/100
|
|
206
239
|
```
|
|
207
240
|
|
|
208
241
|
Two things worth drawing out.
|
|
@@ -211,11 +244,11 @@ Two things worth drawing out.
|
|
|
211
244
|
in this tool, not in Stripe.** Stripe documents errors via OpenAPI's `default`
|
|
212
245
|
response rather than enumerating explicit 4xx/5xx codes. Perfectly valid, and
|
|
213
246
|
arguably cleaner. The check only looked for `4`/`5` prefixes, so it failed all
|
|
214
|
-
587 endpoints. Fixed, regression-tested, and the reason
|
|
215
|
-
real spec before trusting it.
|
|
247
|
+
587 endpoints. Fixed, regression-tested, and the reason the README says to run
|
|
248
|
+
this against a real spec before trusting it.
|
|
216
249
|
|
|
217
250
|
**Stripe is widely and correctly regarded as agent-ready, yet its raw OpenAPI
|
|
218
|
-
spec scores
|
|
251
|
+
spec scores 62.9.** That isn't a contradiction — it's the whole point. Stripe's
|
|
219
252
|
agent-readiness comes from the *Agent Toolkit* layered on top: curated tool
|
|
220
253
|
definitions, an MCP server, managed OAuth. The raw spec is the input to that
|
|
221
254
|
layer, not the layer itself. Which is exactly why this tool ships with
|
|
@@ -20,18 +20,17 @@ agent-ready https://example.com/openapi.yaml
|
|
|
20
20
|
|
|
21
21
|
```
|
|
22
22
|
Generic Booking Platform API (v1.0)
|
|
23
|
-
AI-readiness score:
|
|
24
|
-
Endpoints: 4 Fails:
|
|
23
|
+
AI-readiness score: 54.4/100
|
|
24
|
+
Endpoints: 4 Fails: 6 Warnings: 12
|
|
25
25
|
|
|
26
26
|
Category scores:
|
|
27
|
-
Description Clarity
|
|
28
|
-
Side Effects
|
|
29
|
-
Error Responses
|
|
30
|
-
Auth Clarity
|
|
31
|
-
Ambiguity
|
|
32
|
-
Parameter Explanation
|
|
33
|
-
Usage Guidelines
|
|
34
|
-
Tool Surface 100.0/100
|
|
27
|
+
Description Clarity 37.5/100
|
|
28
|
+
Side Effects 70.0/100
|
|
29
|
+
Error Responses 62.5/100
|
|
30
|
+
Auth Clarity 50.0/100
|
|
31
|
+
Ambiguity 100.0/100
|
|
32
|
+
Parameter Explanation 16.7/100
|
|
33
|
+
Usage Guidelines 50.0/100
|
|
35
34
|
```
|
|
36
35
|
|
|
37
36
|
## Why this exists
|
|
@@ -73,7 +72,7 @@ agent failures in production.
|
|
|
73
72
|
|
|
74
73
|
```bash
|
|
75
74
|
pip install openapi-agent-ready # core
|
|
76
|
-
pip install "openapi-agent-ready[server]"
|
|
75
|
+
pip install "openapi-agent-ready[server]" # + deps to RUN a generated scaffold
|
|
77
76
|
```
|
|
78
77
|
|
|
79
78
|
From source:
|
|
@@ -101,6 +100,10 @@ agent-ready openapi.yaml --format json
|
|
|
101
100
|
|
|
102
101
|
# Generate an MCP server for the endpoints that pass
|
|
103
102
|
agent-ready openapi.yaml --generate-mcp server.py
|
|
103
|
+
|
|
104
|
+
# Generating only writes a file — running it needs the server deps:
|
|
105
|
+
pip install "openapi-agent-ready[server]"
|
|
106
|
+
python server.py
|
|
104
107
|
```
|
|
105
108
|
|
|
106
109
|
### Use it as a CI gate
|
|
@@ -125,9 +128,42 @@ Exit codes:
|
|
|
125
128
|
| Code | Meaning |
|
|
126
129
|
|---|---|
|
|
127
130
|
| `0` | All configured gates passed |
|
|
128
|
-
| `1` | A gate failed (`--min-score` / `--max-fails`) |
|
|
131
|
+
| `1` | A gate failed (`--min-score` / `--max-fails` / `--max-new`) |
|
|
129
132
|
| `2` | The spec could not be loaded or parsed |
|
|
130
133
|
|
|
134
|
+
### Adopting on an existing API
|
|
135
|
+
|
|
136
|
+
A mature API audited for the first time will produce hundreds of findings. No
|
|
137
|
+
team can fix all of them before merging anything, so a score gate is unusable
|
|
138
|
+
on day one and gets switched off within a week.
|
|
139
|
+
|
|
140
|
+
Record the current state instead, and block only what gets worse:
|
|
141
|
+
|
|
142
|
+
```bash
|
|
143
|
+
# once, when you adopt the tool — commit this file
|
|
144
|
+
agent-ready openapi.yaml --write-baseline .agent-ready-baseline.json
|
|
145
|
+
|
|
146
|
+
# in CI, from then on
|
|
147
|
+
agent-ready openapi.yaml --baseline .agent-ready-baseline.json --max-new 0
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
Existing findings stay visible in the report but don't fail the build. A pull
|
|
151
|
+
request that introduces a new one does, and the output names exactly which
|
|
152
|
+
findings it added:
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
Compared against .agent-ready-baseline.json: 5 new, 0 fixed, 11 unchanged
|
|
156
|
+
NEW FAIL GET /invoices — Description is a bare label (12 chars) ...
|
|
157
|
+
NEW FAIL GET /invoices — No error responses documented ...
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Findings are matched on category, endpoint, and severity — not on message text,
|
|
161
|
+
which contains counts and parameter names that shift when unrelated things
|
|
162
|
+
change. A warning escalating to a fail counts as new; rewording a finding in a
|
|
163
|
+
future release does not invalidate your baseline.
|
|
164
|
+
|
|
165
|
+
As the team fixes the backlog, regenerate the baseline to lock in the progress.
|
|
166
|
+
|
|
131
167
|
## Real-world results
|
|
132
168
|
|
|
133
169
|
Running it against Stripe's production spec (587 endpoints), as of v0.1.0.
|
|
@@ -136,10 +172,8 @@ numbers:
|
|
|
136
172
|
|
|
137
173
|
```
|
|
138
174
|
Stripe API (v2026-06-24.dahlia)
|
|
139
|
-
AI-readiness score:
|
|
140
|
-
Endpoints: 587 Fails: 393 Warnings: 2133
|
|
175
|
+
AI-readiness score: 62.9/100
|
|
141
176
|
|
|
142
|
-
Category scores:
|
|
143
177
|
Description Clarity 68.3/100
|
|
144
178
|
Side Effects 64.7/100
|
|
145
179
|
Error Responses 100.0/100
|
|
@@ -147,7 +181,6 @@ Category scores:
|
|
|
147
181
|
Ambiguity 50.0/100
|
|
148
182
|
Parameter Explanation 43.0/100
|
|
149
183
|
Usage Guidelines 51.2/100
|
|
150
|
-
Tool Surface 49.6/100
|
|
151
184
|
```
|
|
152
185
|
|
|
153
186
|
Two things worth drawing out.
|
|
@@ -156,11 +189,11 @@ Two things worth drawing out.
|
|
|
156
189
|
in this tool, not in Stripe.** Stripe documents errors via OpenAPI's `default`
|
|
157
190
|
response rather than enumerating explicit 4xx/5xx codes. Perfectly valid, and
|
|
158
191
|
arguably cleaner. The check only looked for `4`/`5` prefixes, so it failed all
|
|
159
|
-
587 endpoints. Fixed, regression-tested, and the reason
|
|
160
|
-
real spec before trusting it.
|
|
192
|
+
587 endpoints. Fixed, regression-tested, and the reason the README says to run
|
|
193
|
+
this against a real spec before trusting it.
|
|
161
194
|
|
|
162
195
|
**Stripe is widely and correctly regarded as agent-ready, yet its raw OpenAPI
|
|
163
|
-
spec scores
|
|
196
|
+
spec scores 62.9.** That isn't a contradiction — it's the whole point. Stripe's
|
|
164
197
|
agent-readiness comes from the *Agent Toolkit* layered on top: curated tool
|
|
165
198
|
definitions, an MCP server, managed OAuth. The raw spec is the input to that
|
|
166
199
|
layer, not the layer itself. Which is exactly why this tool ships with
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "openapi-agent-ready"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.4.0"
|
|
8
8
|
description = "Audit any OpenAPI spec for AI-agent and MCP readiness, and generate an MCP server scaffold."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -124,8 +124,14 @@ def extract_endpoints(spec: dict) -> list[EndpointInfo]:
|
|
|
124
124
|
return endpoints
|
|
125
125
|
|
|
126
126
|
|
|
127
|
-
def run_audit(spec: dict) -> dict:
|
|
128
|
-
"""
|
|
127
|
+
def run_audit(spec: dict, progress=None) -> dict:
|
|
128
|
+
"""
|
|
129
|
+
Run every rubric check and aggregate into a scored result.
|
|
130
|
+
|
|
131
|
+
`progress` is an optional callable receiving (completed, total) for the
|
|
132
|
+
ambiguity check, which dominates runtime on large specs and would otherwise
|
|
133
|
+
look like the tool had hung.
|
|
134
|
+
"""
|
|
129
135
|
endpoints = extract_endpoints(spec)
|
|
130
136
|
security_schemes = (spec.get("components") or {}).get("securitySchemes") or {}
|
|
131
137
|
|
|
@@ -138,7 +144,7 @@ def run_audit(spec: dict) -> dict:
|
|
|
138
144
|
findings += check_parameter_explanation(ep)
|
|
139
145
|
findings += check_usage_guidelines(ep)
|
|
140
146
|
findings += check_schema_strictness(ep)
|
|
141
|
-
findings += check_ambiguity(endpoints)
|
|
147
|
+
findings += check_ambiguity(endpoints, progress=progress)
|
|
142
148
|
findings += check_tool_surface(endpoints)
|
|
143
149
|
|
|
144
150
|
# Score per category. Spec-level findings (e.g. "this catalogue has 587
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Baseline support: fail CI only on *new* findings.
|
|
3
|
+
|
|
4
|
+
The problem this solves: a mature API audited for the first time produces
|
|
5
|
+
hundreds of findings. A team cannot fix them all before merging anything, so
|
|
6
|
+
`--min-score` and `--max-fails` are unusable as gates and get switched off
|
|
7
|
+
within a week. A baseline records the current state and lets the gate fail only
|
|
8
|
+
when something gets worse.
|
|
9
|
+
|
|
10
|
+
The whole design rests on one question: what makes a finding "the same finding"
|
|
11
|
+
across two runs? Message text is the obvious answer and the wrong one — it
|
|
12
|
+
embeds counts and parameter names ("3/3 parameters have no description
|
|
13
|
+
(date_from, date_to, location)") that change when an unrelated parameter is
|
|
14
|
+
added, which would report a fixed finding as new.
|
|
15
|
+
|
|
16
|
+
So findings are fingerprinted on (category, endpoint, severity) instead:
|
|
17
|
+
|
|
18
|
+
- **category** — which rule fired.
|
|
19
|
+
- **endpoint** — where. Stable across edits to the endpoint's content.
|
|
20
|
+
- **severity** — a warning becoming a fail is a regression worth catching, so
|
|
21
|
+
it belongs in the identity rather than being treated as the same finding.
|
|
22
|
+
|
|
23
|
+
Message is deliberately excluded. Rewording a finding's text in a future
|
|
24
|
+
release must not invalidate every stored baseline.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
import json
|
|
28
|
+
from collections import Counter
|
|
29
|
+
from dataclasses import asdict
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
BASELINE_VERSION = 1
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class BaselineError(Exception):
|
|
36
|
+
"""Raised when a baseline file can't be read or is malformed."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def fingerprint(finding) -> str:
|
|
40
|
+
"""A stable identity for a finding across runs. See module docstring."""
|
|
41
|
+
return f"{finding.category}|{finding.endpoint}|{finding.severity}"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _counted(findings) -> Counter:
|
|
45
|
+
"""
|
|
46
|
+
Fingerprints with multiplicity.
|
|
47
|
+
|
|
48
|
+
A single endpoint can legitimately produce two findings in one category —
|
|
49
|
+
for example a parameter-explanation fail for undocumented parameters plus a
|
|
50
|
+
warning for a missing date format. Counting rather than set-membership means
|
|
51
|
+
going from one such finding to two is correctly reported as new.
|
|
52
|
+
"""
|
|
53
|
+
return Counter(fingerprint(f) for f in findings)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def write_baseline(path: str, spec: dict, result: dict) -> dict:
|
|
57
|
+
"""Record the current findings so future runs can compare against them."""
|
|
58
|
+
info = spec.get("info") or {}
|
|
59
|
+
payload = {
|
|
60
|
+
"baseline_version": BASELINE_VERSION,
|
|
61
|
+
"api": {
|
|
62
|
+
"title": info.get("title", "Untitled API"),
|
|
63
|
+
"version": str(info.get("version", "unknown")),
|
|
64
|
+
},
|
|
65
|
+
"overall_score": result["overall_score"],
|
|
66
|
+
"endpoint_count": result["endpoint_count"],
|
|
67
|
+
"fingerprints": sorted(_counted(result["gaps"]).elements()),
|
|
68
|
+
# Stored for human readability when reviewing a baseline diff; never
|
|
69
|
+
# read back during comparison.
|
|
70
|
+
"findings": [asdict(f) for f in result["gaps"]],
|
|
71
|
+
}
|
|
72
|
+
Path(path).write_text(json.dumps(payload, indent=2), encoding="utf-8")
|
|
73
|
+
return payload
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def load_baseline(path: str) -> dict:
|
|
77
|
+
p = Path(path)
|
|
78
|
+
if not p.exists():
|
|
79
|
+
raise BaselineError(
|
|
80
|
+
f"Baseline file not found: {path}\n"
|
|
81
|
+
f"Create one from the current state with: --write-baseline {path}"
|
|
82
|
+
)
|
|
83
|
+
try:
|
|
84
|
+
data = json.loads(p.read_text(encoding="utf-8"))
|
|
85
|
+
except json.JSONDecodeError as exc:
|
|
86
|
+
raise BaselineError(f"Baseline file is not valid JSON: {exc}") from exc
|
|
87
|
+
|
|
88
|
+
if not isinstance(data, dict) or "fingerprints" not in data:
|
|
89
|
+
raise BaselineError(
|
|
90
|
+
f"{path} does not look like an agent-ready baseline "
|
|
91
|
+
"(no 'fingerprints' key)."
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
version = data.get("baseline_version", 0)
|
|
95
|
+
if version > BASELINE_VERSION:
|
|
96
|
+
raise BaselineError(
|
|
97
|
+
f"Baseline was written by a newer version of agent-ready "
|
|
98
|
+
f"(format v{version}, this build understands v{BASELINE_VERSION}). "
|
|
99
|
+
"Upgrade agent-ready or regenerate the baseline."
|
|
100
|
+
)
|
|
101
|
+
return data
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def compare_to_baseline(result: dict, baseline: dict) -> dict:
|
|
105
|
+
"""
|
|
106
|
+
Classify current findings against a baseline.
|
|
107
|
+
|
|
108
|
+
Returns counts plus the actual new findings, so the CLI can show a team
|
|
109
|
+
exactly what their change introduced rather than the whole backlog.
|
|
110
|
+
"""
|
|
111
|
+
baseline_counts = Counter(baseline.get("fingerprints", []))
|
|
112
|
+
current_counts = _counted(result["gaps"])
|
|
113
|
+
|
|
114
|
+
# A fingerprint appearing more often than the baseline recorded is new,
|
|
115
|
+
# by the excess amount.
|
|
116
|
+
new_counts = current_counts - baseline_counts
|
|
117
|
+
fixed_counts = baseline_counts - current_counts
|
|
118
|
+
|
|
119
|
+
# Map fingerprints back to findings so the report can name them. Consume
|
|
120
|
+
# the allowance so duplicates beyond it are still reported.
|
|
121
|
+
remaining = Counter(new_counts)
|
|
122
|
+
new_findings = []
|
|
123
|
+
for f in result["gaps"]:
|
|
124
|
+
fp = fingerprint(f)
|
|
125
|
+
if remaining.get(fp, 0) > 0:
|
|
126
|
+
new_findings.append(f)
|
|
127
|
+
remaining[fp] -= 1
|
|
128
|
+
|
|
129
|
+
return {
|
|
130
|
+
"new_findings": new_findings,
|
|
131
|
+
"new_count": sum(new_counts.values()),
|
|
132
|
+
"fixed_count": sum(fixed_counts.values()),
|
|
133
|
+
"unchanged_count": sum((current_counts & baseline_counts).values()),
|
|
134
|
+
"new_fails": sum(1 for f in new_findings if f.severity == "fail"),
|
|
135
|
+
"baseline_score": baseline.get("overall_score"),
|
|
136
|
+
"current_score": result["overall_score"],
|
|
137
|
+
}
|