lap-score 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lap/README.md +120 -0
- lap/__init__.py +8 -0
- lap/cli.py +37 -0
- lap/estimate.py +115 -0
- lap/examples/bookstore.openapi.json +79 -0
- lap/examples/gnarly.openapi.json +68 -0
- lap/examples/swagger2.json +65 -0
- lap/lint.py +149 -0
- lap/mcp_client.py +65 -0
- lap/mcp_form.py +45 -0
- lap/menu.py +108 -0
- lap/openapi_ir.py +331 -0
- lap/score.py +181 -0
- lap/tokens.py +80 -0
- lap_score-0.3.0.dist-info/METADATA +153 -0
- lap_score-0.3.0.dist-info/RECORD +20 -0
- lap_score-0.3.0.dist-info/WHEEL +5 -0
- lap_score-0.3.0.dist-info/entry_points.txt +2 -0
- lap_score-0.3.0.dist-info/licenses/LICENSE +21 -0
- lap_score-0.3.0.dist-info/top_level.txt +1 -0
lap/README.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# lap — token-efficiency scorer for agent-facing APIs
|
|
2
|
+
|
|
3
|
+
`lap` is the open, neutral, standalone toolkit (no pet-zoo dependency) that measures
|
|
4
|
+
how many **tokens** an API's definitions cost an LLM. It answers: *is my agent-API
|
|
5
|
+
menu efficient, and by how much could it shrink?* — an open, reproducible number to set
|
|
6
|
+
beside the fast-growing MCP/OpenAPI tooling (see [`../docs/LANDSCAPE.md`](../docs/LANDSCAPE.md)
|
|
7
|
+
for the neighbors LAP builds on and credits).
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
pip install -e . # from the repo root (or: pip install lap-score once published)
|
|
13
|
+
pip install -e ".[mcp]" # + real-MCP baseline (fastmcp)
|
|
14
|
+
pip install -e ".[faithful]" # + faithful Anthropic count_tokens
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
Core deps are just `httpx` + `tiktoken` + `pyyaml`; `fastmcp` and `anthropic` are optional extras.
|
|
18
|
+
Robust to real specs: `allOf`/`oneOf`/`anyOf`, `$ref` in params/requestBodies/responses,
|
|
19
|
+
path-item-level parameters, OpenAPI 3.1 `type` lists, external `$ref`s (left intact), YAML input,
|
|
20
|
+
**Swagger/OpenAPI 2.0** (response `schema`, `in: body` params, type-on-parameter, `#/definitions`),
|
|
21
|
+
and non-JSON media types (`*+json`, form, XML). Verified crash-free + non-degenerate across 175+
|
|
22
|
+
real APIs.guru specs — re-run with [`../experiments/fuzz_corpus.py`](../experiments/fuzz_corpus.py).
|
|
23
|
+
|
|
24
|
+
## Quickstart
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
lap score https://petstore3.swagger.io/api/v3/openapi.json # menu (bucket A) token cost
|
|
28
|
+
lap lint https://petstore3.swagger.io/api/v3/openapi.json # flag LAP rule violations
|
|
29
|
+
lap score --mcp-url http://localhost:8080/mcp # score a live MCP server's tools
|
|
30
|
+
lap score lap/examples/bookstore.openapi.json
|
|
31
|
+
|
|
32
|
+
# no install needed, from the repo root:
|
|
33
|
+
python -m lap.score lap/examples/bookstore.openapi.json
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Example output:
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
LAP menu score - Bookstore API
|
|
40
|
+
operations: 6 referenced component schemas: 2
|
|
41
|
+
|
|
42
|
+
variant A tokens saved vs full form
|
|
43
|
+
openapi_full 418 +0% 6 tool(s)
|
|
44
|
+
compact_sig 205 +51% manifest text
|
|
45
|
+
numbered 168 +60% manifest text
|
|
46
|
+
|
|
47
|
+
Menu efficiency: compact signatures are +51% vs naive OpenAPI->tools (418 -> 205 tokens).
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
When `fastmcp` is installed, the score also includes a **real-MCP baseline**
|
|
51
|
+
(`FastMCP.from_openapi`) — what an actual MCP generator emits — plus its
|
|
52
|
+
output-schema-inclusive figure (`--no-mcp` to skip). On a real public API
|
|
53
|
+
(**Swagger Petstore**, 19 ops) the real MCP server costs **2226** menu tokens
|
|
54
|
+
(**3844** with output schemas) vs **415** for compact signatures — an ~81%
|
|
55
|
+
reduction. The toy finding holds in the wild: a real MCP generator is *heavier*
|
|
56
|
+
than the naive baseline.
|
|
57
|
+
|
|
58
|
+
The score also includes a lazy **`tool_search`** form (the Anthropic Tool Search /
|
|
59
|
+
Cloudflare Code Mode pattern: a fixed 2-tool menu + a name index, schemas loaded on
|
|
60
|
+
demand). Because it doesn't preload schemas, its bucket A is ~flat in the number of
|
|
61
|
+
operations — on a 120-operation API it collapses the menu ~83% vs full schemas,
|
|
62
|
+
beating even compact signatures at scale (Petstore: 1740 → 207, −88%).
|
|
63
|
+
|
|
64
|
+
- **Faithful counts:** set `ANTHROPIC_API_KEY` (uses the free Anthropic `count_tokens`
|
|
65
|
+
endpoint; tool defs counted via the real `tools=` parameter). Without it, a GPT-style
|
|
66
|
+
`tiktoken` approximation — absolute numbers approximate, **relative ordering robust**.
|
|
67
|
+
|
|
68
|
+
## CI gate
|
|
69
|
+
|
|
70
|
+
`--json` makes both commands machine-readable; thresholds set the exit code so LAP can fail a build:
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
lap score openapi.json --gate-form compact_sig --max-menu-tokens 800 # exit 1 if the menu is too heavy
|
|
74
|
+
lap lint openapi.json --fail-on warn # exit 1 on any warning
|
|
75
|
+
lap lint openapi.json --ignore R2,A1 # suppress rules (or a ./.lapignore file)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
GitHub Actions:
|
|
79
|
+
|
|
80
|
+
```yaml
|
|
81
|
+
- run: pip install lap-score # or: pip install -e .
|
|
82
|
+
- run: lap score api/openapi.json --gate-form compact_sig --max-menu-tokens 800
|
|
83
|
+
- run: lap lint api/openapi.json --fail-on warn
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
…or the bundled composite **Action** (one step, no manual install):
|
|
87
|
+
|
|
88
|
+
```yaml
|
|
89
|
+
- uses: lCrazyblindl/lap@v0.3.0
|
|
90
|
+
with:
|
|
91
|
+
spec: api/openapi.json
|
|
92
|
+
max-menu-tokens: "800" # gate the compact_sig menu (omit = report only)
|
|
93
|
+
fail-on: warn # fail on any lint warning (omit = report only)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
Already lint OpenAPI with **Spectral**? The same LAP rules ship as a ruleset —
|
|
97
|
+
see [`../spectral/`](../spectral/README.md).
|
|
98
|
+
|
|
99
|
+
## What it measures (and what it doesn't)
|
|
100
|
+
|
|
101
|
+
It measures **bucket A** (the definitions/menu the model carries in context) and **estimates
|
|
102
|
+
C** (result size, from each response schema + an assumed `--page-size` — a structural lower
|
|
103
|
+
bound that captures keys/nesting/types). **B** (the call) still depends on per-API tasks; for a
|
|
104
|
+
full measured A/B/C run see
|
|
105
|
+
[`../experiments/token-bench`](../experiments/token-bench/README.md). The conventions
|
|
106
|
+
behind the compact form are the [LAP profile](../profile/llm-api-profile.md).
|
|
107
|
+
|
|
108
|
+
## Files
|
|
109
|
+
|
|
110
|
+
| file | role |
|
|
111
|
+
| --- | --- |
|
|
112
|
+
| `openapi_ir.py` | load any OpenAPI (file/URL) → normalized operations + `inline_refs` |
|
|
113
|
+
| `menu.py` | render the menu forms (openapi_full / compact_sig / numbered) from the IR |
|
|
114
|
+
| `mcp_form.py` | real-MCP baseline via `FastMCP.from_openapi` (optional; `--no-mcp` to skip) |
|
|
115
|
+
| `mcp_client.py` | scores a live MCP server's advertised tools (`lap score --mcp-url`) |
|
|
116
|
+
| `estimate.py` | estimates bucket C (result size) from response schemas (`--page-size`) |
|
|
117
|
+
| `tokens.py` | token counting (Anthropic endpoint, or tiktoken approx) |
|
|
118
|
+
| `score.py` | the `lap score` CLI |
|
|
119
|
+
| `lint.py` | the `lap lint` CLI — checks a spec against the LAP profile rules (D3/R1/R2/R3/W1/E1/A1) |
|
|
120
|
+
| `examples/` | sample specs: a Bookstore API, a gnarly OpenAPI 3.1 (allOf / $ref-params / nullable / external-ref), and a Swagger 2.0 spec (`swagger2.json`) |
|
lap/__init__.py
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
"""LAP - an open, neutral token-efficiency toolkit for agent-facing APIs.
|
|
2
|
+
|
|
3
|
+
`lap score <openapi>` measures the menu (bucket A) token cost of any OpenAPI spec
|
|
4
|
+
under several interface forms. Reuses the parsing proven in the token-bench, but
|
|
5
|
+
standalone (no pet-zoo dependency). See ../ROADMAP.md for the staged plan.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
__version__ = "0.1.0"
|
lap/cli.py
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"""Unified `lap` command: `lap score <openapi>` / `lap lint <openapi>`.
|
|
2
|
+
|
|
3
|
+
Thin dispatcher so the toolkit is one console command after `pip install`. Each
|
|
4
|
+
subcommand reuses its module's own argument parsing.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
_USAGE = "usage: lap {score|lint} <openapi-file-or-url> [options]\n" \
|
|
12
|
+
" score measure the menu (bucket A) token cost\n" \
|
|
13
|
+
" lint flag LAP profile rule violations"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def main() -> None:
|
|
17
|
+
argv = sys.argv[1:]
|
|
18
|
+
if not argv or argv[0] in ("-h", "--help"):
|
|
19
|
+
print(_USAGE)
|
|
20
|
+
return
|
|
21
|
+
cmd, rest = argv[0], argv[1:]
|
|
22
|
+
sys.argv = [f"lap {cmd}", *rest] # so each subcommand's argparse sees clean args
|
|
23
|
+
if cmd == "score":
|
|
24
|
+
from . import score
|
|
25
|
+
|
|
26
|
+
score.main()
|
|
27
|
+
elif cmd == "lint":
|
|
28
|
+
from . import lint
|
|
29
|
+
|
|
30
|
+
lint.main()
|
|
31
|
+
else:
|
|
32
|
+
print(f"lap: unknown command {cmd!r}\n{_USAGE}")
|
|
33
|
+
sys.exit(2)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if __name__ == "__main__":
|
|
37
|
+
main()
|
lap/estimate.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Estimate bucket C (result tokens) from response schemas - no runtime needed.
|
|
2
|
+
|
|
3
|
+
For each operation we synthesize a representative instance from its success
|
|
4
|
+
response schema, serialize it compactly, and count tokens. For array (collection)
|
|
5
|
+
responses we multiply the per-item cost by an assumed page size. Values are a
|
|
6
|
+
*structural lower bound* - they capture keys + nesting + types (which dominate
|
|
7
|
+
repeated-JSON cost), not real string lengths, so a live payload is >= this.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
|
|
14
|
+
from . import openapi_ir as ir
|
|
15
|
+
from . import tokens
|
|
16
|
+
|
|
17
|
+
_PLACEHOLDER = {"string": "string", "integer": 0, "number": 0, "boolean": True, "null": None}
|
|
18
|
+
|
|
19
|
+
# Common envelope keys real APIs wrap a list in, preferred in this order when
|
|
20
|
+
# more than one array-typed property is present (rare, but pick deterministically).
|
|
21
|
+
_ENVELOPE_KEYS = ("data", "items", "results", "value", "values", "content", "entries", "records")
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def example_instance(spec: dict, schema, stack: frozenset = frozenset(), depth: int = 0):
|
|
25
|
+
if not isinstance(schema, dict) or depth > 6:
|
|
26
|
+
return None
|
|
27
|
+
if "$ref" in schema:
|
|
28
|
+
ref = schema["$ref"]
|
|
29
|
+
if not ir._local(ref) or ref in stack:
|
|
30
|
+
return "ref"
|
|
31
|
+
return example_instance(spec, ir._resolve_ref(spec, ref), stack | {ref}, depth + 1)
|
|
32
|
+
if "allOf" in schema:
|
|
33
|
+
return {
|
|
34
|
+
fname: example_instance(spec, prop, stack, depth + 1)
|
|
35
|
+
for fname, prop in ir._collect_properties(spec, schema).items()
|
|
36
|
+
}
|
|
37
|
+
for comb in ("oneOf", "anyOf"):
|
|
38
|
+
if schema.get(comb):
|
|
39
|
+
return example_instance(spec, schema[comb][0], stack, depth + 1)
|
|
40
|
+
if "enum" in schema:
|
|
41
|
+
return schema["enum"][0]
|
|
42
|
+
t = schema.get("type")
|
|
43
|
+
if isinstance(t, list):
|
|
44
|
+
t = next((x for x in t if x != "null"), "string")
|
|
45
|
+
if t == "object" or "properties" in schema:
|
|
46
|
+
return {
|
|
47
|
+
k: example_instance(spec, v, stack, depth + 1)
|
|
48
|
+
for k, v in schema.get("properties", {}).items()
|
|
49
|
+
}
|
|
50
|
+
if t == "array":
|
|
51
|
+
return [example_instance(spec, schema.get("items", {}), stack, depth + 1)]
|
|
52
|
+
return _PLACEHOLDER.get(t, "x")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _success_schema(spec: dict, op: ir.Op):
|
|
56
|
+
# OpenAPI 3 (`content`) and 2.0 (`schema`), JSON-ish media types — see ir._response_schema.
|
|
57
|
+
return ir._response_schema(spec, op.raw)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _dumps(value) -> str:
|
|
61
|
+
return json.dumps(value, separators=(",", ":"), default=str, ensure_ascii=False)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _is_array_schema(spec: dict, schema) -> bool:
|
|
65
|
+
if not isinstance(schema, dict):
|
|
66
|
+
return False
|
|
67
|
+
deref = ir._deref(spec, schema)
|
|
68
|
+
return deref.get("type") == "array" or "items" in deref
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _find_envelope_key(spec: dict, schema: dict) -> str | None:
|
|
72
|
+
"""If `schema` is an object with an array-typed property - a common
|
|
73
|
+
`{"data": [...]}` / k8s `{"items": [...]}` / OData `{"value": [...]}` envelope -
|
|
74
|
+
return that property's name. Real APIs almost always wrap collections this
|
|
75
|
+
way; without this, an enveloped list scores as a tiny "object" (one item deep
|
|
76
|
+
in the envelope) and bucket C is badly undercounted."""
|
|
77
|
+
props = ir._collect_properties(spec, schema)
|
|
78
|
+
candidates = [fname for fname, prop in props.items() if _is_array_schema(spec, prop)]
|
|
79
|
+
if not candidates:
|
|
80
|
+
return None
|
|
81
|
+
for key in _ENVELOPE_KEYS:
|
|
82
|
+
if key in candidates:
|
|
83
|
+
return key
|
|
84
|
+
return candidates[0]
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def estimate(spec: dict, op: ir.Op, page_size: int = 20) -> tuple[str, int, int]:
|
|
88
|
+
"""Returns (kind, per_unit_tokens, estimated_C_tokens) for an operation's
|
|
89
|
+
success response. kind in {"void", "object", "list"}.
|
|
90
|
+
|
|
91
|
+
A bare top-level array is scaled by `page_size` directly. A list wrapped in
|
|
92
|
+
an envelope object (`{"data": [...], "total_count": ...}`, k8s `{"items": [...],
|
|
93
|
+
"kind": ..., "metadata": ...}`) is detected too: the sibling fields are kept
|
|
94
|
+
(counted once) and the array property is scaled to `page_size` items, so the
|
|
95
|
+
*whole* envelope at a page is what's estimated - not just one wrapped item."""
|
|
96
|
+
schema = _success_schema(spec, op)
|
|
97
|
+
if not isinstance(schema, dict):
|
|
98
|
+
return ("void", 0, 0)
|
|
99
|
+
deref = ir._deref(spec, schema)
|
|
100
|
+
if schema.get("type") == "array" or "items" in deref:
|
|
101
|
+
per = tokens.count(_dumps(example_instance(spec, deref.get("items", {}))))
|
|
102
|
+
return ("list", per, per * page_size + 5) # +5 for the array brackets/commas
|
|
103
|
+
|
|
104
|
+
envelope_key = _find_envelope_key(spec, schema)
|
|
105
|
+
if envelope_key:
|
|
106
|
+
instance = example_instance(spec, schema)
|
|
107
|
+
arr = instance.get(envelope_key) if isinstance(instance, dict) else None
|
|
108
|
+
if isinstance(arr, list) and arr:
|
|
109
|
+
item = arr[0]
|
|
110
|
+
per = tokens.count(_dumps(item))
|
|
111
|
+
instance[envelope_key] = [item] * page_size
|
|
112
|
+
return ("list", per, tokens.count(_dumps(instance)))
|
|
113
|
+
|
|
114
|
+
per = tokens.count(_dumps(example_instance(spec, schema)))
|
|
115
|
+
return ("object", per, per)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
{
|
|
2
|
+
"openapi": "3.0.3",
|
|
3
|
+
"info": { "title": "Bookstore API", "version": "1.0.0" },
|
|
4
|
+
"paths": {
|
|
5
|
+
"/books": {
|
|
6
|
+
"get": {
|
|
7
|
+
"operationId": "list_books",
|
|
8
|
+
"summary": "List all books",
|
|
9
|
+
"responses": {
|
|
10
|
+
"200": {
|
|
11
|
+
"description": "ok",
|
|
12
|
+
"content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Book" } } } }
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"post": {
|
|
17
|
+
"operationId": "create_book",
|
|
18
|
+
"summary": "Add a book",
|
|
19
|
+
"requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BookCreate" } } } },
|
|
20
|
+
"responses": { "201": { "description": "created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Book" } } } } }
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"/books/{book_id}": {
|
|
24
|
+
"get": {
|
|
25
|
+
"operationId": "get_book",
|
|
26
|
+
"summary": "Get a book by id",
|
|
27
|
+
"parameters": [ { "name": "book_id", "in": "path", "required": true, "schema": { "type": "integer" } } ],
|
|
28
|
+
"responses": { "200": { "description": "ok", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Book" } } } } }
|
|
29
|
+
},
|
|
30
|
+
"put": {
|
|
31
|
+
"operationId": "update_book",
|
|
32
|
+
"summary": "Replace a book",
|
|
33
|
+
"parameters": [ { "name": "book_id", "in": "path", "required": true, "schema": { "type": "integer" } } ],
|
|
34
|
+
"requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/BookCreate" } } } },
|
|
35
|
+
"responses": { "200": { "description": "ok", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Book" } } } } }
|
|
36
|
+
},
|
|
37
|
+
"delete": {
|
|
38
|
+
"operationId": "delete_book",
|
|
39
|
+
"summary": "Delete a book",
|
|
40
|
+
"parameters": [ { "name": "book_id", "in": "path", "required": true, "schema": { "type": "integer" } } ],
|
|
41
|
+
"responses": { "204": { "description": "deleted" } }
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
"/authors/{author_id}/books": {
|
|
45
|
+
"get": {
|
|
46
|
+
"operationId": "list_author_books",
|
|
47
|
+
"summary": "List books by an author",
|
|
48
|
+
"parameters": [ { "name": "author_id", "in": "path", "required": true, "schema": { "type": "integer" } } ],
|
|
49
|
+
"responses": { "200": { "description": "ok", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Book" } } } } } }
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"components": {
|
|
54
|
+
"schemas": {
|
|
55
|
+
"Genre": { "type": "string", "enum": ["fiction", "nonfiction", "poetry", "reference"] },
|
|
56
|
+
"BookCreate": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"required": ["title", "author", "price", "genre"],
|
|
59
|
+
"properties": {
|
|
60
|
+
"title": { "type": "string", "minLength": 1 },
|
|
61
|
+
"author": { "type": "string" },
|
|
62
|
+
"price": { "type": "number", "minimum": 0 },
|
|
63
|
+
"genre": { "$ref": "#/components/schemas/Genre" }
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
"Book": {
|
|
67
|
+
"type": "object",
|
|
68
|
+
"required": ["id", "title", "author", "price", "genre"],
|
|
69
|
+
"properties": {
|
|
70
|
+
"id": { "type": "integer" },
|
|
71
|
+
"title": { "type": "string", "minLength": 1 },
|
|
72
|
+
"author": { "type": "string" },
|
|
73
|
+
"price": { "type": "number", "minimum": 0 },
|
|
74
|
+
"genre": { "$ref": "#/components/schemas/Genre" }
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"openapi": "3.1.0",
|
|
3
|
+
"info": { "title": "Gnarly API", "version": "1.0.0" },
|
|
4
|
+
"paths": {
|
|
5
|
+
"/pets": {
|
|
6
|
+
"parameters": [
|
|
7
|
+
{ "name": "limit", "in": "query", "schema": { "type": "integer" } }
|
|
8
|
+
],
|
|
9
|
+
"get": {
|
|
10
|
+
"operationId": "listPets",
|
|
11
|
+
"responses": {
|
|
12
|
+
"200": { "description": "ok", "content": { "application/json": { "schema": { "type": "array", "items": { "$ref": "#/components/schemas/Pet" } } } } }
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"post": {
|
|
16
|
+
"operationId": "createPet",
|
|
17
|
+
"requestBody": { "$ref": "#/components/requestBodies/PetBody" },
|
|
18
|
+
"responses": { "201": { "description": "created", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pet" } } } } }
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"/pets/{petId}": {
|
|
22
|
+
"get": {
|
|
23
|
+
"operationId": "getPet",
|
|
24
|
+
"parameters": [ { "$ref": "#/components/parameters/PetId" } ],
|
|
25
|
+
"responses": { "200": { "$ref": "#/components/responses/PetResp" } }
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
"components": {
|
|
30
|
+
"parameters": {
|
|
31
|
+
"PetId": { "name": "petId", "in": "path", "required": true, "schema": { "type": "integer" } }
|
|
32
|
+
},
|
|
33
|
+
"requestBodies": {
|
|
34
|
+
"PetBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/PetCreate" } } } }
|
|
35
|
+
},
|
|
36
|
+
"responses": {
|
|
37
|
+
"PetResp": { "description": "a pet", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/Pet" } } } }
|
|
38
|
+
},
|
|
39
|
+
"schemas": {
|
|
40
|
+
"Status": { "type": "string", "enum": ["available", "sold"] },
|
|
41
|
+
"Base": {
|
|
42
|
+
"type": "object",
|
|
43
|
+
"properties": { "id": { "type": ["integer", "null"] } }
|
|
44
|
+
},
|
|
45
|
+
"Pet": {
|
|
46
|
+
"allOf": [
|
|
47
|
+
{ "$ref": "#/components/schemas/Base" },
|
|
48
|
+
{
|
|
49
|
+
"type": "object",
|
|
50
|
+
"required": ["name"],
|
|
51
|
+
"properties": {
|
|
52
|
+
"name": { "type": "string" },
|
|
53
|
+
"status": { "$ref": "#/components/schemas/Status" }
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
]
|
|
57
|
+
},
|
|
58
|
+
"PetCreate": {
|
|
59
|
+
"type": "object",
|
|
60
|
+
"required": ["name"],
|
|
61
|
+
"properties": {
|
|
62
|
+
"name": { "type": "string", "minLength": 1 },
|
|
63
|
+
"owner": { "$ref": "https://example.com/external.json#/Owner" }
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
{
|
|
2
|
+
"swagger": "2.0",
|
|
3
|
+
"info": { "title": "Swagger2 Pets", "version": "1.0.0" },
|
|
4
|
+
"basePath": "/v1",
|
|
5
|
+
"consumes": ["application/json"],
|
|
6
|
+
"produces": ["application/json"],
|
|
7
|
+
"paths": {
|
|
8
|
+
"/pets": {
|
|
9
|
+
"get": {
|
|
10
|
+
"operationId": "listPets",
|
|
11
|
+
"summary": "List pets",
|
|
12
|
+
"parameters": [
|
|
13
|
+
{ "name": "breed", "in": "query", "type": "string" }
|
|
14
|
+
],
|
|
15
|
+
"responses": {
|
|
16
|
+
"200": {
|
|
17
|
+
"description": "a list of pets",
|
|
18
|
+
"schema": { "type": "array", "items": { "$ref": "#/definitions/Pet" } }
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"post": {
|
|
23
|
+
"operationId": "createPet",
|
|
24
|
+
"summary": "Create a pet",
|
|
25
|
+
"parameters": [
|
|
26
|
+
{ "name": "body", "in": "body", "required": true, "schema": { "$ref": "#/definitions/PetCreate" } }
|
|
27
|
+
],
|
|
28
|
+
"responses": {
|
|
29
|
+
"201": { "description": "created", "schema": { "$ref": "#/definitions/Pet" } }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"/pets/{petId}": {
|
|
34
|
+
"get": {
|
|
35
|
+
"operationId": "getPet",
|
|
36
|
+
"summary": "Get a pet",
|
|
37
|
+
"parameters": [
|
|
38
|
+
{ "name": "petId", "in": "path", "required": true, "type": "integer" }
|
|
39
|
+
],
|
|
40
|
+
"responses": {
|
|
41
|
+
"200": { "description": "ok", "schema": { "$ref": "#/definitions/Pet" } },
|
|
42
|
+
"404": { "description": "not found" }
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
"definitions": {
|
|
48
|
+
"Pet": {
|
|
49
|
+
"type": "object",
|
|
50
|
+
"properties": {
|
|
51
|
+
"id": { "type": "integer" },
|
|
52
|
+
"name": { "type": "string" },
|
|
53
|
+
"status": { "type": "string", "enum": ["available", "sold"] }
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
"PetCreate": {
|
|
57
|
+
"type": "object",
|
|
58
|
+
"required": ["name"],
|
|
59
|
+
"properties": {
|
|
60
|
+
"name": { "type": "string" },
|
|
61
|
+
"status": { "type": "string" }
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
}
|
lap/lint.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""`lap lint <openapi>` - check an API against the LAP profile rules.
|
|
2
|
+
|
|
3
|
+
Static, advisory checks over an OpenAPI spec for the LAP conventions that are
|
|
4
|
+
detectable without runtime: opaque names (D3), read shaping on collections
|
|
5
|
+
(projection R1 / filter R2 / pagination R3), aggregation (A1), minimal writes
|
|
6
|
+
(W1), and uniform errors (E1). Heuristic by nature - it flags likely token/clarity
|
|
7
|
+
costs for a human to confirm, each tied to a profile rule.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import re
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from . import openapi_ir as ir
|
|
18
|
+
|
|
19
|
+
PAGINATION = {"limit", "offset", "page", "page_size", "pagesize", "per_page", "perpage",
|
|
20
|
+
"cursor", "top", "skip", "$top", "$skip"}
|
|
21
|
+
PROJECTION = {"fields", "field", "select", "$select", "include", "expand", "$expand"}
|
|
22
|
+
FILTER_HINTS = {"filter", "$filter", "q", "query", "where", "search"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@dataclass
|
|
26
|
+
class Finding:
|
|
27
|
+
rule: str
|
|
28
|
+
severity: str # "warn" | "info"
|
|
29
|
+
where: str
|
|
30
|
+
message: str
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _query_names(op: ir.Op) -> set[str]:
|
|
34
|
+
return {p["name"].lower() for p in op.params if p.get("in") == "query"}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _error_codes(op: ir.Op) -> list[str]:
|
|
38
|
+
return [c for c in op.raw.get("responses", {}) if c[:1] in ("4", "5")]
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def lint(spec: dict) -> list[Finding]:
|
|
42
|
+
ops = ir.operations(spec)
|
|
43
|
+
out: list[Finding] = []
|
|
44
|
+
|
|
45
|
+
for op in ops:
|
|
46
|
+
where = f"{op.method} {op.path}"
|
|
47
|
+
|
|
48
|
+
# D3 - opaque operation name
|
|
49
|
+
if re.fullmatch(r"\d+", op.name) or not re.search(r"[A-Za-z]", op.name) or len(op.name) < 3:
|
|
50
|
+
out.append(Finding("D3", "warn", where,
|
|
51
|
+
f"opaque operation name '{op.name}' - LLMs ground on readable names"))
|
|
52
|
+
|
|
53
|
+
# Read shaping on collection (array-returning) GETs
|
|
54
|
+
if op.method == "GET" and op.returns.endswith("[]"):
|
|
55
|
+
q = _query_names(op)
|
|
56
|
+
if not (q & PAGINATION):
|
|
57
|
+
out.append(Finding("R3", "warn", where,
|
|
58
|
+
"collection GET has no pagination (limit/offset/cursor) - agents pull the whole list (big bucket C)"))
|
|
59
|
+
if not (q & PROJECTION):
|
|
60
|
+
out.append(Finding("R1", "info", where,
|
|
61
|
+
"no field projection (fields/select) - responses carry every field"))
|
|
62
|
+
if not (q & (FILTER_HINTS | (q - PAGINATION - PROJECTION))):
|
|
63
|
+
out.append(Finding("R2", "info", where,
|
|
64
|
+
"no server-side filter params - agents fetch then filter in-context"))
|
|
65
|
+
|
|
66
|
+
# W1 - writes returning a full representation by default
|
|
67
|
+
if op.method in ("POST", "PUT", "PATCH") and op.returns not in ("void", ""):
|
|
68
|
+
out.append(Finding("W1", "info", where,
|
|
69
|
+
f"write returns a full representation ({op.returns}) by default - consider Prefer: return=minimal (server-generated fields only)"))
|
|
70
|
+
|
|
71
|
+
# E1 - no error responses declared
|
|
72
|
+
if not _error_codes(op):
|
|
73
|
+
out.append(Finding("E1", "warn", where,
|
|
74
|
+
"no 4xx/5xx error response declared - agents can't distinguish success/empty/error"))
|
|
75
|
+
|
|
76
|
+
# A1 - no aggregate/count endpoint anywhere
|
|
77
|
+
if not any(re.search(r"count|aggregate|stats|summary", op.name + op.path, re.I) for op in ops):
|
|
78
|
+
out.append(Finding("A1", "info", "(global)",
|
|
79
|
+
"no aggregate/count endpoint - 'how many...' questions force pulling the full list"))
|
|
80
|
+
|
|
81
|
+
return out
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def filter_ignored(findings: list[Finding], ignore) -> list[Finding]:
|
|
85
|
+
ignore = {r.upper() for r in ignore}
|
|
86
|
+
return [f for f in findings if f.rule.upper() not in ignore]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _load_ignore_file() -> set[str]:
|
|
90
|
+
if not os.path.exists(".lapignore"):
|
|
91
|
+
return set()
|
|
92
|
+
with open(".lapignore", encoding="utf-8") as fh:
|
|
93
|
+
toks = re.split(r"[,\s]+", fh.read())
|
|
94
|
+
return {t.strip().upper() for t in toks if t.strip() and not t.startswith("#")}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _print_human(title: str, source: str, findings: list[Finding], warns: int, infos: int) -> None:
|
|
98
|
+
print(f"\nLAP lint - {title}\nsource: {source}\n")
|
|
99
|
+
if not findings:
|
|
100
|
+
print(" No LAP rule violations detected. ✓\n")
|
|
101
|
+
return
|
|
102
|
+
order = {"warn": 0, "info": 1}
|
|
103
|
+
by_rule: dict[str, list[Finding]] = {}
|
|
104
|
+
for f in sorted(findings, key=lambda f: (order[f.severity], f.rule)):
|
|
105
|
+
by_rule.setdefault(f"{f.severity.upper()} {f.rule}", []).append(f)
|
|
106
|
+
for header, items in by_rule.items():
|
|
107
|
+
print(f" [{header}] {items[0].message.split(' - ')[0]}")
|
|
108
|
+
for f in items[:6]:
|
|
109
|
+
print(f" {f.where}")
|
|
110
|
+
if len(items) > 6:
|
|
111
|
+
print(f" ... +{len(items) - 6} more")
|
|
112
|
+
print()
|
|
113
|
+
print(f" {warns} warning(s), {infos} suggestion(s). See profile/llm-api-profile.md for the rules.\n")
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def main() -> None:
|
|
117
|
+
import argparse
|
|
118
|
+
|
|
119
|
+
ap = argparse.ArgumentParser(description="Lint an OpenAPI against the LAP profile rules.")
|
|
120
|
+
ap.add_argument("source", help="OpenAPI spec: file path or http(s) URL")
|
|
121
|
+
ap.add_argument("--json", action="store_true", help="emit machine-readable JSON")
|
|
122
|
+
ap.add_argument("--ignore", default="", help="comma-separated rule codes to suppress (also reads ./.lapignore)")
|
|
123
|
+
ap.add_argument("--fail-on", choices=["none", "info", "warn"], default="none",
|
|
124
|
+
help="CI gate: exit 1 if any finding at/above this severity remains")
|
|
125
|
+
args = ap.parse_args()
|
|
126
|
+
|
|
127
|
+
spec = ir.load_spec(args.source)
|
|
128
|
+
ignore = _load_ignore_file() | {r.strip() for r in args.ignore.split(",") if r.strip()}
|
|
129
|
+
findings = filter_ignored(lint(spec), ignore)
|
|
130
|
+
title = spec.get("info", {}).get("title", "(untitled API)")
|
|
131
|
+
warns = sum(1 for f in findings if f.severity == "warn")
|
|
132
|
+
infos = len(findings) - warns
|
|
133
|
+
|
|
134
|
+
if args.json:
|
|
135
|
+
print(json.dumps({
|
|
136
|
+
"api": title, "source": args.source,
|
|
137
|
+
"findings": [{"rule": f.rule, "severity": f.severity, "where": f.where, "message": f.message}
|
|
138
|
+
for f in findings],
|
|
139
|
+
"warnings": warns, "suggestions": infos,
|
|
140
|
+
}, indent=2))
|
|
141
|
+
else:
|
|
142
|
+
_print_human(title, args.source, findings, warns, infos)
|
|
143
|
+
|
|
144
|
+
if (args.fail_on == "warn" and warns) or (args.fail_on == "info" and findings):
|
|
145
|
+
raise SystemExit(1)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
if __name__ == "__main__":
|
|
149
|
+
main()
|