k11-api-functional-qa-mcp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- k11_api_functional_qa_mcp-0.1.0.dist-info/METADATA +201 -0
- k11_api_functional_qa_mcp-0.1.0.dist-info/RECORD +17 -0
- k11_api_functional_qa_mcp-0.1.0.dist-info/WHEEL +4 -0
- k11_api_functional_qa_mcp-0.1.0.dist-info/entry_points.txt +2 -0
- k11_api_functional_qa_mcp-0.1.0.dist-info/licenses/LICENSE +21 -0
- k11_api_qa/__init__.py +3 -0
- k11_api_qa/__main__.py +3 -0
- k11_api_qa/auth.py +54 -0
- k11_api_qa/cli.py +86 -0
- k11_api_qa/config.py +98 -0
- k11_api_qa/db.py +185 -0
- k11_api_qa/export.py +168 -0
- k11_api_qa/generate.py +299 -0
- k11_api_qa/report.py +109 -0
- k11_api_qa/runner.py +472 -0
- k11_api_qa/server.py +210 -0
- k11_api_qa/values.py +234 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: k11-api-functional-qa-mcp
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server and Claude Code plugin for functional API testing: user stories to multi-step API test flows, negative and boundary tests from the spec, database checks and fixtures, with evidence for every step.
|
|
5
|
+
Project-URL: Homepage, https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp
|
|
6
|
+
Project-URL: Issues, https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp/issues
|
|
7
|
+
Author: K11 Software Solutions
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: api-testing,claude,database-testing,functional-testing,mcp,negative-testing,openapi,qa,rest
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Requires-Dist: k11-api-contract-qa-mcp>=0.1.0
|
|
17
|
+
Provides-Extra: dev
|
|
18
|
+
Requires-Dist: pytest-timeout>=2; extra == 'dev'
|
|
19
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
20
|
+
Provides-Extra: mssql
|
|
21
|
+
Requires-Dist: pymssql>=2.3; extra == 'mssql'
|
|
22
|
+
Provides-Extra: mysql
|
|
23
|
+
Requires-Dist: pymysql>=1.1; extra == 'mysql'
|
|
24
|
+
Provides-Extra: postgres
|
|
25
|
+
Requires-Dist: psycopg[binary]>=3.1; extra == 'postgres'
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# k11 API Functional QA — MCP server & Claude Code plugin
|
|
29
|
+
|
|
30
|
+
<!-- mcp-name: io.github.K11-Software-Solutions/api-functional-qa -->
|
|
31
|
+
|
|
32
|
+
**Functional API tests that prove the API did the right thing, down to the database row.**
|
|
33
|
+
|
|
34
|
+
Contract checks tell you an API still has the right *shape*. They don't tell you whether placing an order charges the right amount, whether cancelling really soft-deletes the row, or whether a malformed request crashes the server. This MCP server lets Claude (or any MCP client) write and run those tests with you:
|
|
35
|
+
|
|
36
|
+
- **User story → multi-step test flow.** Chain calls (`POST /orders` → save the `id` → `GET /orders/{id}` → `DELETE`) with readable YAML and value checks.
|
|
37
|
+
- **Database checks in the same flow.** After the API call, check the row really changed: `SELECT status, deleted_at FROM orders WHERE id = :orderId`. Read-only by default, with masked columns.
|
|
38
|
+
- **Negative and boundary tests from the spec,** including nested fields and array items: missing required fields, wrong types, values outside enums and min/max/length limits, no credentials, unknown IDs. Every case must get a 4xx, **never a 5xx**.
|
|
39
|
+
- **Evidence for every step:** the request, response, query, rows and each check, in Markdown (PR comment), JSON and JUnit XML for CI dashboards.
|
|
40
|
+
- **Guardrails:** data-changing calls are refused on production-like environments, database writes only in fixtures and cleanup on allowlisted test databases, secrets only from environment variables and masked everywhere, and requests never leave the environment's host.
|
|
41
|
+
- **No lock-in:** export any flow as plain pytest + httpx code.
|
|
42
|
+
|
|
43
|
+
Built on the parsers of [k11 API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp) (OpenAPI 3.x and Swagger 2.0 for test generation and response-schema checks).
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## See it find real bugs
|
|
48
|
+
|
|
49
|
+
The demo shop API (`demo/shop/`) has two planted bugs. Two hand-written flows and one generated negative suite found them, plus one I didn't plant.
|
|
50
|
+
|
|
51
|
+
**1. The order total is wrong only in the API response.** `order_total.yaml` orders two bags at 42.50:
|
|
52
|
+
|
|
53
|
+
> ❌ **Order total is quantity times price**
|
|
54
|
+
> - **place an order for two bags**: failed. Response: HTTP 201. ✗ `$.total 85.0`: expected 85.0, got 42.5
|
|
55
|
+
> - **the database stores 85.00**: passed
|
|
56
|
+
> - **order history shows 85.00**: passed
|
|
57
|
+
|
|
58
|
+
The database check shows exactly where the bug is: the stored total is right, and only the `POST /orders` response is wrong.
|
|
59
|
+
|
|
60
|
+
**2. Malformed input crashes the server.** `generate_negative_tests("POST /orders")` produced 18 cases from the spec. 16 behaved correctly, and two returned **HTTP 500**:
|
|
61
|
+
|
|
62
|
+
> - ✗ wrong type for `items[0].qty` is rejected: got 500. *A 5xx is a bug.*
|
|
63
|
+
> - ✗ missing required field `items[0].qty` is rejected: got 500. *A 5xx is a bug.*
|
|
64
|
+
|
|
65
|
+
The second crash wasn't planted. It's the same code path, and the generator found it because it tests nested array items, not just top-level fields.
|
|
66
|
+
|
|
67
|
+
**3. The soft delete works, proven in the database.** `cancel_order.yaml` passes all 7 steps, including checks that the row is kept with `status = cancelled` and a `deleted_at` timestamp.
|
|
68
|
+
|
|
69
|
+
## A flow
|
|
70
|
+
|
|
71
|
+
```yaml
|
|
72
|
+
name: Customer cancels an order and it is soft-deleted
|
|
73
|
+
fixtures: [customer] # seeded in the test database, removed afterwards
|
|
74
|
+
steps:
|
|
75
|
+
- name: place an order
|
|
76
|
+
call: POST /orders
|
|
77
|
+
body: { customerId: "${fixture.customer.id}", channel: web, items: [{ sku: HAT-2, qty: 1 }] }
|
|
78
|
+
expect:
|
|
79
|
+
status: 201
|
|
80
|
+
json: { "$.status": pending, "$.total": 15.0, "$.items": { length: 1 } }
|
|
81
|
+
save: { orderId: "$.id" }
|
|
82
|
+
|
|
83
|
+
- name: the row is kept and marked deleted
|
|
84
|
+
db: SELECT status, deleted_at FROM orders WHERE id = :orderId
|
|
85
|
+
params: { orderId: "${orderId}" }
|
|
86
|
+
expect: { rows: 1, row: { status: cancelled, deleted_at: { ne: null } } }
|
|
87
|
+
|
|
88
|
+
cleanup:
|
|
89
|
+
- db: DELETE FROM orders WHERE id = :orderId
|
|
90
|
+
params: { orderId: "${orderId}" }
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
- **Checks** (`expect`): `status` (`201`, `"2xx"`, `[400, 422]`), `json` paths (`$.items[0].qty`, `$.items[*].sku`, `$.items.length`), `headers`, `body_contains`, `max_ms`, and the response schema from the spec (on by default). Values compare as literals or with operators: `eq ne gt gte lt lte approx contains matches length type in exists empty`.
|
|
94
|
+
- **Variables:** `save` from JSON paths, headers (`header:Location`) or `status`; use them as `${orderId}`. Generators: `${uuid}`, `${random_email}`, `${random_int}`, `${now}`, `${today}`. Secrets: `${ENV:NAME}`.
|
|
95
|
+
- **Failures:** a step that only fails value checks lets the flow continue, to gather more evidence. A wrong status, a 5xx, a blocked write or a missing saved value stops it. `continue_on_failure: true` runs every step (generated negative suites use it). Cleanup always runs.
|
|
96
|
+
|
|
97
|
+
## Tools
|
|
98
|
+
|
|
99
|
+
| Tool | What it does |
|
|
100
|
+
|---|---|
|
|
101
|
+
| `list_environments` | Environments and whether each allows writes (and why), auth profiles, databases, fixtures, flows. |
|
|
102
|
+
| `draft_flow` | User story → scenarios, the spec operations most likely involved, and a flow skeleton. |
|
|
103
|
+
| `generate_negative_tests` | Negative and boundary cases for one operation, with real base values from fixtures. |
|
|
104
|
+
| `write_flow` | Validates and saves a flow under `tests/api/`. |
|
|
105
|
+
| `run_flows` | Runs flows (all, by path or by tag) with full evidence and reports. |
|
|
106
|
+
| `call_api` | One request with the same guardrails, to explore an API while writing tests. |
|
|
107
|
+
| `query_db` | One read-only query against a test database, with masking. |
|
|
108
|
+
| `export_pytest` | A flow as plain pytest + httpx code. |
|
|
109
|
+
|
|
110
|
+
## Configure: `.qa-mcp/api-test.yaml`
|
|
111
|
+
|
|
112
|
+
```yaml
|
|
113
|
+
spec: openapi.yaml
|
|
114
|
+
default_environment: local
|
|
115
|
+
environments:
|
|
116
|
+
local: { base_url: http://localhost:8080 }
|
|
117
|
+
staging: { base_url: https://staging.example.com, allow_writes: true }
|
|
118
|
+
prod: { base_url: https://api.example.com } # read-only: writes refused
|
|
119
|
+
auth:
|
|
120
|
+
default: { type: bearer, token: "${ENV:API_TOKEN}" }
|
|
121
|
+
# basic: {username, password} · api_key: {header, value} · oauth2: {token_url, client_id, client_secret, scope}
|
|
122
|
+
databases:
|
|
123
|
+
main: { url: "${ENV:TEST_DB_URL}", allow_writes: true, mask: [email, card_number] }
|
|
124
|
+
fixtures:
|
|
125
|
+
customer:
|
|
126
|
+
db: main
|
|
127
|
+
insert: { table: customers, values: { id: "${uuid}", email: "${random_email}", name: Test Customer } }
|
|
128
|
+
delete: { table: customers, where: { id: "${id}" } }
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
**Databases:** SQLite (built in), PostgreSQL (`pip install "k11-api-functional-qa-mcp[postgres]"`), MySQL/MariaDB (`[mysql]`) and SQL Server (`[mssql]`).
|
|
132
|
+
|
|
133
|
+
## Guardrails
|
|
134
|
+
|
|
135
|
+
| Risk | What happens |
|
|
136
|
+
|---|---|
|
|
137
|
+
| Changing data in production | `POST/PUT/PATCH/DELETE` are refused on environments named like production (`prod`, `production`, `live`) unless they set both `allow_writes: true` and `production_writes_acknowledged: true`. Other environments must be local, named like a test environment, or set `allow_writes: true`. |
|
|
138
|
+
| Writing to a database | `db` steps may only read (SELECT/WITH/SHOW/EXPLAIN; no write verbs anywhere, no `FOR UPDATE`), on a read-only connection inside a rolled-back transaction. Writes happen only in fixtures and cleanup, on databases with `allow_writes: true`. |
|
|
139
|
+
| SQL injection | Values go in `:named` parameters; `${...}` inside SQL text is refused, as is more than one statement. Table and column names in fixtures are validated. |
|
|
140
|
+
| Leaking secrets | Credentials come from `${ENV:NAME}`. Authorization headers show as `•••`, and any secret value is masked in requests, responses, errors and reports, as are database columns listed under `mask`. |
|
|
141
|
+
| Sending credentials elsewhere | Requests must stay on the environment's host; a flow can't send your token to another URL. |
|
|
142
|
+
| Runaway queries | At most 1,000 rows are read per query, with timeouts on every connection and request. |
|
|
143
|
+
|
|
144
|
+
## In CI
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
k11-api-functional-qa-mcp run --env staging
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
It writes `api-test-report.md` (PR comment), `api-test-report.json` (evidence) and `api-test-junit.xml`, sets GitHub Actions outputs and exits non-zero when a flow fails (`--fail-on any|failed|never`). See [`docs/api-tests.yml`](docs/api-tests.yml).
|
|
151
|
+
|
|
152
|
+
## Install
|
|
153
|
+
|
|
154
|
+
Requires Python 3.10+.
|
|
155
|
+
|
|
156
|
+
```text
|
|
157
|
+
/plugin marketplace add K11-Software-Solutions/k11techlab-api-functional-qa-mcp
|
|
158
|
+
/plugin install api-functional-qa@k11-api-functional-qa
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
The plugin adds `/api-story-test` and `/api-negative-tests`. To add only the MCP server:
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
claude mcp add api-functional-qa -- uvx k11-api-functional-qa-mcp
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
Claude Desktop and other clients: run `uvx k11-api-functional-qa-mcp` with `QA_MCP_ROOT` set to your project.
|
|
168
|
+
|
|
169
|
+
## Try the demo
|
|
170
|
+
|
|
171
|
+
```bash
|
|
172
|
+
python demo/shop/server.py --port 8080
|
|
173
|
+
SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token QA_MCP_ROOT=demo/shop k11-api-functional-qa-mcp run
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
## Scope
|
|
177
|
+
|
|
178
|
+
This server is for functional REST testing with database checks. Related K11 tools:
|
|
179
|
+
- **Contract and breaking-change checks** (REST, GraphQL, gRPC, AsyncAPI, SOAP): [k11 API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp)
|
|
180
|
+
- **UI tests:** [Playwright QA](https://github.com/K11-Software-Solutions/k11techlab-playwright-qa-mcp) and [Selenium QA](https://github.com/K11-Software-Solutions/k11techlab-selenium-qa-mcp)
|
|
181
|
+
- **LLM red-teaming:** [LLM Assure](https://github.com/K11-Software-Solutions/k11techlab-llm-assurance-toolkit)
|
|
182
|
+
|
|
183
|
+
Not covered: load testing (use k6 or Gatling) and deep fuzzing (use Schemathesis).
|
|
184
|
+
|
|
185
|
+
## Development
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
python -m venv .venv && .venv/Scripts/pip install -e ".[dev]" # bin/pip on macOS/Linux
|
|
189
|
+
pytest # values, guardrails, flows against the demo shop, negative tests, export, CLI, MCP (25)
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Roadmap
|
|
193
|
+
|
|
194
|
+
- GraphQL, gRPC and SOAP calls as flow steps
|
|
195
|
+
- Java export (REST Assured)
|
|
196
|
+
- MongoDB checks
|
|
197
|
+
- Schema-driven test data (realistic values per field format)
|
|
198
|
+
|
|
199
|
+
---
|
|
200
|
+
|
|
201
|
+
Built by [K11 Software Solutions](https://k11softwaresolutions.com). Part of the K11 Tech Lab agentic QA series. MIT licensed.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
k11_api_qa/__init__.py,sha256=nOwWcvfCkTMcgFP1s-lM_NQwVTDkzj9F_bCaJAIDywk,120
|
|
2
|
+
k11_api_qa/__main__.py,sha256=k1ocEWawweo1qCJWNFAAvyxz3tcY13dzvCenHszij30,48
|
|
3
|
+
k11_api_qa/auth.py,sha256=o5GWDqL-2MoMw4IENk1ji5w0mLG17HXG5yFq9YjaxaA,2174
|
|
4
|
+
k11_api_qa/cli.py,sha256=ToB3f-ek_81sZXn319cV0_ZGtb5IBAiV6AIBMb--p4k,3325
|
|
5
|
+
k11_api_qa/config.py,sha256=4p4JXNrSC-Y-uQOT3RijOrUOebL6kqsBoSQYshlUt-8,4693
|
|
6
|
+
k11_api_qa/db.py,sha256=09nznjh-RwQK0wMuzbAHc06bWN8cGXXhEdq0UOoqNKo,7933
|
|
7
|
+
k11_api_qa/export.py,sha256=vTMIvJDjqReg9Y1aOx4A0CIMbOwboeeIr6MUABtl3Bk,7433
|
|
8
|
+
k11_api_qa/generate.py,sha256=hrYjb5goeKd9LLu6nSSvypmmGhBXEDx58gI4MToQ2BU,14600
|
|
9
|
+
k11_api_qa/report.py,sha256=JQ3q-MGt2sYD7ryeHW3wjUSyzb5TuAOKwgyyRZRRDlY,5771
|
|
10
|
+
k11_api_qa/runner.py,sha256=4Khy-MuYqjvil-CB7_eeqNcTYudXaU6pb_RywQTJhgI,23754
|
|
11
|
+
k11_api_qa/server.py,sha256=eKtniHjbnNX68FNL9j6z9Xe7TCNxioZ7hYCTkDHfxRU,10872
|
|
12
|
+
k11_api_qa/values.py,sha256=K2rK4trWcfgq49ZV22CnYiS_9Xw_sLTXSD3RMaSh9wc,9298
|
|
13
|
+
k11_api_functional_qa_mcp-0.1.0.dist-info/METADATA,sha256=M5NG3n1tna41Y5r0Wase149S7dyrG0iOMjMfk3lKFBY,11186
|
|
14
|
+
k11_api_functional_qa_mcp-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
15
|
+
k11_api_functional_qa_mcp-0.1.0.dist-info/entry_points.txt,sha256=v0IZ8h4CcJr-6dEEmYdNaxLYnF8Sh_1PXE_Mrq9mkWQ,66
|
|
16
|
+
k11_api_functional_qa_mcp-0.1.0.dist-info/licenses/LICENSE,sha256=g-8tYo5PlaRgQntQTgc7h4ECTuGrwtYuzwpMT2sKPzo,1079
|
|
17
|
+
k11_api_functional_qa_mcp-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 K11 Software Solutions
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
k11_api_qa/__init__.py
ADDED
k11_api_qa/__main__.py
ADDED
k11_api_qa/auth.py
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Auth profiles: bearer, basic, api_key, oauth2 client credentials, or none.
|
|
2
|
+
|
|
3
|
+
Secrets come from ``${ENV:NAME}``; they are masked in every result. OAuth tokens are cached
|
|
4
|
+
for the run.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
import httpx
|
|
13
|
+
|
|
14
|
+
from .values import Vars
|
|
15
|
+
|
|
16
|
+
_cache: dict[str, tuple[str, float]] = {}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def headers(profile_name: str | None, profiles: dict, v: Vars, timeout: float = 20) -> dict[str, str]:
|
|
20
|
+
if not profile_name or profile_name == "none":
|
|
21
|
+
return {}
|
|
22
|
+
if profile_name not in profiles:
|
|
23
|
+
if profile_name == "default":
|
|
24
|
+
return {}
|
|
25
|
+
raise ValueError(f"Unknown auth profile '{profile_name}'. Known: {sorted(profiles) or ['(none)']}")
|
|
26
|
+
p = v.sub(profiles[profile_name])
|
|
27
|
+
kind = p.get("type", "bearer")
|
|
28
|
+
if kind == "bearer":
|
|
29
|
+
return {"Authorization": f"Bearer {p['token']}"}
|
|
30
|
+
if kind == "basic":
|
|
31
|
+
raw = f"{p['username']}:{p['password']}".encode()
|
|
32
|
+
token = base64.b64encode(raw).decode()
|
|
33
|
+
v.secrets.add(token)
|
|
34
|
+
return {"Authorization": f"Basic {token}"}
|
|
35
|
+
if kind == "api_key":
|
|
36
|
+
return {p.get("header", "X-API-Key"): str(p["value"])}
|
|
37
|
+
if kind in ("oauth2", "oauth2_client_credentials"):
|
|
38
|
+
key = f"{p['token_url']}|{p['client_id']}|{p.get('scope', '')}"
|
|
39
|
+
tok, exp = _cache.get(key, ("", 0.0))
|
|
40
|
+
if not tok or exp < time.time() + 30:
|
|
41
|
+
data = {"grant_type": "client_credentials", "client_id": p["client_id"], "client_secret": p["client_secret"]}
|
|
42
|
+
if p.get("scope"):
|
|
43
|
+
data["scope"] = p["scope"]
|
|
44
|
+
if p.get("audience"):
|
|
45
|
+
data["audience"] = p["audience"]
|
|
46
|
+
r = httpx.post(p["token_url"], data=data, timeout=timeout)
|
|
47
|
+
if r.status_code >= 400:
|
|
48
|
+
raise ValueError(f"OAuth token request failed with HTTP {r.status_code}")
|
|
49
|
+
body = r.json()
|
|
50
|
+
tok = body["access_token"]
|
|
51
|
+
_cache[key] = (tok, time.time() + float(body.get("expires_in", 300)))
|
|
52
|
+
v.secrets.add(tok)
|
|
53
|
+
return {"Authorization": f"Bearer {tok}"}
|
|
54
|
+
raise ValueError(f"Unknown auth type '{kind}' (bearer, basic, api_key, oauth2)")
|
k11_api_qa/cli.py
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
"""Command line: start the MCP server (default) or run flows in CI."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import __version__
|
|
11
|
+
|
|
12
|
+
USAGE = f"""k11-api-functional-qa-mcp {__version__}
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
k11-api-functional-qa-mcp start the MCP server on stdio
|
|
16
|
+
k11-api-functional-qa-mcp run [paths] [opts] run API test flows and write reports
|
|
17
|
+
k11-api-functional-qa-mcp --version
|
|
18
|
+
|
|
19
|
+
run options:
|
|
20
|
+
--env NAME environment from .qa-mcp/api-test.yaml (default: default_environment)
|
|
21
|
+
--tag TAG only flows with this tag (repeatable)
|
|
22
|
+
--report-dir PATH default .qa-mcp/api-test-report
|
|
23
|
+
--fail-on LEVEL any (default: fail on failed, blocked or error) | failed | never
|
|
24
|
+
|
|
25
|
+
The project is QA_MCP_ROOT or the current directory.
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def main(argv: list[str] | None = None) -> int:
|
|
30
|
+
argv = sys.argv[1:] if argv is None else argv
|
|
31
|
+
if not argv:
|
|
32
|
+
from .server import serve
|
|
33
|
+
|
|
34
|
+
serve()
|
|
35
|
+
return 0
|
|
36
|
+
if argv[0] in ("--version", "-v"):
|
|
37
|
+
print(__version__)
|
|
38
|
+
return 0
|
|
39
|
+
if argv[0] in ("--help", "-h", "help"):
|
|
40
|
+
print(USAGE)
|
|
41
|
+
return 0
|
|
42
|
+
if argv[0] != "run":
|
|
43
|
+
print(f"Unknown command: {argv[0]}\n\n{USAGE}", file=sys.stderr)
|
|
44
|
+
return 2
|
|
45
|
+
p = argparse.ArgumentParser(prog="k11-api-functional-qa-mcp run")
|
|
46
|
+
p.add_argument("paths", nargs="*")
|
|
47
|
+
p.add_argument("--env")
|
|
48
|
+
p.add_argument("--tag", action="append")
|
|
49
|
+
p.add_argument("--report-dir", default=".qa-mcp/api-test-report")
|
|
50
|
+
p.add_argument("--fail-on", default="any", choices=["any", "failed", "never"])
|
|
51
|
+
a = p.parse_args(argv[1:])
|
|
52
|
+
|
|
53
|
+
from k11_contract_qa.spec import SpecError, inside
|
|
54
|
+
|
|
55
|
+
from . import config, report
|
|
56
|
+
from .runner import flow_files, run_files
|
|
57
|
+
|
|
58
|
+
root = Path(os.environ.get("QA_MCP_ROOT") or os.getcwd()).resolve()
|
|
59
|
+
try:
|
|
60
|
+
cfg = config.load(root)
|
|
61
|
+
files = flow_files(root, a.paths or None)
|
|
62
|
+
if not files:
|
|
63
|
+
print("No flows found in tests/api/", file=sys.stderr)
|
|
64
|
+
return 1
|
|
65
|
+
results = run_files(cfg, files, a.env, a.tag)
|
|
66
|
+
env = results[0].environment if results else (a.env or cfg.default_environment)
|
|
67
|
+
paths = report.write(results, env, inside(root, a.report_dir))
|
|
68
|
+
except (SpecError, ValueError, OSError) as e:
|
|
69
|
+
print(f"run failed: {e}", file=sys.stderr)
|
|
70
|
+
return 1
|
|
71
|
+
c = report.counts(results)
|
|
72
|
+
print(f"{c['passed']}/{c['flows']} flows passed ({c['failed']} failed, {c['blocked']} blocked, {c['error']} errors, "
|
|
73
|
+
f"{c['server_errors']} server errors). Report: {paths['markdown']}")
|
|
74
|
+
if os.environ.get("GITHUB_OUTPUT"):
|
|
75
|
+
with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf8") as fh:
|
|
76
|
+
fh.write(f"passed={c['passed']}\nfailed={c['failed']}\nblocked={c['blocked']}\nerrors={c['error']}\n"
|
|
77
|
+
f"report={paths['markdown']}\njunit={paths['junit']}\n")
|
|
78
|
+
if os.environ.get("GITHUB_STEP_SUMMARY"):
|
|
79
|
+
with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf8") as fh:
|
|
80
|
+
fh.write(paths["markdown"].read_text("utf8") + "\n")
|
|
81
|
+
bad = {"any": c["failed"] + c["blocked"] + c["error"], "failed": c["failed"], "never": 0}[a.fail_on]
|
|
82
|
+
return 1 if bad else 0
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
if __name__ == "__main__":
|
|
86
|
+
raise SystemExit(main())
|
k11_api_qa/config.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Project config: ``.qa-mcp/api-test.yaml``.
|
|
2
|
+
|
|
3
|
+
spec: openapi.yaml # used for schema checks and negative tests
|
|
4
|
+
default_environment: local
|
|
5
|
+
environments:
|
|
6
|
+
local: { base_url: http://localhost:8080 }
|
|
7
|
+
staging: { base_url: https://staging.example.com, allow_writes: true }
|
|
8
|
+
prod: { base_url: https://api.example.com } # read-only: writes refused
|
|
9
|
+
auth:
|
|
10
|
+
default: { type: bearer, token: "${ENV:API_TOKEN}" }
|
|
11
|
+
# basic: {username, password} · api_key: {header, value} · oauth2: {token_url, client_id, client_secret, scope}
|
|
12
|
+
databases:
|
|
13
|
+
main: { url: "${ENV:TEST_DB_URL}", allow_writes: true, mask: [email] }
|
|
14
|
+
fixtures:
|
|
15
|
+
customer:
|
|
16
|
+
db: main
|
|
17
|
+
insert: { table: customers, values: { id: "${uuid}", email: "${random_email}" } }
|
|
18
|
+
delete: { table: customers, where: { id: "${id}" } }
|
|
19
|
+
|
|
20
|
+
Writes (POST/PUT/PATCH/DELETE, and database writes for fixtures/cleanup) are refused on an
|
|
21
|
+
environment unless it is local, its name looks like a test environment, or it sets
|
|
22
|
+
``allow_writes: true``. Environments named like production never allow writes unless they
|
|
23
|
+
also set ``production_writes_acknowledged: true``.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import re
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
from typing import Any
|
|
32
|
+
from urllib.parse import urlsplit
|
|
33
|
+
|
|
34
|
+
import yaml
|
|
35
|
+
|
|
36
|
+
CONFIG = ".qa-mcp/api-test.yaml"
|
|
37
|
+
PROD_NAME = re.compile(r"(^|[-_.])(prod|production|live|prd)([-_.]|$)", re.I)
|
|
38
|
+
TEST_NAME = re.compile(r"(^|[-_.])(local|dev|test|qa|ci|staging|stage|sandbox|preview|uat)([-_.]|$)", re.I)
|
|
39
|
+
LOCAL_HOST = re.compile(r"^(localhost|127\.0\.0\.1|\[::1\]|0\.0\.0\.0|.*\.(localhost|test|local))$", re.I)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass
|
|
43
|
+
class Config:
|
|
44
|
+
root: Path
|
|
45
|
+
spec: str | None = None
|
|
46
|
+
default_environment: str = "local"
|
|
47
|
+
environments: dict[str, dict] = field(default_factory=dict)
|
|
48
|
+
auth: dict[str, dict] = field(default_factory=dict)
|
|
49
|
+
databases: dict[str, dict] = field(default_factory=dict)
|
|
50
|
+
fixtures: dict[str, dict] = field(default_factory=dict)
|
|
51
|
+
timeout_s: float = 30
|
|
52
|
+
|
|
53
|
+
def environment(self, name: str | None) -> tuple[str, dict]:
|
|
54
|
+
name = name or self.default_environment
|
|
55
|
+
if name not in self.environments:
|
|
56
|
+
if not self.environments and name == "local":
|
|
57
|
+
return name, {"base_url": "http://localhost:8080"}
|
|
58
|
+
raise ValueError(f"Unknown environment '{name}'. Known: {sorted(self.environments) or ['(none)']}")
|
|
59
|
+
return name, self.environments[name]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def load(root: Path) -> Config:
|
|
63
|
+
path = Path(root) / CONFIG
|
|
64
|
+
data: dict[str, Any] = {}
|
|
65
|
+
if path.is_file():
|
|
66
|
+
data = yaml.safe_load(path.read_text("utf8")) or {}
|
|
67
|
+
return Config(root=Path(root), spec=data.get("spec"), default_environment=data.get("default_environment", "local"),
|
|
68
|
+
environments=data.get("environments") or {}, auth=data.get("auth") or {},
|
|
69
|
+
databases=data.get("databases") or {}, fixtures=data.get("fixtures") or {},
|
|
70
|
+
timeout_s=float(data.get("timeout_s", 30)))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def writes_allowed(env_name: str, env: dict, base_url: str) -> tuple[bool, str]:
|
|
74
|
+
"""Whether data-changing calls may run against this environment, and why."""
|
|
75
|
+
if PROD_NAME.search(env_name):
|
|
76
|
+
if env.get("allow_writes") and env.get("production_writes_acknowledged"):
|
|
77
|
+
return True, "production writes explicitly acknowledged"
|
|
78
|
+
return False, f"'{env_name}' looks like production; writes are refused"
|
|
79
|
+
if "allow_writes" in env:
|
|
80
|
+
return bool(env["allow_writes"]), ("allow_writes: true" if env["allow_writes"] else "allow_writes: false")
|
|
81
|
+
host = urlsplit(base_url).hostname or ""
|
|
82
|
+
if LOCAL_HOST.match(host):
|
|
83
|
+
return True, "local host"
|
|
84
|
+
if TEST_NAME.search(env_name):
|
|
85
|
+
return True, f"'{env_name}' is a test environment"
|
|
86
|
+
return False, f"'{env_name}' is not marked as a test environment; set allow_writes: true to permit writes"
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def summary(cfg: Config) -> dict:
|
|
90
|
+
envs = {}
|
|
91
|
+
for name, env in cfg.environments.items():
|
|
92
|
+
ok, why = writes_allowed(name, env, str(env.get("base_url", "")))
|
|
93
|
+
envs[name] = {"base_url": env.get("base_url"), "writes": ok, "why": why, "auth": env.get("auth", "default")}
|
|
94
|
+
return {"config": CONFIG if (cfg.root / CONFIG).is_file() else None, "spec": cfg.spec,
|
|
95
|
+
"default_environment": cfg.default_environment, "environments": envs,
|
|
96
|
+
"auth_profiles": {k: v.get("type", "bearer") for k, v in cfg.auth.items()},
|
|
97
|
+
"databases": {k: {"allow_writes": bool(v.get("allow_writes")), "mask": v.get("mask", [])} for k, v in cfg.databases.items()},
|
|
98
|
+
"fixtures": sorted(cfg.fixtures)}
|