k11-performance-qa-mcp 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- k11_performance_qa_mcp-0.1.0/.gitignore +10 -0
- k11_performance_qa_mcp-0.1.0/LICENSE +21 -0
- k11_performance_qa_mcp-0.1.0/PKG-INFO +200 -0
- k11_performance_qa_mcp-0.1.0/README.md +178 -0
- k11_performance_qa_mcp-0.1.0/pyproject.toml +42 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/__init__.py +3 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/__main__.py +3 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/analyze.py +156 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/cli.py +150 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/engine.py +314 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/jmeter.py +114 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/k6engine.py +348 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/perfconfig.py +184 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/report.py +116 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/run.py +84 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/server.py +166 -0
- k11_performance_qa_mcp-0.1.0/src/k11_perf_qa/stats.py +74 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 K11 Software Solutions
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: k11-performance-qa-mcp
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: MCP server and CI tool for load and performance testing as release evidence: API flows as load scenarios, per-endpoint latency and SLOs, statistically significant regressions against a baseline, k6 or builtin engine or imported JMeter results, with guardrails.
|
|
5
|
+
Project-URL: Homepage, https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp
|
|
6
|
+
Project-URL: Issues, https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/issues
|
|
7
|
+
Author: K11 Software Solutions
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: claude,jmeter,k6,latency,load-testing,mcp,performance-testing,qa,regression,slo
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Software Development :: Testing
|
|
15
|
+
Classifier: Topic :: Software Development :: Testing :: Traffic Generation
|
|
16
|
+
Requires-Python: >=3.10
|
|
17
|
+
Requires-Dist: k11-api-functional-qa-mcp<0.2,>=0.1.1
|
|
18
|
+
Provides-Extra: dev
|
|
19
|
+
Requires-Dist: pytest-timeout>=2; extra == 'dev'
|
|
20
|
+
Requires-Dist: pytest>=8; extra == 'dev'
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# k11 Performance QA: MCP server and CI tool
|
|
24
|
+
|
|
25
|
+
<!-- mcp-name: io.github.K11-Software-Solutions/performance-qa -->
|
|
26
|
+
|
|
27
|
+
[](https://pypi.org/project/k11-performance-qa-mcp/)
|
|
28
|
+
[](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml)
|
|
29
|
+
[](LICENSE)
|
|
30
|
+
|
|
31
|
+
**Load testing that answers a release question: did this change make the API slower, or break it under load?**
|
|
32
|
+
|
|
33
|
+
Most load-testing tools give you numbers. This one gives you **evidence**: per-endpoint latency against your SLOs, and regressions against a baseline that are **statistically significant**, so a noisy run doesn't block a release and a real slowdown doesn't slip through. The report feeds straight into the [k11 Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp).
|
|
34
|
+
|
|
35
|
+
- **Your API tests become load scenarios.** Scenarios reuse the user-story flows from [k11 API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) (`tests/api/*.yaml`): chained calls, saved IDs, fixtures and auth. No second set of scripts to maintain.
|
|
36
|
+
- **Per-endpoint metrics:** p50, p90, p95, p99, max, error rate, 5xx count and throughput for every `METHOD /path/{param}`.
|
|
37
|
+
- **SLOs** per endpoint or pattern (`p95_ms`, `p99_ms`, `error_rate`, `min_rps`, …).
|
|
38
|
+
- **Regressions that mean something:** slower than the baseline by more than a tolerance **and** a minimum number of milliseconds **and** significant (one-sided Mann-Whitney U test).
|
|
39
|
+
- **Guardrails:** no load on production-like environments, data-changing flows only where writes are allowed, caps on virtual users, duration and request rate, and an automatic stop when too many requests fail.
|
|
40
|
+
- **Two engines:** [Grafana k6](https://k6.io) when it's installed, for heavy load; a builtin engine (Python, no install) for quick checks and CI.
|
|
41
|
+
- **Already on JMeter?** Import your existing JMeter results (`.jtl`) and get the same SLO checks, regressions and Release Gate evidence, without rewriting a single test plan.
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
```bash
|
|
46
|
+
pip install k11-performance-qa-mcp # or: uvx k11-performance-qa-mcp
|
|
47
|
+
k11-performance-qa-mcp init # writes .qa-mcp/perf.yaml
|
|
48
|
+
k11-performance-qa-mcp baseline # on main: run and save the baseline
|
|
49
|
+
k11-performance-qa-mcp run # on a branch: run, compare, write the report
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
In Claude Code, as a plugin with the `/perf-check` and `/perf-baseline` commands:
|
|
53
|
+
|
|
54
|
+
```text
|
|
55
|
+
/plugin marketplace add K11-Software-Solutions/k11techlab-performance-qa-mcp
|
|
56
|
+
/plugin install performance-qa@k11-performance-qa
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Or as an MCP server in any client:
|
|
60
|
+
|
|
61
|
+
```json
|
|
62
|
+
{ "mcpServers": { "performance-qa": { "command": "uvx", "args": ["k11-performance-qa-mcp"] } } }
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Configure
|
|
66
|
+
|
|
67
|
+
Environments and auth come from `.qa-mcp/api-test.yaml` (shared with API Functional QA). Load settings live in `.qa-mcp/perf.yaml`:
|
|
68
|
+
|
|
69
|
+
```yaml
|
|
70
|
+
environment: staging
|
|
71
|
+
engine: auto # k6 if installed, else builtin
|
|
72
|
+
profile: { vus: 10, duration: 2m, ramp_up: 20s, think_time_ms: 200 }
|
|
73
|
+
|
|
74
|
+
scenarios:
|
|
75
|
+
browse:
|
|
76
|
+
flows:
|
|
77
|
+
- { flow: tests/api/view_order.yaml, weight: 3 } # 3x more often
|
|
78
|
+
- tests/api/cancel_order.yaml
|
|
79
|
+
|
|
80
|
+
slos:
|
|
81
|
+
"GET /orders/{id}": { p95_ms: 250, error_rate: 0.01 }
|
|
82
|
+
"POST /orders": { p95_ms: 600 }
|
|
83
|
+
"*": { p95_ms: 800, error_rate: 0.02 }
|
|
84
|
+
|
|
85
|
+
regression: { metric: p95, tolerance_pct: 10, min_delta_ms: 10, min_samples: 30, p_value: 0.01 }
|
|
86
|
+
guardrails: { max_vus: 50, max_duration: 10m, max_rps: 200, abort_error_rate: 0.5 }
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
Each virtual user runs a flow's fixtures once, then repeats: pick a flow by weight, run its HTTP steps in order (IDs saved by one step feed the next), run its cleanup, think, repeat. Database checks are skipped under load; they check correctness, not speed, and still run in your functional tests.
|
|
90
|
+
|
|
91
|
+
## Already using JMeter?
|
|
92
|
+
|
|
93
|
+
Keep your test plans. Run them as usual, then import the results file:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
jmeter -n -t checkout.jmx -l results.jtl # your existing JMeter run
|
|
97
|
+
k11-performance-qa-mcp import results.jtl --env staging --save-baseline # on main
|
|
98
|
+
k11-performance-qa-mcp import results.jtl --env staging # on a branch: compare, report
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
CSV results (JMeter's default, with the header row) and XML results both work. Each sampler label becomes an endpoint, so name samplers like `GET /orders/{id}` to match your SLO patterns. Transaction Controller samples are skipped (they sum their children), connection failures count as errors, and 5xx responses are server errors. Since JMeter generated the load, k11's guardrails don't apply; the analysis, baseline comparison and report are exactly the same as for a k6 or builtin run.
|
|
102
|
+
|
|
103
|
+
## How a regression is decided
|
|
104
|
+
|
|
105
|
+
For each endpoint, the run is compared with the saved baseline on the configured metric (p95 by default). It's a **regression** only when all of these hold:
|
|
106
|
+
|
|
107
|
+
1. it's slower by more than `tolerance_pct` (default 10%);
|
|
108
|
+
2. it's slower by more than `min_delta_ms` (default 10 ms), so 2 ms → 3 ms isn't "50% slower";
|
|
109
|
+
3. both runs have at least `min_samples` requests for that endpoint;
|
|
110
|
+
4. a one-sided Mann-Whitney U test says the new latencies are higher with p < `p_value` (default 0.01).
|
|
111
|
+
|
|
112
|
+
A rise in the error rate of more than 1 percentage point is reported separately. Improvements are reported too.
|
|
113
|
+
|
|
114
|
+
## What the report contains
|
|
115
|
+
|
|
116
|
+
`.qa-mcp/perf-report/perf-report.json` holds the verdict, per-endpoint metrics and status codes, SLO results, the baseline comparison (with p-values), findings, error examples, a per-second timeline and the profile and engine used. `perf-report.md` is ready for a pull request or CI step summary. Findings:
|
|
117
|
+
|
|
118
|
+
| Finding | Release Gate rule | Default outcome |
|
|
119
|
+
|---|---|---|
|
|
120
|
+
| 5xx responses under load | `perf.server_errors` | block |
|
|
121
|
+
| An SLO missed | `perf.slo_breached` | block |
|
|
122
|
+
| Run aborted (too many failures) | `perf.aborted` | block |
|
|
123
|
+
| Significant regression vs baseline | `perf.regression` | needs a person |
|
|
124
|
+
| Run refused by guardrails / didn't start | `perf.not_run` | needs a person |
|
|
125
|
+
| No baseline, or too few samples | `perf.no_baseline`, `perf.insufficient_samples` | note |
|
|
126
|
+
|
|
127
|
+
## Tools (MCP)
|
|
128
|
+
|
|
129
|
+
| Tool | What it does |
|
|
130
|
+
|---|---|
|
|
131
|
+
| `perf_config` | Environments (and whether load is allowed, and why), scenarios and their flows, SLOs, guardrails, the engine that will run, saved baselines |
|
|
132
|
+
| `run_load_test` | Run a scenario; returns the verdict, findings, per-endpoint latency and the baseline comparison; writes the report |
|
|
133
|
+
| `save_baseline` | Save the last completed run (or a fresh run) as the baseline |
|
|
134
|
+
| `import_jmeter_results` | Analyse a JMeter results file (CSV or XML) like a run: SLOs, baseline comparison, report |
|
|
135
|
+
| `get_report` | The last report, or one endpoint's detail: percentiles, status codes, SLOs, comparison, error examples |
|
|
136
|
+
| `init_perf_config` | Write a starter `.qa-mcp/perf.yaml` |
|
|
137
|
+
|
|
138
|
+
## Command line
|
|
139
|
+
|
|
140
|
+
```text
|
|
141
|
+
k11-performance-qa-mcp run [--scenario S] [--env E] [--vus N] [--duration 2m] [--ramp-up 20s] [--max-rps N]
|
|
142
|
+
[--engine auto|k6|builtin] [--no-compare] [--fail-on any|slo|regression|never]
|
|
143
|
+
k11-performance-qa-mcp baseline [same options] | baseline --from-last-run
|
|
144
|
+
k11-performance-qa-mcp import FILE.jtl [--scenario S] [--env E] [--save-baseline] [--no-compare] [--fail-on ...]
|
|
145
|
+
k11-performance-qa-mcp init
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
In GitHub Actions it writes the Markdown report to the step summary and sets `verdict`, `p95`, `slo_breaches` and `regressions` outputs. See [docs/perf-tests.yml](docs/perf-tests.yml) for a workflow that saves the baseline on main and compares every pull request with it.
|
|
149
|
+
|
|
150
|
+
## Safety
|
|
151
|
+
|
|
152
|
+
- **No production load:** environments named like production are refused unless they set both `allow_load: true` and `production_load_acknowledged: true` in `api-test.yaml`. Other environments need a local host, a test-like name (`staging`, `qa`, `dev`, …) or `allow_load: true`.
|
|
153
|
+
- **Writes follow API Functional QA's rules:** a scenario with data-changing calls is refused on an environment that doesn't allow writes, before any request is sent.
|
|
154
|
+
- **Caps:** `max_vus`, `max_duration` and `max_rps`; the run stops when the error rate passes `abort_error_rate`.
|
|
155
|
+
- **Secrets** come from `${ENV:NAME}` and never appear in reports or the generated k6 script; k6 receives auth headers through its environment.
|
|
156
|
+
|
|
157
|
+
## Engines
|
|
158
|
+
|
|
159
|
+
| | builtin | k6 |
|
|
160
|
+
|---|---|---|
|
|
161
|
+
| Install | none | [install k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) (or set `K6_PATH`) |
|
|
162
|
+
| Load | up to a few hundred requests per second | thousands per second |
|
|
163
|
+
| Checks | status and value checks from the flow | status codes (value checks run in functional tests) |
|
|
164
|
+
| Fixtures and auth | yes | yes (prepared in Python, handed to k6) |
|
|
165
|
+
| Database cleanup steps | yes | no; use a disposable test database |
|
|
166
|
+
|
|
167
|
+
Keep the engine the same between a baseline and the runs compared with it; the report warns when they differ. JMeter imports are a third source and compare best with JMeter baselines.
|
|
168
|
+
|
|
169
|
+
## Demo
|
|
170
|
+
|
|
171
|
+
`demo/shop` is a small order API with switches to simulate a bad release: `--slow-ms 80` slows `GET /orders/{id}`, `--error-every 5` makes every fifth call fail.
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
python demo/shop/server.py --port 8080 &
|
|
175
|
+
cd demo/shop && SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp baseline --scenario browse
|
|
176
|
+
# restart the server with --slow-ms 80, then:
|
|
177
|
+
SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp run --scenario browse
|
|
178
|
+
# ❌ regression: GET /orders/{id}: p95 32.1 → 92.1 ms (+186.9%, p=0.0000)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
## Part of the K11 QA suite
|
|
182
|
+
|
|
183
|
+
[Playwright QA](https://github.com/K11-Software-Solutions/k11techlab-playwright-qa-mcp) · [Selenium QA](https://github.com/K11-Software-Solutions/k11techlab-selenium-qa-mcp) · [API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp) · [API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) · [LLM Assure](https://github.com/K11-Software-Solutions/k11techlab-llm-assurance-toolkit) · **Performance QA** · [Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp) · [Release Readiness Agent](https://github.com/K11-Software-Solutions/k11techlab-release-readiness-agent)
|
|
184
|
+
|
|
185
|
+
## Development
|
|
186
|
+
|
|
187
|
+
```bash
|
|
188
|
+
python -m venv .venv && .venv/Scripts/pip install -e ".[dev]" # bin/pip on macOS/Linux
|
|
189
|
+
pytest # statistics, guardrails, real load against the demo shop, CLI, MCP; the k6 test runs when k6 is installed
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
## Roadmap
|
|
193
|
+
|
|
194
|
+
- LLM performance and cost: time to first token, tokens per second, cost per request and per conversation.
|
|
195
|
+
- Open-model (arrival-rate) scenarios and soak tests.
|
|
196
|
+
- Comparing two branches deployed side by side.
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
Built by [K11 Software Solutions](https://k11softwaresolutions.com). MIT licensed.
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
# k11 Performance QA: MCP server and CI tool
|
|
2
|
+
|
|
3
|
+
<!-- mcp-name: io.github.K11-Software-Solutions/performance-qa -->
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/k11-performance-qa-mcp/)
|
|
6
|
+
[](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml)
|
|
7
|
+
[](LICENSE)
|
|
8
|
+
|
|
9
|
+
**Load testing that answers a release question: did this change make the API slower, or break it under load?**
|
|
10
|
+
|
|
11
|
+
Most load-testing tools give you numbers. This one gives you **evidence**: per-endpoint latency against your SLOs, and regressions against a baseline that are **statistically significant**, so a noisy run doesn't block a release and a real slowdown doesn't slip through. The report feeds straight into the [k11 Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp).
|
|
12
|
+
|
|
13
|
+
- **Your API tests become load scenarios.** Scenarios reuse the user-story flows from [k11 API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) (`tests/api/*.yaml`): chained calls, saved IDs, fixtures and auth. No second set of scripts to maintain.
|
|
14
|
+
- **Per-endpoint metrics:** p50, p90, p95, p99, max, error rate, 5xx count and throughput for every `METHOD /path/{param}`.
|
|
15
|
+
- **SLOs** per endpoint or pattern (`p95_ms`, `p99_ms`, `error_rate`, `min_rps`, …).
|
|
16
|
+
- **Regressions that mean something:** slower than the baseline by more than a tolerance **and** a minimum number of milliseconds **and** significant (one-sided Mann-Whitney U test).
|
|
17
|
+
- **Guardrails:** no load on production-like environments, data-changing flows only where writes are allowed, caps on virtual users, duration and request rate, and an automatic stop when too many requests fail.
|
|
18
|
+
- **Two engines:** [Grafana k6](https://k6.io) when it's installed, for heavy load; a builtin engine (Python, no install) for quick checks and CI.
|
|
19
|
+
- **Already on JMeter?** Import your existing JMeter results (`.jtl`) and get the same SLO checks, regressions and Release Gate evidence, without rewriting a single test plan.
|
|
20
|
+
|
|
21
|
+
## Quick start
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install k11-performance-qa-mcp # or: uvx k11-performance-qa-mcp
|
|
25
|
+
k11-performance-qa-mcp init # writes .qa-mcp/perf.yaml
|
|
26
|
+
k11-performance-qa-mcp baseline # on main: run and save the baseline
|
|
27
|
+
k11-performance-qa-mcp run # on a branch: run, compare, write the report
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
In Claude Code, as a plugin with the `/perf-check` and `/perf-baseline` commands:
|
|
31
|
+
|
|
32
|
+
```text
|
|
33
|
+
/plugin marketplace add K11-Software-Solutions/k11techlab-performance-qa-mcp
|
|
34
|
+
/plugin install performance-qa@k11-performance-qa
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Or as an MCP server in any client:
|
|
38
|
+
|
|
39
|
+
```json
|
|
40
|
+
{ "mcpServers": { "performance-qa": { "command": "uvx", "args": ["k11-performance-qa-mcp"] } } }
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Configure
|
|
44
|
+
|
|
45
|
+
Environments and auth come from `.qa-mcp/api-test.yaml` (shared with API Functional QA). Load settings live in `.qa-mcp/perf.yaml`:
|
|
46
|
+
|
|
47
|
+
```yaml
|
|
48
|
+
environment: staging
|
|
49
|
+
engine: auto # k6 if installed, else builtin
|
|
50
|
+
profile: { vus: 10, duration: 2m, ramp_up: 20s, think_time_ms: 200 }
|
|
51
|
+
|
|
52
|
+
scenarios:
|
|
53
|
+
browse:
|
|
54
|
+
flows:
|
|
55
|
+
- { flow: tests/api/view_order.yaml, weight: 3 } # 3x more often
|
|
56
|
+
- tests/api/cancel_order.yaml
|
|
57
|
+
|
|
58
|
+
slos:
|
|
59
|
+
"GET /orders/{id}": { p95_ms: 250, error_rate: 0.01 }
|
|
60
|
+
"POST /orders": { p95_ms: 600 }
|
|
61
|
+
"*": { p95_ms: 800, error_rate: 0.02 }
|
|
62
|
+
|
|
63
|
+
regression: { metric: p95, tolerance_pct: 10, min_delta_ms: 10, min_samples: 30, p_value: 0.01 }
|
|
64
|
+
guardrails: { max_vus: 50, max_duration: 10m, max_rps: 200, abort_error_rate: 0.5 }
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Each virtual user runs a flow's fixtures once, then repeats: pick a flow by weight, run its HTTP steps in order (IDs saved by one step feed the next), run its cleanup, think, repeat. Database checks are skipped under load; they check correctness, not speed, and still run in your functional tests.
|
|
68
|
+
|
|
69
|
+
## Already using JMeter?
|
|
70
|
+
|
|
71
|
+
Keep your test plans. Run them as usual, then import the results file:
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
jmeter -n -t checkout.jmx -l results.jtl # your existing JMeter run
|
|
75
|
+
k11-performance-qa-mcp import results.jtl --env staging --save-baseline # on main
|
|
76
|
+
k11-performance-qa-mcp import results.jtl --env staging # on a branch: compare, report
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
CSV results (JMeter's default, with the header row) and XML results both work. Each sampler label becomes an endpoint, so name samplers like `GET /orders/{id}` to match your SLO patterns. Transaction Controller samples are skipped (they sum their children), connection failures count as errors, and 5xx responses are server errors. Since JMeter generated the load, k11's guardrails don't apply; the analysis, baseline comparison and report are exactly the same as for a k6 or builtin run.
|
|
80
|
+
|
|
81
|
+
## How a regression is decided
|
|
82
|
+
|
|
83
|
+
For each endpoint, the run is compared with the saved baseline on the configured metric (p95 by default). It's a **regression** only when all of these hold:
|
|
84
|
+
|
|
85
|
+
1. it's slower by more than `tolerance_pct` (default 10%);
|
|
86
|
+
2. it's slower by more than `min_delta_ms` (default 10 ms), so 2 ms → 3 ms isn't "50% slower";
|
|
87
|
+
3. both runs have at least `min_samples` requests for that endpoint;
|
|
88
|
+
4. a one-sided Mann-Whitney U test says the new latencies are higher with p < `p_value` (default 0.01).
|
|
89
|
+
|
|
90
|
+
A rise in the error rate of more than 1 percentage point is reported separately. Improvements are reported too.
|
|
91
|
+
|
|
92
|
+
## What the report contains
|
|
93
|
+
|
|
94
|
+
`.qa-mcp/perf-report/perf-report.json` holds the verdict, per-endpoint metrics and status codes, SLO results, the baseline comparison (with p-values), findings, error examples, a per-second timeline and the profile and engine used. `perf-report.md` is ready for a pull request or CI step summary. Findings:
|
|
95
|
+
|
|
96
|
+
| Finding | Release Gate rule | Default outcome |
|
|
97
|
+
|---|---|---|
|
|
98
|
+
| 5xx responses under load | `perf.server_errors` | block |
|
|
99
|
+
| An SLO missed | `perf.slo_breached` | block |
|
|
100
|
+
| Run aborted (too many failures) | `perf.aborted` | block |
|
|
101
|
+
| Significant regression vs baseline | `perf.regression` | needs a person |
|
|
102
|
+
| Run refused by guardrails / didn't start | `perf.not_run` | needs a person |
|
|
103
|
+
| No baseline, or too few samples | `perf.no_baseline`, `perf.insufficient_samples` | note |
|
|
104
|
+
|
|
105
|
+
## Tools (MCP)
|
|
106
|
+
|
|
107
|
+
| Tool | What it does |
|
|
108
|
+
|---|---|
|
|
109
|
+
| `perf_config` | Environments (and whether load is allowed, and why), scenarios and their flows, SLOs, guardrails, the engine that will run, saved baselines |
|
|
110
|
+
| `run_load_test` | Run a scenario; returns the verdict, findings, per-endpoint latency and the baseline comparison; writes the report |
|
|
111
|
+
| `save_baseline` | Save the last completed run (or a fresh run) as the baseline |
|
|
112
|
+
| `import_jmeter_results` | Analyse a JMeter results file (CSV or XML) like a run: SLOs, baseline comparison, report |
|
|
113
|
+
| `get_report` | The last report, or one endpoint's detail: percentiles, status codes, SLOs, comparison, error examples |
|
|
114
|
+
| `init_perf_config` | Write a starter `.qa-mcp/perf.yaml` |
|
|
115
|
+
|
|
116
|
+
## Command line
|
|
117
|
+
|
|
118
|
+
```text
|
|
119
|
+
k11-performance-qa-mcp run [--scenario S] [--env E] [--vus N] [--duration 2m] [--ramp-up 20s] [--max-rps N]
|
|
120
|
+
[--engine auto|k6|builtin] [--no-compare] [--fail-on any|slo|regression|never]
|
|
121
|
+
k11-performance-qa-mcp baseline [same options] | baseline --from-last-run
|
|
122
|
+
k11-performance-qa-mcp import FILE.jtl [--scenario S] [--env E] [--save-baseline] [--no-compare] [--fail-on ...]
|
|
123
|
+
k11-performance-qa-mcp init
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
In GitHub Actions it writes the Markdown report to the step summary and sets `verdict`, `p95`, `slo_breaches` and `regressions` outputs. See [docs/perf-tests.yml](docs/perf-tests.yml) for a workflow that saves the baseline on main and compares every pull request with it.
|
|
127
|
+
|
|
128
|
+
## Safety
|
|
129
|
+
|
|
130
|
+
- **No production load:** environments named like production are refused unless they set both `allow_load: true` and `production_load_acknowledged: true` in `api-test.yaml`. Other environments need a local host, a test-like name (`staging`, `qa`, `dev`, …) or `allow_load: true`.
|
|
131
|
+
- **Writes follow API Functional QA's rules:** a scenario with data-changing calls is refused on an environment that doesn't allow writes, before any request is sent.
|
|
132
|
+
- **Caps:** `max_vus`, `max_duration` and `max_rps`; the run stops when the error rate passes `abort_error_rate`.
|
|
133
|
+
- **Secrets** come from `${ENV:NAME}` and never appear in reports or the generated k6 script; k6 receives auth headers through its environment.
|
|
134
|
+
|
|
135
|
+
## Engines
|
|
136
|
+
|
|
137
|
+
| | builtin | k6 |
|
|
138
|
+
|---|---|---|
|
|
139
|
+
| Install | none | [install k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) (or set `K6_PATH`) |
|
|
140
|
+
| Load | up to a few hundred requests per second | thousands per second |
|
|
141
|
+
| Checks | status and value checks from the flow | status codes (value checks run in functional tests) |
|
|
142
|
+
| Fixtures and auth | yes | yes (prepared in Python, handed to k6) |
|
|
143
|
+
| Database cleanup steps | yes | no; use a disposable test database |
|
|
144
|
+
|
|
145
|
+
Keep the engine the same between a baseline and the runs compared with it; the report warns when they differ. JMeter imports are a third source and compare best with JMeter baselines.
|
|
146
|
+
|
|
147
|
+
## Demo
|
|
148
|
+
|
|
149
|
+
`demo/shop` is a small order API with switches to simulate a bad release: `--slow-ms 80` slows `GET /orders/{id}`, `--error-every 5` makes every fifth call fail.
|
|
150
|
+
|
|
151
|
+
```bash
|
|
152
|
+
python demo/shop/server.py --port 8080 &
|
|
153
|
+
cd demo/shop && SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp baseline --scenario browse
|
|
154
|
+
# restart the server with --slow-ms 80, then:
|
|
155
|
+
SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp run --scenario browse
|
|
156
|
+
# ❌ regression: GET /orders/{id}: p95 32.1 → 92.1 ms (+186.9%, p=0.0000)
|
|
157
|
+
```
|
|
158
|
+
|
|
159
|
+
## Part of the K11 QA suite
|
|
160
|
+
|
|
161
|
+
[Playwright QA](https://github.com/K11-Software-Solutions/k11techlab-playwright-qa-mcp) · [Selenium QA](https://github.com/K11-Software-Solutions/k11techlab-selenium-qa-mcp) · [API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp) · [API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) · [LLM Assure](https://github.com/K11-Software-Solutions/k11techlab-llm-assurance-toolkit) · **Performance QA** · [Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp) · [Release Readiness Agent](https://github.com/K11-Software-Solutions/k11techlab-release-readiness-agent)
|
|
162
|
+
|
|
163
|
+
## Development
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
python -m venv .venv && .venv/Scripts/pip install -e ".[dev]" # bin/pip on macOS/Linux
|
|
167
|
+
pytest # statistics, guardrails, real load against the demo shop, CLI, MCP; the k6 test runs when k6 is installed
|
|
168
|
+
```
|
|
169
|
+
|
|
170
|
+
## Roadmap
|
|
171
|
+
|
|
172
|
+
- LLM performance and cost: time to first token, tokens per second, cost per request and per conversation.
|
|
173
|
+
- Open-model (arrival-rate) scenarios and soak tests.
|
|
174
|
+
- Comparing two branches deployed side by side.
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
Built by [K11 Software Solutions](https://k11softwaresolutions.com). MIT licensed.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.25"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "k11-performance-qa-mcp"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "MCP server and CI tool for load and performance testing as release evidence: API flows as load scenarios, per-endpoint latency and SLOs, statistically significant regressions against a baseline, k6 or builtin engine or imported JMeter results, with guardrails."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "K11 Software Solutions" }]
|
|
13
|
+
requires-python = ">=3.10"
|
|
14
|
+
keywords = ["mcp", "performance-testing", "load-testing", "k6", "jmeter", "slo", "latency", "regression", "qa", "claude"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Software Development :: Testing",
|
|
20
|
+
"Topic :: Software Development :: Testing :: Traffic Generation",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["k11-api-functional-qa-mcp>=0.1.1,<0.2"]
|
|
23
|
+
|
|
24
|
+
[project.optional-dependencies]
|
|
25
|
+
dev = ["pytest>=8", "pytest-timeout>=2"]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp"
|
|
29
|
+
Issues = "https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/issues"
|
|
30
|
+
|
|
31
|
+
[project.scripts]
|
|
32
|
+
k11-performance-qa-mcp = "k11_perf_qa.cli:main"
|
|
33
|
+
|
|
34
|
+
[tool.hatch.build.targets.wheel]
|
|
35
|
+
packages = ["src/k11_perf_qa"]
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.targets.sdist]
|
|
38
|
+
include = ["src", "README.md", "LICENSE"]
|
|
39
|
+
|
|
40
|
+
[tool.pytest.ini_options]
|
|
41
|
+
testpaths = ["tests"]
|
|
42
|
+
timeout = 300
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
"""From raw samples to evidence: per-endpoint metrics, SLO checks, baseline comparison, findings.
|
|
2
|
+
|
|
3
|
+
Findings (what the Release Gate reads):
|
|
4
|
+
server_errors 5xx responses under load (a bug, whatever the latency)
|
|
5
|
+
slo_breach an endpoint missed an SLO from perf.yaml
|
|
6
|
+
regression slower than the baseline beyond tolerance, min_delta_ms AND significance
|
|
7
|
+
error_rate_regression more failed requests than the baseline
|
|
8
|
+
aborted / not_run the run stopped on the error-rate guardrail, or never started
|
|
9
|
+
insufficient_samples too few requests to compare an endpoint with its baseline
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import fnmatch
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
from .engine import RunResult
|
|
18
|
+
from .stats import mann_whitney_greater, summarize
|
|
19
|
+
|
|
20
|
+
SLO_KEYS = {"avg_ms": "avg", "p50_ms": "p50", "p90_ms": "p90", "p95_ms": "p95", "p99_ms": "p99", "max_ms": "max",
|
|
21
|
+
"error_rate": "error_rate", "min_rps": "rps"}
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def endpoint_metrics(res: RunResult) -> list[dict]:
|
|
25
|
+
dur = max(res.duration_s, 0.001)
|
|
26
|
+
out = []
|
|
27
|
+
for ep in res.endpoints:
|
|
28
|
+
n = res.requests(ep)
|
|
29
|
+
errs = res.errors.get(ep, 0)
|
|
30
|
+
m = {"endpoint": ep, "requests": n, "errors": errs, "error_rate": round(errs / n, 4) if n else 0.0,
|
|
31
|
+
"server_errors": res.server_errors.get(ep, 0), "check_failures": res.check_failures.get(ep, 0),
|
|
32
|
+
"rps": round(n / dur, 2), "status_codes": {str(k): v for k, v in sorted(res.codes.get(ep, {}).items())},
|
|
33
|
+
**summarize(res.samples.get(ep, []))}
|
|
34
|
+
out.append(m)
|
|
35
|
+
return out
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def overall(res: RunResult, eps: list[dict]) -> dict:
|
|
39
|
+
allv = [v for ep in res.samples.values() for v in ep]
|
|
40
|
+
n = sum(e["requests"] for e in eps)
|
|
41
|
+
errs = sum(e["errors"] for e in eps)
|
|
42
|
+
return {"requests": n, "errors": errs, "error_rate": round(errs / n, 4) if n else 0.0,
|
|
43
|
+
"server_errors": sum(e["server_errors"] for e in eps), "check_failures": sum(e["check_failures"] for e in eps),
|
|
44
|
+
"rps": round(n / max(res.duration_s, 0.001), 2), "iterations": res.iterations, **summarize(allv)}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _slo_for(ep: str, slos: dict[str, dict]) -> tuple[str | None, dict]:
|
|
48
|
+
if ep in slos:
|
|
49
|
+
return ep, slos[ep]
|
|
50
|
+
for pat, spec in slos.items():
|
|
51
|
+
if pat != "*" and fnmatch.fnmatchcase(ep, pat):
|
|
52
|
+
return pat, spec
|
|
53
|
+
if "*" in slos:
|
|
54
|
+
return "*", slos["*"]
|
|
55
|
+
return None, {}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def check_slos(eps: list[dict], slos: dict[str, dict]) -> list[dict]:
|
|
59
|
+
out = []
|
|
60
|
+
for e in eps:
|
|
61
|
+
pat, spec = _slo_for(e["endpoint"], slos)
|
|
62
|
+
for key, target in (spec or {}).items():
|
|
63
|
+
if key not in SLO_KEYS:
|
|
64
|
+
raise ValueError(f"Unknown SLO '{key}' for {pat} (use {sorted(SLO_KEYS)})")
|
|
65
|
+
metric = SLO_KEYS[key]
|
|
66
|
+
actual = e[metric]
|
|
67
|
+
ok = actual >= float(target) if key == "min_rps" else actual <= float(target)
|
|
68
|
+
if e["requests"] == 0 and key != "error_rate":
|
|
69
|
+
continue
|
|
70
|
+
out.append({"endpoint": e["endpoint"], "slo": key, "pattern": pat, "target": target, "actual": actual, "ok": ok})
|
|
71
|
+
return out
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def compare(res: RunResult, eps: list[dict], baseline: dict, reg: dict) -> dict:
|
|
75
|
+
metric = reg["metric"]
|
|
76
|
+
tol, min_ms, min_n, alpha = float(reg["tolerance_pct"]), float(reg["min_delta_ms"]), int(reg["min_samples"]), float(reg["p_value"])
|
|
77
|
+
base_eps = {e["endpoint"]: e for e in baseline.get("endpoints", [])}
|
|
78
|
+
base_samples = baseline.get("samples", {})
|
|
79
|
+
rows, regressions, improvements, insufficient = [], [], [], []
|
|
80
|
+
for e in eps:
|
|
81
|
+
b = base_eps.get(e["endpoint"])
|
|
82
|
+
if not b:
|
|
83
|
+
rows.append({"endpoint": e["endpoint"], "status": "new"})
|
|
84
|
+
continue
|
|
85
|
+
cur, old = float(e[metric]), float(b[metric])
|
|
86
|
+
delta = round(cur - old, 1)
|
|
87
|
+
pct = round((cur - old) / old * 100, 1) if old else None
|
|
88
|
+
n_cur, n_base = len(res.samples.get(e["endpoint"], [])), len(base_samples.get(e["endpoint"], []))
|
|
89
|
+
row: dict[str, Any] = {"endpoint": e["endpoint"], "metric": metric, "baseline": old, "current": cur,
|
|
90
|
+
"delta_ms": delta, "delta_pct": pct, "samples": [n_base, n_cur]}
|
|
91
|
+
if n_cur < min_n or n_base < min_n:
|
|
92
|
+
row["status"] = "insufficient_samples"
|
|
93
|
+
insufficient.append(row)
|
|
94
|
+
else:
|
|
95
|
+
p_slower = mann_whitney_greater(base_samples[e["endpoint"]], res.samples[e["endpoint"]])
|
|
96
|
+
p_faster = mann_whitney_greater(res.samples[e["endpoint"]], base_samples[e["endpoint"]])
|
|
97
|
+
row["p_value"] = p_slower
|
|
98
|
+
if pct is not None and pct > tol and delta > min_ms and p_slower < alpha:
|
|
99
|
+
row["status"] = "regression"
|
|
100
|
+
regressions.append(row)
|
|
101
|
+
elif pct is not None and pct < -tol and -delta > min_ms and p_faster < alpha:
|
|
102
|
+
row["status"] = "improvement"
|
|
103
|
+
row["p_value"] = p_faster
|
|
104
|
+
improvements.append(row)
|
|
105
|
+
else:
|
|
106
|
+
row["status"] = "unchanged"
|
|
107
|
+
er_delta = round(e["error_rate"] - float(b.get("error_rate", 0)), 4)
|
|
108
|
+
row["error_rate"] = [b.get("error_rate", 0), e["error_rate"]]
|
|
109
|
+
if er_delta > float(reg.get("error_rate_delta", 0.01)) and e["errors"]:
|
|
110
|
+
row["error_rate_regression"] = er_delta
|
|
111
|
+
rows.append(row)
|
|
112
|
+
missing = sorted(set(base_eps) - {e["endpoint"] for e in eps})
|
|
113
|
+
return {"baseline_generated_at": baseline.get("generatedAt"), "baseline_engine": baseline.get("engine"),
|
|
114
|
+
"baseline_environment": baseline.get("environment"), "metric": metric, "tolerance_pct": tol,
|
|
115
|
+
"min_delta_ms": min_ms, "min_samples": min_n, "p_value": alpha, "endpoints": rows,
|
|
116
|
+
"regressions": regressions, "improvements": improvements, "insufficient_samples": insufficient,
|
|
117
|
+
"error_rate_regressions": [r for r in rows if r.get("error_rate_regression")], "missing_endpoints": missing}
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def findings(res: RunResult, eps: list[dict], slo_results: list[dict], comparison: dict | None) -> list[dict]:
|
|
121
|
+
out: list[dict] = []
|
|
122
|
+
if res.status == "blocked":
|
|
123
|
+
out.append({"type": "not_run", "message": f"Load test did not run: {res.reason}"})
|
|
124
|
+
elif res.status == "error":
|
|
125
|
+
out.append({"type": "not_run", "message": f"Load test failed to run: {res.reason}"})
|
|
126
|
+
elif res.status == "aborted":
|
|
127
|
+
out.append({"type": "aborted", "message": f"Load test aborted: {res.reason}"})
|
|
128
|
+
for e in eps:
|
|
129
|
+
if e["server_errors"]:
|
|
130
|
+
out.append({"type": "server_errors", "endpoint": e["endpoint"],
|
|
131
|
+
"message": f"{e['endpoint']}: {e['server_errors']} server error(s) (5xx) under load"})
|
|
132
|
+
for s in slo_results:
|
|
133
|
+
if not s["ok"]:
|
|
134
|
+
unit = "" if s["slo"] in ("error_rate", "min_rps") else " ms"
|
|
135
|
+
shown = f"{s['actual']:.2%}" if s["slo"] == "error_rate" else f"{s['actual']}{unit}"
|
|
136
|
+
target = f"{float(s['target']):.2%}" if s["slo"] == "error_rate" else f"{s['target']}{unit}"
|
|
137
|
+
out.append({"type": "slo_breach", "endpoint": s["endpoint"],
|
|
138
|
+
"message": f"{s['endpoint']}: {s['slo']} {shown} misses the SLO ({target})"})
|
|
139
|
+
if comparison:
|
|
140
|
+
for r in comparison["regressions"]:
|
|
141
|
+
out.append({"type": "regression", "endpoint": r["endpoint"],
|
|
142
|
+
"message": f"{r['endpoint']}: {r['metric']} {r['baseline']} → {r['current']} ms "
|
|
143
|
+
f"(+{r['delta_pct']}%, p={r['p_value']:.4f})"})
|
|
144
|
+
for r in comparison["error_rate_regressions"]:
|
|
145
|
+
out.append({"type": "error_rate_regression", "endpoint": r["endpoint"],
|
|
146
|
+
"message": f"{r['endpoint']}: error rate {r['error_rate'][0]:.2%} → {r['error_rate'][1]:.2%}"})
|
|
147
|
+
for r in comparison["insufficient_samples"]:
|
|
148
|
+
out.append({"type": "insufficient_samples", "endpoint": r["endpoint"],
|
|
149
|
+
"message": f"{r['endpoint']}: too few samples to compare ({r['samples'][0]} baseline, "
|
|
150
|
+
f"{r['samples'][1]} now; need {comparison['min_samples']})"})
|
|
151
|
+
return out
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def verdict(fs: list[dict]) -> str:
|
|
155
|
+
bad = {"not_run", "aborted", "server_errors", "slo_breach", "regression", "error_rate_regression"}
|
|
156
|
+
return "fail" if any(f["type"] in bad for f in fs) else "pass"
|