k11-performance-qa-mcp 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,10 @@
1
+ .venv/
2
+ __pycache__/
3
+ *.egg-info/
4
+ dist/
5
+ build/
6
+ .pytest_cache/
7
+ .tmp*
8
+ *.db
9
+ .qa-mcp/perf-report/
10
+ .qa-mcp/api-test-report/
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 K11 Software Solutions
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,200 @@
1
+ Metadata-Version: 2.5
2
+ Name: k11-performance-qa-mcp
3
+ Version: 0.1.0
4
+ Summary: MCP server and CI tool for load and performance testing as release evidence: API flows as load scenarios, per-endpoint latency and SLOs, statistically significant regressions against a baseline, k6 or builtin engine or imported JMeter results, with guardrails.
5
+ Project-URL: Homepage, https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp
6
+ Project-URL: Issues, https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/issues
7
+ Author: K11 Software Solutions
8
+ License-Expression: MIT
9
+ License-File: LICENSE
10
+ Keywords: claude,jmeter,k6,latency,load-testing,mcp,performance-testing,qa,regression,slo
11
+ Classifier: Development Status :: 4 - Beta
12
+ Classifier: Intended Audience :: Developers
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Software Development :: Testing
15
+ Classifier: Topic :: Software Development :: Testing :: Traffic Generation
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: k11-api-functional-qa-mcp<0.2,>=0.1.1
18
+ Provides-Extra: dev
19
+ Requires-Dist: pytest-timeout>=2; extra == 'dev'
20
+ Requires-Dist: pytest>=8; extra == 'dev'
21
+ Description-Content-Type: text/markdown
22
+
23
+ # k11 Performance QA: MCP server and CI tool
24
+
25
+ <!-- mcp-name: io.github.K11-Software-Solutions/performance-qa -->
26
+
27
+ [![PyPI](https://img.shields.io/pypi/v/k11-performance-qa-mcp?label=PyPI)](https://pypi.org/project/k11-performance-qa-mcp/)
28
+ [![CI](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml/badge.svg)](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml)
29
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
30
+
31
+ **Load testing that answers a release question: did this change make the API slower, or break it under load?**
32
+
33
+ Most load-testing tools give you numbers. This one gives you **evidence**: per-endpoint latency against your SLOs, and regressions against a baseline that are **statistically significant**, so a noisy run doesn't block a release and a real slowdown doesn't slip through. The report feeds straight into the [k11 Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp).
34
+
35
+ - **Your API tests become load scenarios.** Scenarios reuse the user-story flows from [k11 API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) (`tests/api/*.yaml`): chained calls, saved IDs, fixtures and auth. No second set of scripts to maintain.
36
+ - **Per-endpoint metrics:** p50, p90, p95, p99, max, error rate, 5xx count and throughput for every `METHOD /path/{param}`.
37
+ - **SLOs** per endpoint or pattern (`p95_ms`, `p99_ms`, `error_rate`, `min_rps`, …).
38
+ - **Regressions that mean something:** slower than the baseline by more than a tolerance **and** a minimum number of milliseconds **and** significant (one-sided Mann-Whitney U test).
39
+ - **Guardrails:** no load on production-like environments, data-changing flows only where writes are allowed, caps on virtual users, duration and request rate, and an automatic stop when too many requests fail.
40
+ - **Two engines:** [Grafana k6](https://k6.io) when it's installed, for heavy load; a builtin engine (Python, no install) for quick checks and CI.
41
+ - **Already on JMeter?** Import your existing JMeter results (`.jtl`) and get the same SLO checks, regressions and Release Gate evidence, without rewriting a single test plan.
42
+
43
+ ## Quick start
44
+
45
+ ```bash
46
+ pip install k11-performance-qa-mcp # or: uvx k11-performance-qa-mcp
47
+ k11-performance-qa-mcp init # writes .qa-mcp/perf.yaml
48
+ k11-performance-qa-mcp baseline # on main: run and save the baseline
49
+ k11-performance-qa-mcp run # on a branch: run, compare, write the report
50
+ ```
51
+
52
+ In Claude Code, as a plugin with the `/perf-check` and `/perf-baseline` commands:
53
+
54
+ ```text
55
+ /plugin marketplace add K11-Software-Solutions/k11techlab-performance-qa-mcp
56
+ /plugin install performance-qa@k11-performance-qa
57
+ ```
58
+
59
+ Or as an MCP server in any client:
60
+
61
+ ```json
62
+ { "mcpServers": { "performance-qa": { "command": "uvx", "args": ["k11-performance-qa-mcp"] } } }
63
+ ```
64
+
65
+ ## Configure
66
+
67
+ Environments and auth come from `.qa-mcp/api-test.yaml` (shared with API Functional QA). Load settings live in `.qa-mcp/perf.yaml`:
68
+
69
+ ```yaml
70
+ environment: staging
71
+ engine: auto # k6 if installed, else builtin
72
+ profile: { vus: 10, duration: 2m, ramp_up: 20s, think_time_ms: 200 }
73
+
74
+ scenarios:
75
+ browse:
76
+ flows:
77
+ - { flow: tests/api/view_order.yaml, weight: 3 } # 3x more often
78
+ - tests/api/cancel_order.yaml
79
+
80
+ slos:
81
+ "GET /orders/{id}": { p95_ms: 250, error_rate: 0.01 }
82
+ "POST /orders": { p95_ms: 600 }
83
+ "*": { p95_ms: 800, error_rate: 0.02 }
84
+
85
+ regression: { metric: p95, tolerance_pct: 10, min_delta_ms: 10, min_samples: 30, p_value: 0.01 }
86
+ guardrails: { max_vus: 50, max_duration: 10m, max_rps: 200, abort_error_rate: 0.5 }
87
+ ```
88
+
89
+ Each virtual user runs a flow's fixtures once, then repeats: pick a flow by weight, run its HTTP steps in order (IDs saved by one step feed the next), run its cleanup, think, repeat. Database checks are skipped under load; they check correctness, not speed, and still run in your functional tests.
90
+
91
+ ## Already using JMeter?
92
+
93
+ Keep your test plans. Run them as usual, then import the results file:
94
+
95
+ ```bash
96
+ jmeter -n -t checkout.jmx -l results.jtl # your existing JMeter run
97
+ k11-performance-qa-mcp import results.jtl --env staging --save-baseline # on main
98
+ k11-performance-qa-mcp import results.jtl --env staging # on a branch: compare, report
99
+ ```
100
+
101
+ CSV results (JMeter's default, with the header row) and XML results both work. Each sampler label becomes an endpoint, so name samplers like `GET /orders/{id}` to match your SLO patterns. Transaction Controller samples are skipped (they sum their children), connection failures count as errors, and 5xx responses are server errors. Since JMeter generated the load, k11's guardrails don't apply; the analysis, baseline comparison and report are exactly the same as for a k6 or builtin run.
102
+
103
+ ## How a regression is decided
104
+
105
+ For each endpoint, the run is compared with the saved baseline on the configured metric (p95 by default). It's a **regression** only when all of these hold:
106
+
107
+ 1. it's slower by more than `tolerance_pct` (default 10%);
108
+ 2. it's slower by more than `min_delta_ms` (default 10 ms), so 2 ms → 3 ms isn't "50% slower";
109
+ 3. both runs have at least `min_samples` requests for that endpoint;
110
+ 4. a one-sided Mann-Whitney U test says the new latencies are higher with p < `p_value` (default 0.01).
111
+
112
+ A rise in the error rate of more than 1 percentage point is reported separately. Improvements are reported too.
113
+
114
+ ## What the report contains
115
+
116
+ `.qa-mcp/perf-report/perf-report.json` holds the verdict, per-endpoint metrics and status codes, SLO results, the baseline comparison (with p-values), findings, error examples, a per-second timeline and the profile and engine used. `perf-report.md` is ready for a pull request or CI step summary. Findings:
117
+
118
+ | Finding | Release Gate rule | Default outcome |
119
+ |---|---|---|
120
+ | 5xx responses under load | `perf.server_errors` | block |
121
+ | An SLO missed | `perf.slo_breached` | block |
122
+ | Run aborted (too many failures) | `perf.aborted` | block |
123
+ | Significant regression vs baseline | `perf.regression` | needs a person |
124
+ | Run refused by guardrails / didn't start | `perf.not_run` | needs a person |
125
+ | No baseline, or too few samples | `perf.no_baseline`, `perf.insufficient_samples` | note |
126
+
127
+ ## Tools (MCP)
128
+
129
+ | Tool | What it does |
130
+ |---|---|
131
+ | `perf_config` | Environments (and whether load is allowed, and why), scenarios and their flows, SLOs, guardrails, the engine that will run, saved baselines |
132
+ | `run_load_test` | Run a scenario; returns the verdict, findings, per-endpoint latency and the baseline comparison; writes the report |
133
+ | `save_baseline` | Save the last completed run (or a fresh run) as the baseline |
134
+ | `import_jmeter_results` | Analyse a JMeter results file (CSV or XML) like a run: SLOs, baseline comparison, report |
135
+ | `get_report` | The last report, or one endpoint's detail: percentiles, status codes, SLOs, comparison, error examples |
136
+ | `init_perf_config` | Write a starter `.qa-mcp/perf.yaml` |
137
+
138
+ ## Command line
139
+
140
+ ```text
141
+ k11-performance-qa-mcp run [--scenario S] [--env E] [--vus N] [--duration 2m] [--ramp-up 20s] [--max-rps N]
142
+ [--engine auto|k6|builtin] [--no-compare] [--fail-on any|slo|regression|never]
143
+ k11-performance-qa-mcp baseline [same options] | baseline --from-last-run
144
+ k11-performance-qa-mcp import FILE.jtl [--scenario S] [--env E] [--save-baseline] [--no-compare] [--fail-on ...]
145
+ k11-performance-qa-mcp init
146
+ ```
147
+
148
+ In GitHub Actions it writes the Markdown report to the step summary and sets `verdict`, `p95`, `slo_breaches` and `regressions` outputs. See [docs/perf-tests.yml](docs/perf-tests.yml) for a workflow that saves the baseline on main and compares every pull request with it.
149
+
150
+ ## Safety
151
+
152
+ - **No production load:** environments named like production are refused unless they set both `allow_load: true` and `production_load_acknowledged: true` in `api-test.yaml`. Other environments need a local host, a test-like name (`staging`, `qa`, `dev`, …) or `allow_load: true`.
153
+ - **Writes follow API Functional QA's rules:** a scenario with data-changing calls is refused on an environment that doesn't allow writes, before any request is sent.
154
+ - **Caps:** `max_vus`, `max_duration` and `max_rps`; the run stops when the error rate passes `abort_error_rate`.
155
+ - **Secrets** come from `${ENV:NAME}` and never appear in reports or the generated k6 script; k6 receives auth headers through its environment.
156
+
157
+ ## Engines
158
+
159
+ | | builtin | k6 |
160
+ |---|---|---|
161
+ | Install | none | [install k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) (or set `K6_PATH`) |
162
+ | Load | up to a few hundred requests per second | thousands per second |
163
+ | Checks | status and value checks from the flow | status codes (value checks run in functional tests) |
164
+ | Fixtures and auth | yes | yes (prepared in Python, handed to k6) |
165
+ | Database cleanup steps | yes | no; use a disposable test database |
166
+
167
+ Keep the engine the same between a baseline and the runs compared with it; the report warns when they differ. JMeter imports are a third source and compare best with JMeter baselines.
168
+
169
+ ## Demo
170
+
171
+ `demo/shop` is a small order API with switches to simulate a bad release: `--slow-ms 80` slows `GET /orders/{id}`, `--error-every 5` makes every fifth call fail.
172
+
173
+ ```bash
174
+ python demo/shop/server.py --port 8080 &
175
+ cd demo/shop && SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp baseline --scenario browse
176
+ # restart the server with --slow-ms 80, then:
177
+ SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp run --scenario browse
178
+ # ❌ regression: GET /orders/{id}: p95 32.1 → 92.1 ms (+186.9%, p=0.0000)
179
+ ```
180
+
181
+ ## Part of the K11 QA suite
182
+
183
+ [Playwright QA](https://github.com/K11-Software-Solutions/k11techlab-playwright-qa-mcp) · [Selenium QA](https://github.com/K11-Software-Solutions/k11techlab-selenium-qa-mcp) · [API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp) · [API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) · [LLM Assure](https://github.com/K11-Software-Solutions/k11techlab-llm-assurance-toolkit) · **Performance QA** · [Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp) · [Release Readiness Agent](https://github.com/K11-Software-Solutions/k11techlab-release-readiness-agent)
184
+
185
+ ## Development
186
+
187
+ ```bash
188
+ python -m venv .venv && .venv/Scripts/pip install -e ".[dev]" # bin/pip on macOS/Linux
189
+ pytest # statistics, guardrails, real load against the demo shop, CLI, MCP; the k6 test runs when k6 is installed
190
+ ```
191
+
192
+ ## Roadmap
193
+
194
+ - LLM performance and cost: time to first token, tokens per second, cost per request and per conversation.
195
+ - Open-model (arrival-rate) scenarios and soak tests.
196
+ - Comparing two branches deployed side by side.
197
+
198
+ ---
199
+
200
+ Built by [K11 Software Solutions](https://k11softwaresolutions.com). MIT licensed.
@@ -0,0 +1,178 @@
1
+ # k11 Performance QA: MCP server and CI tool
2
+
3
+ <!-- mcp-name: io.github.K11-Software-Solutions/performance-qa -->
4
+
5
+ [![PyPI](https://img.shields.io/pypi/v/k11-performance-qa-mcp?label=PyPI)](https://pypi.org/project/k11-performance-qa-mcp/)
6
+ [![CI](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml/badge.svg)](https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/actions/workflows/ci.yml)
7
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue)](LICENSE)
8
+
9
+ **Load testing that answers a release question: did this change make the API slower, or break it under load?**
10
+
11
+ Most load-testing tools give you numbers. This one gives you **evidence**: per-endpoint latency against your SLOs, and regressions against a baseline that are **statistically significant**, so a noisy run doesn't block a release and a real slowdown doesn't slip through. The report feeds straight into the [k11 Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp).
12
+
13
+ - **Your API tests become load scenarios.** Scenarios reuse the user-story flows from [k11 API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) (`tests/api/*.yaml`): chained calls, saved IDs, fixtures and auth. No second set of scripts to maintain.
14
+ - **Per-endpoint metrics:** p50, p90, p95, p99, max, error rate, 5xx count and throughput for every `METHOD /path/{param}`.
15
+ - **SLOs** per endpoint or pattern (`p95_ms`, `p99_ms`, `error_rate`, `min_rps`, …).
16
+ - **Regressions that mean something:** slower than the baseline by more than a tolerance **and** a minimum number of milliseconds **and** significant (one-sided Mann-Whitney U test).
17
+ - **Guardrails:** no load on production-like environments, data-changing flows only where writes are allowed, caps on virtual users, duration and request rate, and an automatic stop when too many requests fail.
18
+ - **Two engines:** [Grafana k6](https://k6.io) when it's installed, for heavy load; a builtin engine (Python, no install) for quick checks and CI.
19
+ - **Already on JMeter?** Import your existing JMeter results (`.jtl`) and get the same SLO checks, regressions and Release Gate evidence, without rewriting a single test plan.
20
+
21
+ ## Quick start
22
+
23
+ ```bash
24
+ pip install k11-performance-qa-mcp # or: uvx k11-performance-qa-mcp
25
+ k11-performance-qa-mcp init # writes .qa-mcp/perf.yaml
26
+ k11-performance-qa-mcp baseline # on main: run and save the baseline
27
+ k11-performance-qa-mcp run # on a branch: run, compare, write the report
28
+ ```
29
+
30
+ In Claude Code, as a plugin with the `/perf-check` and `/perf-baseline` commands:
31
+
32
+ ```text
33
+ /plugin marketplace add K11-Software-Solutions/k11techlab-performance-qa-mcp
34
+ /plugin install performance-qa@k11-performance-qa
35
+ ```
36
+
37
+ Or as an MCP server in any client:
38
+
39
+ ```json
40
+ { "mcpServers": { "performance-qa": { "command": "uvx", "args": ["k11-performance-qa-mcp"] } } }
41
+ ```
42
+
43
+ ## Configure
44
+
45
+ Environments and auth come from `.qa-mcp/api-test.yaml` (shared with API Functional QA). Load settings live in `.qa-mcp/perf.yaml`:
46
+
47
+ ```yaml
48
+ environment: staging
49
+ engine: auto # k6 if installed, else builtin
50
+ profile: { vus: 10, duration: 2m, ramp_up: 20s, think_time_ms: 200 }
51
+
52
+ scenarios:
53
+ browse:
54
+ flows:
55
+ - { flow: tests/api/view_order.yaml, weight: 3 } # 3x more often
56
+ - tests/api/cancel_order.yaml
57
+
58
+ slos:
59
+ "GET /orders/{id}": { p95_ms: 250, error_rate: 0.01 }
60
+ "POST /orders": { p95_ms: 600 }
61
+ "*": { p95_ms: 800, error_rate: 0.02 }
62
+
63
+ regression: { metric: p95, tolerance_pct: 10, min_delta_ms: 10, min_samples: 30, p_value: 0.01 }
64
+ guardrails: { max_vus: 50, max_duration: 10m, max_rps: 200, abort_error_rate: 0.5 }
65
+ ```
66
+
67
+ Each virtual user runs a flow's fixtures once, then repeats: pick a flow by weight, run its HTTP steps in order (IDs saved by one step feed the next), run its cleanup, think, repeat. Database checks are skipped under load; they check correctness, not speed, and still run in your functional tests.
68
+
69
+ ## Already using JMeter?
70
+
71
+ Keep your test plans. Run them as usual, then import the results file:
72
+
73
+ ```bash
74
+ jmeter -n -t checkout.jmx -l results.jtl # your existing JMeter run
75
+ k11-performance-qa-mcp import results.jtl --env staging --save-baseline # on main
76
+ k11-performance-qa-mcp import results.jtl --env staging # on a branch: compare, report
77
+ ```
78
+
79
+ CSV results (JMeter's default, with the header row) and XML results both work. Each sampler label becomes an endpoint, so name samplers like `GET /orders/{id}` to match your SLO patterns. Transaction Controller samples are skipped (they sum their children), connection failures count as errors, and 5xx responses are server errors. Since JMeter generated the load, k11's guardrails don't apply; the analysis, baseline comparison and report are exactly the same as for a k6 or builtin run.
80
+
81
+ ## How a regression is decided
82
+
83
+ For each endpoint, the run is compared with the saved baseline on the configured metric (p95 by default). It's a **regression** only when all of these hold:
84
+
85
+ 1. it's slower by more than `tolerance_pct` (default 10%);
86
+ 2. it's slower by more than `min_delta_ms` (default 10 ms), so 2 ms → 3 ms isn't "50% slower";
87
+ 3. both runs have at least `min_samples` requests for that endpoint;
88
+ 4. a one-sided Mann-Whitney U test says the new latencies are higher with p < `p_value` (default 0.01).
89
+
90
+ A rise in the error rate of more than 1 percentage point is reported separately. Improvements are reported too.
91
+
92
+ ## What the report contains
93
+
94
+ `.qa-mcp/perf-report/perf-report.json` holds the verdict, per-endpoint metrics and status codes, SLO results, the baseline comparison (with p-values), findings, error examples, a per-second timeline and the profile and engine used. `perf-report.md` is ready for a pull request or CI step summary. Findings:
95
+
96
+ | Finding | Release Gate rule | Default outcome |
97
+ |---|---|---|
98
+ | 5xx responses under load | `perf.server_errors` | block |
99
+ | An SLO missed | `perf.slo_breached` | block |
100
+ | Run aborted (too many failures) | `perf.aborted` | block |
101
+ | Significant regression vs baseline | `perf.regression` | needs a person |
102
+ | Run refused by guardrails / didn't start | `perf.not_run` | needs a person |
103
+ | No baseline, or too few samples | `perf.no_baseline`, `perf.insufficient_samples` | note |
104
+
105
+ ## Tools (MCP)
106
+
107
+ | Tool | What it does |
108
+ |---|---|
109
+ | `perf_config` | Environments (and whether load is allowed, and why), scenarios and their flows, SLOs, guardrails, the engine that will run, saved baselines |
110
+ | `run_load_test` | Run a scenario; returns the verdict, findings, per-endpoint latency and the baseline comparison; writes the report |
111
+ | `save_baseline` | Save the last completed run (or a fresh run) as the baseline |
112
+ | `import_jmeter_results` | Analyse a JMeter results file (CSV or XML) like a run: SLOs, baseline comparison, report |
113
+ | `get_report` | The last report, or one endpoint's detail: percentiles, status codes, SLOs, comparison, error examples |
114
+ | `init_perf_config` | Write a starter `.qa-mcp/perf.yaml` |
115
+
116
+ ## Command line
117
+
118
+ ```text
119
+ k11-performance-qa-mcp run [--scenario S] [--env E] [--vus N] [--duration 2m] [--ramp-up 20s] [--max-rps N]
120
+ [--engine auto|k6|builtin] [--no-compare] [--fail-on any|slo|regression|never]
121
+ k11-performance-qa-mcp baseline [same options] | baseline --from-last-run
122
+ k11-performance-qa-mcp import FILE.jtl [--scenario S] [--env E] [--save-baseline] [--no-compare] [--fail-on ...]
123
+ k11-performance-qa-mcp init
124
+ ```
125
+
126
+ In GitHub Actions it writes the Markdown report to the step summary and sets `verdict`, `p95`, `slo_breaches` and `regressions` outputs. See [docs/perf-tests.yml](docs/perf-tests.yml) for a workflow that saves the baseline on main and compares every pull request with it.
127
+
128
+ ## Safety
129
+
130
+ - **No production load:** environments named like production are refused unless they set both `allow_load: true` and `production_load_acknowledged: true` in `api-test.yaml`. Other environments need a local host, a test-like name (`staging`, `qa`, `dev`, …) or `allow_load: true`.
131
+ - **Writes follow API Functional QA's rules:** a scenario with data-changing calls is refused on an environment that doesn't allow writes, before any request is sent.
132
+ - **Caps:** `max_vus`, `max_duration` and `max_rps`; the run stops when the error rate passes `abort_error_rate`.
133
+ - **Secrets** come from `${ENV:NAME}` and never appear in reports or the generated k6 script; k6 receives auth headers through its environment.
134
+
135
+ ## Engines
136
+
137
+ | | builtin | k6 |
138
+ |---|---|---|
139
+ | Install | none | [install k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) (or set `K6_PATH`) |
140
+ | Load | up to a few hundred requests per second | thousands per second |
141
+ | Checks | status and value checks from the flow | status codes (value checks run in functional tests) |
142
+ | Fixtures and auth | yes | yes (prepared in Python, handed to k6) |
143
+ | Database cleanup steps | yes | no; use a disposable test database |
144
+
145
+ Keep the engine the same between a baseline and the runs compared with it; the report warns when they differ. JMeter imports are a third source and compare best with JMeter baselines.
146
+
147
+ ## Demo
148
+
149
+ `demo/shop` is a small order API with switches to simulate a bad release: `--slow-ms 80` slows `GET /orders/{id}`, `--error-every 5` makes every fifth call fail.
150
+
151
+ ```bash
152
+ python demo/shop/server.py --port 8080 &
153
+ cd demo/shop && SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp baseline --scenario browse
154
+ # restart the server with --slow-ms 80, then:
155
+ SHOP_URL=http://127.0.0.1:8080 SHOP_TOKEN=test-token k11-performance-qa-mcp run --scenario browse
156
+ # ❌ regression: GET /orders/{id}: p95 32.1 → 92.1 ms (+186.9%, p=0.0000)
157
+ ```
158
+
159
+ ## Part of the K11 QA suite
160
+
161
+ [Playwright QA](https://github.com/K11-Software-Solutions/k11techlab-playwright-qa-mcp) · [Selenium QA](https://github.com/K11-Software-Solutions/k11techlab-selenium-qa-mcp) · [API Contract QA](https://github.com/K11-Software-Solutions/k11techlab-api-contract-qa-mcp) · [API Functional QA](https://github.com/K11-Software-Solutions/k11techlab-api-functional-qa-mcp) · [LLM Assure](https://github.com/K11-Software-Solutions/k11techlab-llm-assurance-toolkit) · **Performance QA** · [Release Gate](https://github.com/K11-Software-Solutions/k11techlab-release-gate-mcp) · [Release Readiness Agent](https://github.com/K11-Software-Solutions/k11techlab-release-readiness-agent)
162
+
163
+ ## Development
164
+
165
+ ```bash
166
+ python -m venv .venv && .venv/Scripts/pip install -e ".[dev]" # bin/pip on macOS/Linux
167
+ pytest # statistics, guardrails, real load against the demo shop, CLI, MCP; the k6 test runs when k6 is installed
168
+ ```
169
+
170
+ ## Roadmap
171
+
172
+ - LLM performance and cost: time to first token, tokens per second, cost per request and per conversation.
173
+ - Open-model (arrival-rate) scenarios and soak tests.
174
+ - Comparing two branches deployed side by side.
175
+
176
+ ---
177
+
178
+ Built by [K11 Software Solutions](https://k11softwaresolutions.com). MIT licensed.
@@ -0,0 +1,42 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.25"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "k11-performance-qa-mcp"
7
+ version = "0.1.0"
8
+ description = "MCP server and CI tool for load and performance testing as release evidence: API flows as load scenarios, per-endpoint latency and SLOs, statistically significant regressions against a baseline, k6 or builtin engine or imported JMeter results, with guardrails."
9
+ readme = "README.md"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "K11 Software Solutions" }]
13
+ requires-python = ">=3.10"
14
+ keywords = ["mcp", "performance-testing", "load-testing", "k6", "jmeter", "slo", "latency", "regression", "qa", "claude"]
15
+ classifiers = [
16
+ "Development Status :: 4 - Beta",
17
+ "Intended Audience :: Developers",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Software Development :: Testing",
20
+ "Topic :: Software Development :: Testing :: Traffic Generation",
21
+ ]
22
+ dependencies = ["k11-api-functional-qa-mcp>=0.1.1,<0.2"]
23
+
24
+ [project.optional-dependencies]
25
+ dev = ["pytest>=8", "pytest-timeout>=2"]
26
+
27
+ [project.urls]
28
+ Homepage = "https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp"
29
+ Issues = "https://github.com/K11-Software-Solutions/k11techlab-performance-qa-mcp/issues"
30
+
31
+ [project.scripts]
32
+ k11-performance-qa-mcp = "k11_perf_qa.cli:main"
33
+
34
+ [tool.hatch.build.targets.wheel]
35
+ packages = ["src/k11_perf_qa"]
36
+
37
+ [tool.hatch.build.targets.sdist]
38
+ include = ["src", "README.md", "LICENSE"]
39
+
40
+ [tool.pytest.ini_options]
41
+ testpaths = ["tests"]
42
+ timeout = 300
@@ -0,0 +1,3 @@
1
+ """k11 Performance QA: load and performance testing as release evidence."""
2
+
3
+ __version__ = "0.1.0"
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,156 @@
1
+ """From raw samples to evidence: per-endpoint metrics, SLO checks, baseline comparison, findings.
2
+
3
+ Findings (what the Release Gate reads):
4
+ server_errors 5xx responses under load (a bug, whatever the latency)
5
+ slo_breach an endpoint missed an SLO from perf.yaml
6
+ regression slower than the baseline beyond tolerance, min_delta_ms AND significance
7
+ error_rate_regression more failed requests than the baseline
8
+ aborted / not_run the run stopped on the error-rate guardrail, or never started
9
+ insufficient_samples too few requests to compare an endpoint with its baseline
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import fnmatch
15
+ from typing import Any
16
+
17
+ from .engine import RunResult
18
+ from .stats import mann_whitney_greater, summarize
19
+
20
+ SLO_KEYS = {"avg_ms": "avg", "p50_ms": "p50", "p90_ms": "p90", "p95_ms": "p95", "p99_ms": "p99", "max_ms": "max",
21
+ "error_rate": "error_rate", "min_rps": "rps"}
22
+
23
+
24
+ def endpoint_metrics(res: RunResult) -> list[dict]:
25
+ dur = max(res.duration_s, 0.001)
26
+ out = []
27
+ for ep in res.endpoints:
28
+ n = res.requests(ep)
29
+ errs = res.errors.get(ep, 0)
30
+ m = {"endpoint": ep, "requests": n, "errors": errs, "error_rate": round(errs / n, 4) if n else 0.0,
31
+ "server_errors": res.server_errors.get(ep, 0), "check_failures": res.check_failures.get(ep, 0),
32
+ "rps": round(n / dur, 2), "status_codes": {str(k): v for k, v in sorted(res.codes.get(ep, {}).items())},
33
+ **summarize(res.samples.get(ep, []))}
34
+ out.append(m)
35
+ return out
36
+
37
+
38
+ def overall(res: RunResult, eps: list[dict]) -> dict:
39
+ allv = [v for ep in res.samples.values() for v in ep]
40
+ n = sum(e["requests"] for e in eps)
41
+ errs = sum(e["errors"] for e in eps)
42
+ return {"requests": n, "errors": errs, "error_rate": round(errs / n, 4) if n else 0.0,
43
+ "server_errors": sum(e["server_errors"] for e in eps), "check_failures": sum(e["check_failures"] for e in eps),
44
+ "rps": round(n / max(res.duration_s, 0.001), 2), "iterations": res.iterations, **summarize(allv)}
45
+
46
+
47
+ def _slo_for(ep: str, slos: dict[str, dict]) -> tuple[str | None, dict]:
48
+ if ep in slos:
49
+ return ep, slos[ep]
50
+ for pat, spec in slos.items():
51
+ if pat != "*" and fnmatch.fnmatchcase(ep, pat):
52
+ return pat, spec
53
+ if "*" in slos:
54
+ return "*", slos["*"]
55
+ return None, {}
56
+
57
+
58
+ def check_slos(eps: list[dict], slos: dict[str, dict]) -> list[dict]:
59
+ out = []
60
+ for e in eps:
61
+ pat, spec = _slo_for(e["endpoint"], slos)
62
+ for key, target in (spec or {}).items():
63
+ if key not in SLO_KEYS:
64
+ raise ValueError(f"Unknown SLO '{key}' for {pat} (use {sorted(SLO_KEYS)})")
65
+ metric = SLO_KEYS[key]
66
+ actual = e[metric]
67
+ ok = actual >= float(target) if key == "min_rps" else actual <= float(target)
68
+ if e["requests"] == 0 and key != "error_rate":
69
+ continue
70
+ out.append({"endpoint": e["endpoint"], "slo": key, "pattern": pat, "target": target, "actual": actual, "ok": ok})
71
+ return out
72
+
73
+
74
+ def compare(res: RunResult, eps: list[dict], baseline: dict, reg: dict) -> dict:
75
+ metric = reg["metric"]
76
+ tol, min_ms, min_n, alpha = float(reg["tolerance_pct"]), float(reg["min_delta_ms"]), int(reg["min_samples"]), float(reg["p_value"])
77
+ base_eps = {e["endpoint"]: e for e in baseline.get("endpoints", [])}
78
+ base_samples = baseline.get("samples", {})
79
+ rows, regressions, improvements, insufficient = [], [], [], []
80
+ for e in eps:
81
+ b = base_eps.get(e["endpoint"])
82
+ if not b:
83
+ rows.append({"endpoint": e["endpoint"], "status": "new"})
84
+ continue
85
+ cur, old = float(e[metric]), float(b[metric])
86
+ delta = round(cur - old, 1)
87
+ pct = round((cur - old) / old * 100, 1) if old else None
88
+ n_cur, n_base = len(res.samples.get(e["endpoint"], [])), len(base_samples.get(e["endpoint"], []))
89
+ row: dict[str, Any] = {"endpoint": e["endpoint"], "metric": metric, "baseline": old, "current": cur,
90
+ "delta_ms": delta, "delta_pct": pct, "samples": [n_base, n_cur]}
91
+ if n_cur < min_n or n_base < min_n:
92
+ row["status"] = "insufficient_samples"
93
+ insufficient.append(row)
94
+ else:
95
+ p_slower = mann_whitney_greater(base_samples[e["endpoint"]], res.samples[e["endpoint"]])
96
+ p_faster = mann_whitney_greater(res.samples[e["endpoint"]], base_samples[e["endpoint"]])
97
+ row["p_value"] = p_slower
98
+ if pct is not None and pct > tol and delta > min_ms and p_slower < alpha:
99
+ row["status"] = "regression"
100
+ regressions.append(row)
101
+ elif pct is not None and pct < -tol and -delta > min_ms and p_faster < alpha:
102
+ row["status"] = "improvement"
103
+ row["p_value"] = p_faster
104
+ improvements.append(row)
105
+ else:
106
+ row["status"] = "unchanged"
107
+ er_delta = round(e["error_rate"] - float(b.get("error_rate", 0)), 4)
108
+ row["error_rate"] = [b.get("error_rate", 0), e["error_rate"]]
109
+ if er_delta > float(reg.get("error_rate_delta", 0.01)) and e["errors"]:
110
+ row["error_rate_regression"] = er_delta
111
+ rows.append(row)
112
+ missing = sorted(set(base_eps) - {e["endpoint"] for e in eps})
113
+ return {"baseline_generated_at": baseline.get("generatedAt"), "baseline_engine": baseline.get("engine"),
114
+ "baseline_environment": baseline.get("environment"), "metric": metric, "tolerance_pct": tol,
115
+ "min_delta_ms": min_ms, "min_samples": min_n, "p_value": alpha, "endpoints": rows,
116
+ "regressions": regressions, "improvements": improvements, "insufficient_samples": insufficient,
117
+ "error_rate_regressions": [r for r in rows if r.get("error_rate_regression")], "missing_endpoints": missing}
118
+
119
+
120
+ def findings(res: RunResult, eps: list[dict], slo_results: list[dict], comparison: dict | None) -> list[dict]:
121
+ out: list[dict] = []
122
+ if res.status == "blocked":
123
+ out.append({"type": "not_run", "message": f"Load test did not run: {res.reason}"})
124
+ elif res.status == "error":
125
+ out.append({"type": "not_run", "message": f"Load test failed to run: {res.reason}"})
126
+ elif res.status == "aborted":
127
+ out.append({"type": "aborted", "message": f"Load test aborted: {res.reason}"})
128
+ for e in eps:
129
+ if e["server_errors"]:
130
+ out.append({"type": "server_errors", "endpoint": e["endpoint"],
131
+ "message": f"{e['endpoint']}: {e['server_errors']} server error(s) (5xx) under load"})
132
+ for s in slo_results:
133
+ if not s["ok"]:
134
+ unit = "" if s["slo"] in ("error_rate", "min_rps") else " ms"
135
+ shown = f"{s['actual']:.2%}" if s["slo"] == "error_rate" else f"{s['actual']}{unit}"
136
+ target = f"{float(s['target']):.2%}" if s["slo"] == "error_rate" else f"{s['target']}{unit}"
137
+ out.append({"type": "slo_breach", "endpoint": s["endpoint"],
138
+ "message": f"{s['endpoint']}: {s['slo']} {shown} misses the SLO ({target})"})
139
+ if comparison:
140
+ for r in comparison["regressions"]:
141
+ out.append({"type": "regression", "endpoint": r["endpoint"],
142
+ "message": f"{r['endpoint']}: {r['metric']} {r['baseline']} → {r['current']} ms "
143
+ f"(+{r['delta_pct']}%, p={r['p_value']:.4f})"})
144
+ for r in comparison["error_rate_regressions"]:
145
+ out.append({"type": "error_rate_regression", "endpoint": r["endpoint"],
146
+ "message": f"{r['endpoint']}: error rate {r['error_rate'][0]:.2%} → {r['error_rate'][1]:.2%}"})
147
+ for r in comparison["insufficient_samples"]:
148
+ out.append({"type": "insufficient_samples", "endpoint": r["endpoint"],
149
+ "message": f"{r['endpoint']}: too few samples to compare ({r['samples'][0]} baseline, "
150
+ f"{r['samples'][1]} now; need {comparison['min_samples']})"})
151
+ return out
152
+
153
+
154
+ def verdict(fs: list[dict]) -> str:
155
+ bad = {"not_run", "aborted", "server_errors", "slo_breach", "regression", "error_rate_regression"}
156
+ return "fail" if any(f["type"] in bad for f in fs) else "pass"