perf-skills 1.0.3 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/.claude-plugin/plugin.json +1 -1
- package/README.md +2 -1
- package/package.json +2 -2
- package/skills/perf/SKILL.md +228 -72
- package/skills/perf/evals/evals.json +149 -0
- package/skills/perf/references/tools/gatling.md +286 -286
- package/skills/perf/references/tools/jmeter.md +251 -251
- package/skills/perf/references/tools/k6.md +269 -269
- package/skills/perf/references/tools/locust.md +133 -133
- package/skills/perf/references/topics/correlation.md +406 -0
- package/skills/perf/references/topics/database-testing.md +8 -8
- package/skills/perf/references/topics/modern-architectures.md +15 -15
- package/skills/perf/references/topics/observability.md +3 -3
- package/skills/perf/references/topics/production-testing.md +13 -13
- package/skills/perf/references/topics/protocol-testing.md +15 -15
- package/skills/perf/references/topics/results-analysis.md +13 -13
- package/skills/perf/references/topics/script-generation.md +9 -9
- package/skills/perf/references/topics/test-data.md +6 -6
- package/skills/perf/references/topics/test-execution.md +5 -5
- package/skills/perf/references/topics/workload-design.md +3 -3
|
@@ -5,15 +5,15 @@
|
|
|
5
5
|
"email": "contact@qainsights.com"
|
|
6
6
|
},
|
|
7
7
|
"metadata": {
|
|
8
|
-
"description": "Expert AI skill for performance testing
|
|
9
|
-
"version": "1.0.
|
|
8
|
+
"description": "Expert AI skill for performance testing - covers the full lifecycle using JMeter, k6, Gatling, Locust, NeoLoad, LoadRunner, and OctoPerf.",
|
|
9
|
+
"version": "1.0.4"
|
|
10
10
|
},
|
|
11
11
|
"plugins": [
|
|
12
12
|
{
|
|
13
13
|
"name": "perf",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Expert guidance for performance testing: planning, scripting, executing, and analyzing load/stress/soak/spike tests across 7 tools and 10 topic areas.",
|
|
16
|
-
"version": "1.0.
|
|
16
|
+
"version": "1.0.4",
|
|
17
17
|
"author": {
|
|
18
18
|
"name": "NaveenKumar Namachivayam",
|
|
19
19
|
"email": "contact@qainsights.com"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "perf",
|
|
3
3
|
"description": "Expert guidance for performance testing: planning, scripting, executing, and analyzing load/stress/soak/spike tests using JMeter, k6, Gatling, Locust, NeoLoad, LoadRunner, OctoPerf.",
|
|
4
|
-
"version": "1.0.
|
|
4
|
+
"version": "1.0.4",
|
|
5
5
|
"skills": "./skills/"
|
|
6
6
|
}
|
package/README.md
CHANGED
|
@@ -180,7 +180,8 @@ perf-skills/ # Repository root
|
|
|
180
180
|
└── topics/ # Cross-cutting concepts (tool-agnostic)
|
|
181
181
|
├── workload-design.md # Concurrency models, load profiles, Little's Law
|
|
182
182
|
├── test-data.md # CSV, DB seeding, Faker, data isolation patterns
|
|
183
|
-
├── script-generation.md #
|
|
183
|
+
├── script-generation.md # Assertions, error handling, naming, structure
|
|
184
|
+
├── correlation.md # Dynamic values, extractors, regex, tool mappings
|
|
184
185
|
├── test-execution.md # Distributed, CI/CD (GitHub Actions, GitLab, Jenkins)
|
|
185
186
|
├── results-analysis.md # Percentiles, bottleneck framework, reporting
|
|
186
187
|
├── observability.md # APM, Prometheus, Grafana, tracing, JVM metrics
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "perf-skills",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "Expert AI skill for performance testing
|
|
3
|
+
"version": "2.0.0",
|
|
4
|
+
"description": "Expert AI skill for performance testing - covers JMeter, k6, Gatling, Locust, NeoLoad, LoadRunner, OctoPerf across the full lifecycle",
|
|
5
5
|
"main": "skills/perf/SKILL.md",
|
|
6
6
|
"files": [
|
|
7
7
|
"skills/",
|
package/skills/perf/SKILL.md
CHANGED
|
@@ -1,63 +1,81 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: perf
|
|
3
|
-
description:
|
|
3
|
+
description: Performance testing expert covering the full lifecycle for
|
|
4
|
+
JMeter, k6, Gatling, Locust, NeoLoad, and LoadRunner. Use this skill
|
|
5
|
+
whenever writing or reviewing load test scripts, setting thresholds,
|
|
6
|
+
choosing executors, configuring CI/CD pipelines, diagnosing latency
|
|
7
|
+
issues, designing workloads, analyzing results, or recommending tools
|
|
8
|
+
- even if the tool is not named explicitly. Always consult before
|
|
9
|
+
suggesting thresholds, executor types, or output configuration.
|
|
10
|
+
Prefer this skill over general knowledge for any performance testing
|
|
11
|
+
decision, debugging session, or tool comparison.
|
|
4
12
|
---
|
|
5
13
|
|
|
6
14
|
# Performance Testing Skill
|
|
7
15
|
|
|
8
|
-
This skill provides expert, opinionated guidance across the full
|
|
16
|
+
This skill provides expert, opinionated guidance across the full
|
|
17
|
+
performance testing lifecycle - from workload design through production
|
|
18
|
+
observation. It covers both commercial tools (LoadRunner, NeoLoad,
|
|
19
|
+
OctoPerf) and open-source tools (JMeter, k6, Gatling, Locust).
|
|
9
20
|
|
|
10
21
|
---
|
|
11
22
|
|
|
12
23
|
## How to Use This Skill
|
|
13
24
|
|
|
14
|
-
Read the relevant reference files based on what the user needs.
|
|
25
|
+
Read the relevant reference files based on what the user needs.
|
|
26
|
+
Multiple files may apply.
|
|
15
27
|
|
|
16
28
|
### Loading Priority Rules
|
|
17
29
|
|
|
18
30
|
1. **Tool-specific syntax/config** → load the tool file only.
|
|
19
|
-
2. **Strategy/concepts** (workload design, test data, analysis) → load
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
31
|
+
2. **Strategy/concepts** (workload design, test data, analysis) → load
|
|
32
|
+
the topic file only.
|
|
33
|
+
3. **Both apply** (e.g., "JMeter CI/CD") → load the topic file first
|
|
34
|
+
for patterns, then the tool file for syntax.
|
|
35
|
+
4. **Never load all files at once** - select the 1–2 most relevant.
|
|
36
|
+
5. **Cross-cutting principles** (assertions, think time,
|
|
37
|
+
parameterization) → this file's Key Principles section is the
|
|
38
|
+
single source of truth.
|
|
23
39
|
|
|
24
40
|
### Reference Map
|
|
25
41
|
|
|
26
|
-
| User needs help with...
|
|
27
|
-
|
|
28
|
-
| Choosing the right tool
|
|
29
|
-
| JMeter scripts, plugins, config
|
|
30
|
-
| k6 scripting, extensions, cloud
|
|
31
|
-
| Gatling simulations, Scala/Java DSL
|
|
32
|
-
| Locust Python tests, distributed
|
|
33
|
-
| NeoLoad projects, GUI, APIs
|
|
34
|
-
| LoadRunner scripts, protocols, VuGen
|
|
35
|
-
| OctoPerf cloud test management
|
|
36
|
-
| Designing workloads, concurrency, pacing
|
|
37
|
-
| Test data, parameterization, CSV feeds
|
|
38
|
-
| Script patterns,
|
|
39
|
-
|
|
|
40
|
-
|
|
|
41
|
-
|
|
|
42
|
-
|
|
|
43
|
-
|
|
|
44
|
-
|
|
|
45
|
-
|
|
|
42
|
+
| User needs help with... | Read this file |
|
|
43
|
+
|--------------------------------------------------|---------------------------------------------------|
|
|
44
|
+
| Choosing the right tool | This file - see Tool Selection Matrix below |
|
|
45
|
+
| JMeter scripts, plugins, config | `references/tools/jmeter.md` |
|
|
46
|
+
| k6 scripting, extensions, cloud | `references/tools/k6.md` |
|
|
47
|
+
| Gatling simulations, Scala/Java DSL | `references/tools/gatling.md` |
|
|
48
|
+
| Locust Python tests, distributed | `references/tools/locust.md` |
|
|
49
|
+
| NeoLoad projects, GUI, APIs | `references/tools/neoload.md` |
|
|
50
|
+
| LoadRunner scripts, protocols, VuGen | `references/tools/loadrunner.md` |
|
|
51
|
+
| OctoPerf cloud test management | `references/tools/octoperf.md` |
|
|
52
|
+
| Designing workloads, concurrency, pacing | `references/topics/workload-design.md` |
|
|
53
|
+
| Test data, parameterization, CSV feeds | `references/topics/test-data.md` |
|
|
54
|
+
| Script patterns, best practices | `references/topics/script-generation.md` |
|
|
55
|
+
| Correlation, extractors, dynamic values | `references/topics/correlation.md` |
|
|
56
|
+
| CI/CD, distributed execution, cloud runners | `references/topics/test-execution.md` |
|
|
57
|
+
| Analyzing results, percentiles, SLAs | `references/topics/results-analysis.md` |
|
|
58
|
+
| APM, metrics, tracing, dashboards | `references/topics/observability.md` |
|
|
59
|
+
| Staging vs production testing strategies | `references/topics/production-testing.md` |
|
|
60
|
+
| gRPC, GraphQL, WebSocket, messaging protocols | `references/topics/protocol-testing.md` |
|
|
61
|
+
| Database load testing (JDBC, connection pools) | `references/topics/database-testing.md` |
|
|
62
|
+
| Microservices, K8s, serverless performance | `references/topics/modern-architectures.md` |
|
|
46
63
|
|
|
47
64
|
### Protocol Routing Table
|
|
48
65
|
|
|
49
|
-
When the user's question is protocol-specific, use this to select the
|
|
66
|
+
When the user's question is protocol-specific, use this to select the
|
|
67
|
+
right tool and reference:
|
|
50
68
|
|
|
51
|
-
| Protocol
|
|
52
|
-
|
|
53
|
-
| HTTP / REST
|
|
54
|
-
| gRPC
|
|
55
|
-
| GraphQL
|
|
56
|
-
| WebSocket / SSE
|
|
57
|
-
| JDBC / Database
|
|
58
|
-
| Kafka / Message Queues | k6 (xk6-kafka), JMeter
|
|
59
|
-
| SOAP / WSDL
|
|
60
|
-
| SAP / Citrix
|
|
69
|
+
| Protocol | Recommended Tools | Reference |
|
|
70
|
+
|------------------------|------------------------------|---------------------------------------------------------------|
|
|
71
|
+
| HTTP / REST | k6, Gatling, JMeter | Tool file |
|
|
72
|
+
| gRPC | k6, Gatling, JMeter (plugin) | `references/topics/protocol-testing.md` + tool file |
|
|
73
|
+
| GraphQL | k6, Gatling | `references/topics/protocol-testing.md` + tool file |
|
|
74
|
+
| WebSocket / SSE | Gatling, k6 | `references/topics/protocol-testing.md` + tool file |
|
|
75
|
+
| JDBC / Database | JMeter | `references/topics/database-testing.md` + `references/tools/jmeter.md` |
|
|
76
|
+
| Kafka / Message Queues | k6 (xk6-kafka), JMeter | `references/topics/protocol-testing.md` |
|
|
77
|
+
| SOAP / WSDL | LoadRunner, JMeter | Tool file |
|
|
78
|
+
| SAP / Citrix | LoadRunner, NeoLoad | Tool file |
|
|
61
79
|
|
|
62
80
|
---
|
|
63
81
|
|
|
@@ -65,45 +83,173 @@ When the user's question is protocol-specific, use this to select the right tool
|
|
|
65
83
|
|
|
66
84
|
Use this to recommend the right tool when the user hasn't decided yet.
|
|
67
85
|
|
|
68
|
-
| Criteria
|
|
69
|
-
|
|
70
|
-
| **Language**
|
|
71
|
-
| **Open source**
|
|
86
|
+
| Criteria | JMeter | k6 | Gatling | Locust | NeoLoad | LoadRunner | OctoPerf |
|
|
87
|
+
|----------------------|---------------------|-----------------------|----------------------|----------------|------------------|-------------------------|-----------------------|
|
|
88
|
+
| **Language** | GUI/XML + Groovy | JavaScript/TypeScript | Scala/Java | Python | GUI + NeoLoad DSL| VuGen C-like | Web UI (JMeter-based) |
|
|
89
|
+
| **Open source** | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ (SaaS) |
|
|
72
90
|
| **Protocol support** | HTTP, JDBC, JMS, MQTT, FTP, gRPC | HTTP, gRPC, WS | HTTP, JMS, gRPC | HTTP, gRPC | HTTP, gRPC, WS, SAP | HTTP, Citrix, SAP, Flex | HTTP (JMeter-backed) |
|
|
73
|
-
| **Developer-friendly** | Medium
|
|
74
|
-
| **Enterprise support** | Community + BlazeMeter | Grafana Cloud
|
|
75
|
-
| **CI/CD integration** | Good (Maven/Gradle) | Excellent
|
|
76
|
-
| **Cloud execution**
|
|
77
|
-
| **Best for**
|
|
91
|
+
| **Developer-friendly** | Medium | High | High | High | Low | Low | Medium |
|
|
92
|
+
| **Enterprise support** | Community + BlazeMeter | Grafana Cloud | Gatling Enterprise | Limited | ✅ | ✅ | ✅ |
|
|
93
|
+
| **CI/CD integration** | Good (Maven/Gradle) | Excellent | Excellent | Good | Good | Moderate | Good |
|
|
94
|
+
| **Cloud execution** | BlazeMeter, OctoPerf | Grafana Cloud | Gatling Enterprise | Self-managed | NeoLoad Cloud | AWS/on-prem | OctoPerf Cloud |
|
|
95
|
+
| **Best for** | Legacy systems, JDBC, protocols | Modern APIs, TypeScript devs | High-throughput HTTP | Python teams, flexible | SAP/Citrix enterprise | Mainframe, legacy enterprise | JMeter teams needing cloud UI |
|
|
78
96
|
|
|
79
97
|
### Quick decision rules
|
|
98
|
+
|
|
80
99
|
- **Team writes code** → k6 or Gatling
|
|
81
100
|
- **Team uses GUI** → JMeter or NeoLoad
|
|
82
101
|
- **Python shop** → Locust
|
|
83
102
|
- **SAP / mainframe / Citrix** → LoadRunner or NeoLoad
|
|
84
|
-
- **Need cloud SaaS with minimal setup** → OctoPerf (JMeter) or
|
|
103
|
+
- **Need cloud SaaS with minimal setup** → OctoPerf (JMeter) or
|
|
104
|
+
Grafana Cloud (k6)
|
|
85
105
|
- **Free + protocol variety** → JMeter
|
|
86
|
-
- **Correlation needed for session-heavy flows** → JMeter (with
|
|
106
|
+
- **Correlation needed for session-heavy flows** → JMeter (with
|
|
107
|
+
Correlation Recorder) or LoadRunner
|
|
87
108
|
- **gRPC or GraphQL APIs** → k6 or Gatling
|
|
88
109
|
- **Message queues (Kafka, RabbitMQ)** → k6 (xk6-kafka) or JMeter
|
|
89
110
|
|
|
90
111
|
---
|
|
91
112
|
|
|
113
|
+
## Common Mistakes by Tool
|
|
114
|
+
|
|
115
|
+
These are the mistakes that cause silent CI failures, misleading
|
|
116
|
+
results, or test collapse at scale. Flag them proactively whenever
|
|
117
|
+
reviewing scripts or diagnosing problems - users often don't know to
|
|
118
|
+
ask about them.
|
|
119
|
+
|
|
120
|
+
### k6
|
|
121
|
+
|
|
122
|
+
- **`check()` without `thresholds`** - checks log pass/fail but do
|
|
123
|
+
NOT fail the test run. Without thresholds, CI always reports green
|
|
124
|
+
regardless of latency. Always add `thresholds` to `options`.
|
|
125
|
+
- **Data loaded inside `default()`** - loading CSV or JSON inside the
|
|
126
|
+
VU function runs on every iteration, causing massive per-iteration
|
|
127
|
+
overhead and OOM at scale. Always use `SharedArray` in the init
|
|
128
|
+
scope.
|
|
129
|
+
- **`shared-iterations` for user journeys** - VUs race to claim
|
|
130
|
+
iterations and may skip steps, producing incomplete journey metrics.
|
|
131
|
+
Use `per-vu-iterations` for any multi-step flow.
|
|
132
|
+
- **No `sleep()` between steps** - 100 VUs with zero think time
|
|
133
|
+
generates the absolute maximum RPS for that iteration time, far
|
|
134
|
+
exceeding what 100 real concurrent users produce. Always add
|
|
135
|
+
realistic think time.
|
|
136
|
+
- **`console.log` in `default()`** - causes 30–50% throughput drop
|
|
137
|
+
under load. Use custom metrics (`Counter`, `Trend`) instead.
|
|
138
|
+
- **Hardcoded `BASE_URL`** - use `__ENV.BASE_URL` for environment
|
|
139
|
+
portability.
|
|
140
|
+
|
|
141
|
+
### JMeter
|
|
142
|
+
|
|
143
|
+
- **Listeners enabled in non-GUI runs** - View Results Tree, Aggregate
|
|
144
|
+
Report etc. buffer all samples in memory during headless runs,
|
|
145
|
+
causing memory leak and eventual crash. Disable all listeners before
|
|
146
|
+
CI execution; use `-l results.jtl` for output.
|
|
147
|
+
- **Missing correlation on session-heavy apps** - JSESSIONID, CSRF
|
|
148
|
+
tokens, ViewState, OAuth codes must be extracted and reused. Without
|
|
149
|
+
correlation, the test fails for every user after the first.
|
|
150
|
+
- **Hardcoded thread counts** - parameterize via JMeter properties
|
|
151
|
+
(`${__P(threads,10)}`) so CI can override without editing the JMX.
|
|
152
|
+
- **Zero think time** - never reflects real user behavior; always add
|
|
153
|
+
at minimum a Constant Timer (300–500ms) between samplers.
|
|
154
|
+
- **Throughput Controller % mode misuse** - percentage applies per
|
|
155
|
+
iteration of the parent controller, not globally. Most users expect
|
|
156
|
+
global percentage; use `Total Executions` mode instead or be
|
|
157
|
+
explicit.
|
|
158
|
+
|
|
159
|
+
### Gatling
|
|
160
|
+
|
|
161
|
+
- **Blocking calls inside `exec()`** - any blocking I/O inside an exec
|
|
162
|
+
block stalls the entire Akka actor, killing simulation concurrency.
|
|
163
|
+
Use Gatling's async feed/session API exclusively.
|
|
164
|
+
- **Missing `.check()` on responses** - without checks, 4xx and 5xx
|
|
165
|
+
responses are silently counted as successful. Always add at minimum
|
|
166
|
+
`.check(status.is(200))`.
|
|
167
|
+
- **Fixed `pause()` values** - use `uniformPaused(min, max)` or
|
|
168
|
+
`normalPausedWithPercentageDuration` for realistic think time
|
|
169
|
+
distribution.
|
|
170
|
+
|
|
171
|
+
### Locust
|
|
172
|
+
|
|
173
|
+
- **`self.client` without `catch_response=True`** - by default, Locust
|
|
174
|
+
marks any HTTP response as success regardless of status code. Use
|
|
175
|
+
`with self.client.get(..., catch_response=True) as r:` and call
|
|
176
|
+
`r.failure()` explicitly.
|
|
177
|
+
- **Unequal task weights without intent** - tasks default to equal
|
|
178
|
+
weight; if your user journey has unequal step frequency, set weights
|
|
179
|
+
explicitly or the journey ratio will be wrong.
|
|
180
|
+
- **Master + workers on same machine** - causes resource contention
|
|
181
|
+
that skews both throughput and latency measurements. Always run
|
|
182
|
+
workers on separate machines or containers for distributed tests.
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## Cross-Tool Concept Mapping
|
|
187
|
+
|
|
188
|
+
Use this when users are migrating between tools or asking how a
|
|
189
|
+
concept from one tool maps to another. Claude should always provide
|
|
190
|
+
the specific mapping rather than a generic explanation.
|
|
191
|
+
|
|
192
|
+
| Concept | JMeter | k6 | Gatling | Locust | LoadRunner |
|
|
193
|
+
|------------------|-------------------------|-------------------------|--------------------------|--------------------------|---------------------|
|
|
194
|
+
| Virtual user | Thread | VU | User | User | Vuser |
|
|
195
|
+
| Test plan | .jmx file | .js / .ts script | Simulation class | .py file | VuGen script (.usr) |
|
|
196
|
+
| User entrypoint | Thread Group | `default()` function | `scenario()` | task methods | `Action()` |
|
|
197
|
+
| Concurrency ctrl | Thread Group settings | executor | `inject()` | `spawn_rate` | Vuser Group |
|
|
198
|
+
| Think time | Constant/Uniform Timer | `sleep()` | `pause()` | `time.sleep()` | `lr_think_time()` |
|
|
199
|
+
| Inline assertion | Response Assertion | `check()` | `.check()` | `catch_response` | `lr_eval_string()` |
|
|
200
|
+
| SLA enforcement | Duration Assertion | `thresholds` | Assertions (Enterprise) | custom + exit code | SLA definition |
|
|
201
|
+
| Correlation | Regex / CSS Extractor | `res.json()` / regex | `.check()` + `saveAs()` | `response.text` + regex | `web_reg_save_param`|
|
|
202
|
+
| Data feed | CSV Data Set Config | `SharedArray` | `feeder` | CSV reader | `lr_paramarr()` |
|
|
203
|
+
| Grouping | Transaction Controller | `group()` | `group()` | task sets | Transaction |
|
|
204
|
+
| Distributed | Controller + Agents | k6 cloud / k6 operator | Gatling Enterprise | master + workers | Load Generator |
|
|
205
|
+
| Results output | .jtl (CSV/XML) | JSON / InfluxDB / cloud | simulation.log | CSV / Locust web UI | .lrr file |
|
|
206
|
+
|
|
207
|
+
---
|
|
208
|
+
|
|
209
|
+
## Threshold Starting Points
|
|
210
|
+
|
|
211
|
+
These are community baselines - always tell the user to adjust these
|
|
212
|
+
to their actual SLA requirements. Never present them as universal
|
|
213
|
+
targets.
|
|
214
|
+
|
|
215
|
+
| Endpoint type | p95 | p99 | Error rate | Notes |
|
|
216
|
+
|------------------|-----------|------------|------------|------------------------------------|
|
|
217
|
+
| Web page (HTML) | < 3000ms | < 5000ms | < 1% | Aligns with Google CWV LCP < 2.5s |
|
|
218
|
+
| REST API | < 500ms | < 1000ms | < 1% | Common industry baseline |
|
|
219
|
+
| Auth / Login | < 300ms | < 500ms | < 0.1% | Stricter - security-sensitive path |
|
|
220
|
+
| Search / Query | < 800ms | < 1500ms | < 0.5% | Varies heavily by dataset size |
|
|
221
|
+
| Write (POST/PUT) | < 800ms | < 1500ms | < 0.5% | Includes DB write latency |
|
|
222
|
+
| Checkout/Payment | < 1000ms | < 2000ms | < 0.1% | Stricter - revenue-critical path |
|
|
223
|
+
| Background/async | < 5000ms | < 10000ms | < 2% | Batch jobs, async processors |
|
|
224
|
+
|
|
225
|
+
**k6 specific:** Always define both `http_req_duration` AND
|
|
226
|
+
`http_req_waiting` as separate thresholds. `http_req_waiting`
|
|
227
|
+
(TTFB) isolates server-side latency from network overhead - it is
|
|
228
|
+
the first metric to check when diagnosing high p95. Always include
|
|
229
|
+
`checks: ['rate>0.99']`.
|
|
230
|
+
|
|
231
|
+
**JMeter specific:** Add both a Response Assertion and a Duration
|
|
232
|
+
Assertion per sampler. Never rely on listener output alone for
|
|
233
|
+
CI pass/fail; use the JMeter exit code driven by assertions.
|
|
234
|
+
|
|
235
|
+
---
|
|
236
|
+
|
|
92
237
|
## Performance Testing Lifecycle Overview
|
|
93
238
|
|
|
94
|
-
Always think through these phases when helping a user
|
|
239
|
+
Always think through these phases when helping a user - they often ask
|
|
240
|
+
about one phase but need context from others.
|
|
95
241
|
|
|
96
242
|
```
|
|
97
243
|
1. PLAN
|
|
98
|
-
└─ Workload design → concurrency model → SLA targets → test type
|
|
244
|
+
└─ Workload design → concurrency model → SLA targets → test type
|
|
99
245
|
→ references/topics/workload-design.md
|
|
100
246
|
|
|
101
247
|
2. DATA
|
|
102
|
-
└─ Identify variables → parameterization strategy →
|
|
248
|
+
└─ Identify variables → parameterization strategy → data generation
|
|
103
249
|
→ references/topics/test-data.md
|
|
104
250
|
|
|
105
251
|
3. SCRIPT
|
|
106
|
-
└─ Record or code → correlation → parameterization → assertions
|
|
252
|
+
└─ Record or code → correlation → parameterization → assertions
|
|
107
253
|
→ references/topics/script-generation.md + tool-specific file
|
|
108
254
|
|
|
109
255
|
4. EXECUTE
|
|
@@ -127,38 +273,48 @@ Always think through these phases when helping a user — they often ask about o
|
|
|
127
273
|
|
|
128
274
|
## Common Performance Test Types
|
|
129
275
|
|
|
130
|
-
| Test Type
|
|
131
|
-
|
|
132
|
-
| **Load**
|
|
133
|
-
| **Stress**
|
|
134
|
-
| **Soak/Endurance** | Detect memory leaks, slow degradation | Resource trend over time (hours)
|
|
135
|
-
| **Spike**
|
|
136
|
-
| **Capacity**
|
|
137
|
-
| **Smoke**
|
|
138
|
-
| **Breakpoint**
|
|
276
|
+
| Test Type | Goal | Key Metric |
|
|
277
|
+
|------------------|---------------------------------------|-----------------------------------------|
|
|
278
|
+
| **Load** | Validate system at expected load | Response time, throughput, error rate |
|
|
279
|
+
| **Stress** | Find the breaking point | Max VUs before degradation, error onset |
|
|
280
|
+
| **Soak/Endurance** | Detect memory leaks, slow degradation | Resource trend over time (hours) |
|
|
281
|
+
| **Spike** | Behavior under sudden traffic burst | Recovery time, error spike |
|
|
282
|
+
| **Capacity** | Find max sustainable load | Throughput ceiling at SLA thresholds |
|
|
283
|
+
| **Smoke** | Quick sanity check | Single VU - no errors |
|
|
284
|
+
| **Breakpoint** | Incremental ramp until failure | Failure threshold VU count |
|
|
139
285
|
|
|
140
286
|
---
|
|
141
287
|
|
|
142
288
|
## Key Principles to Always Apply
|
|
143
289
|
|
|
144
|
-
1. **Never test against production blindly**
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
290
|
+
1. **Never test against production blindly** - always have a rollback
|
|
291
|
+
plan and alerting in place.
|
|
292
|
+
2. **Baseline first** - always establish a baseline before stress or
|
|
293
|
+
soak runs.
|
|
294
|
+
3. **Think time and pacing matter** - unrealistic zero-think-time tests
|
|
295
|
+
produce misleading results.
|
|
296
|
+
4. **Parameterize everything** - hardcoded credentials, tokens, and IDs
|
|
297
|
+
will fail at scale.
|
|
298
|
+
5. **Assertions are not optional** - tests without assertions are just
|
|
299
|
+
generating traffic, not validating behavior.
|
|
300
|
+
6. **Isolate the system under test** - shared environments invalidate
|
|
301
|
+
results.
|
|
302
|
+
7. **Correlate dynamic values** - session tokens, CSRF, ViewState etc.
|
|
303
|
+
must be extracted and reused.
|
|
151
304
|
|
|
152
305
|
---
|
|
153
306
|
|
|
154
307
|
## Asking the Right Questions
|
|
155
308
|
|
|
156
|
-
When a user brings a performance problem, ask (or infer) these before
|
|
309
|
+
When a user brings a performance problem, ask (or infer) these before
|
|
310
|
+
prescribing a solution:
|
|
157
311
|
|
|
158
312
|
- What is the **target concurrency** (VUs or RPS)?
|
|
159
313
|
- What is the **SLA** (e.g., p95 < 500ms, error rate < 1%)?
|
|
160
314
|
- What is the **protocol** (HTTP/REST, gRPC, JDBC, WebSocket)?
|
|
161
|
-
- Is the app **stateful** (session-based) or **stateless**
|
|
315
|
+
- Is the app **stateful** (session-based) or **stateless**
|
|
316
|
+
(token-based)?
|
|
162
317
|
- Where will tests **run from** (local, CI, cloud)?
|
|
163
318
|
- What **environment** is being tested (dev, staging, prod)?
|
|
164
|
-
- Is there an **APM tool** in place (Datadog, Dynatrace, Grafana,
|
|
319
|
+
- Is there an **APM tool** in place (Datadog, Dynatrace, Grafana,
|
|
320
|
+
New Relic)?
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
{
|
|
2
|
+
"skill_name": "perf",
|
|
3
|
+
"evals": [
|
|
4
|
+
{
|
|
5
|
+
"id": 1,
|
|
6
|
+
"prompt": "I'm using k6 checks to validate all my API responses. My CI pipeline shows all tests passing but users are complaining about slowness. What am I missing?",
|
|
7
|
+
"expected_output": "Identifies that checks do not fail the test run - thresholds are required for CI enforcement. Provides a concrete threshold example.",
|
|
8
|
+
"files": [],
|
|
9
|
+
"assertions": [
|
|
10
|
+
{
|
|
11
|
+
"id": "checks-vs-thresholds-distinction",
|
|
12
|
+
"text": "Explicitly states that checks do NOT produce a non-zero exit code and cannot fail a CI pipeline"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "thresholds-required",
|
|
16
|
+
"text": "States that thresholds in options.thresholds are required to enforce SLA in CI"
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "concrete-example",
|
|
20
|
+
"text": "Provides a concrete threshold code example, not just an explanation"
|
|
21
|
+
}
|
|
22
|
+
]
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": 2,
|
|
26
|
+
"prompt": "I need to load test a Kafka consumer that processes order events. What tool should I use and how do I get started?",
|
|
27
|
+
"expected_output": "Recommends xk6-kafka with the custom build command, or JMeter with Kafka plugin. Does not suggest a generic HTTP tool.",
|
|
28
|
+
"files": [],
|
|
29
|
+
"assertions": [
|
|
30
|
+
{
|
|
31
|
+
"id": "correct-tool",
|
|
32
|
+
"text": "Recommends xk6-kafka or JMeter Kafka plugin - not a generic HTTP tool like plain k6 or Gatling HTTP"
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "xk6-build-command",
|
|
36
|
+
"text": "Includes the xk6 custom build command or JMeter plugin installation reference"
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "not-http",
|
|
40
|
+
"text": "Does not suggest testing Kafka via HTTP endpoints as a primary approach"
|
|
41
|
+
}
|
|
42
|
+
]
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": 3,
|
|
46
|
+
"prompt": "My JMeter test runs fine locally but JMeter crashes after 20 minutes in CI with an OutOfMemoryError. The test plan works on my machine with 50 threads.",
|
|
47
|
+
"expected_output": "Identifies listeners enabled in non-GUI run as the primary cause. Recommends disabling listeners and using -l flag for output.",
|
|
48
|
+
"files": [],
|
|
49
|
+
"assertions": [
|
|
50
|
+
{
|
|
51
|
+
"id": "listener-root-cause",
|
|
52
|
+
"text": "Identifies enabled listeners (View Results Tree, Aggregate Report etc.) in non-GUI runs as the memory leak cause"
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "disable-listeners",
|
|
56
|
+
"text": "Recommends disabling listeners in the test plan for CI runs"
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"id": "jtl-output",
|
|
60
|
+
"text": "Mentions using -l results.jtl flag as the correct output method for non-GUI runs"
|
|
61
|
+
}
|
|
62
|
+
]
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": 4,
|
|
66
|
+
"prompt": "We're migrating our load tests from LoadRunner to k6. What does a VuGen Action map to in k6, and what about think time and correlation?",
|
|
67
|
+
"expected_output": "Maps VuGen Action → k6 default() function. Maps lr_think_time() → sleep(). Maps web_reg_save_param → res.json() or regex extraction.",
|
|
68
|
+
"files": [],
|
|
69
|
+
"assertions": [
|
|
70
|
+
{
|
|
71
|
+
"id": "action-mapping",
|
|
72
|
+
"text": "Maps VuGen Action() specifically to k6 default() function"
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "think-time-mapping",
|
|
76
|
+
"text": "Maps lr_think_time() to k6 sleep()"
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
"id": "correlation-mapping",
|
|
80
|
+
"text": "Maps web_reg_save_param or LoadRunner correlation to k6 res.json() or regex extraction"
|
|
81
|
+
}
|
|
82
|
+
]
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": 5,
|
|
86
|
+
"prompt": "I'm writing a k6 script for an e-commerce checkout flow: login → browse → add to cart → checkout. Should I use shared-iterations or per-vu-iterations?",
|
|
87
|
+
"expected_output": "Recommends per-vu-iterations. Explains that shared-iterations causes VUs to skip steps in multi-step journeys, producing incorrect metrics.",
|
|
88
|
+
"files": [],
|
|
89
|
+
"assertions": [
|
|
90
|
+
{
|
|
91
|
+
"id": "correct-executor",
|
|
92
|
+
"text": "Recommends per-vu-iterations for the user journey"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "shared-iterations-problem",
|
|
96
|
+
"text": "Explains why shared-iterations is wrong for journeys - VUs race for iterations and may skip steps"
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
"id": "not-ambiguous",
|
|
100
|
+
"text": "Gives a clear recommendation, not a 'it depends' answer without a conclusion"
|
|
101
|
+
}
|
|
102
|
+
]
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
"id": 6,
|
|
106
|
+
"prompt": "My Locust test shows zero failures even though I can see 404 and 500 errors in the server logs during the test run.",
|
|
107
|
+
"expected_output": "Identifies missing catch_response=True as the cause. Without it, Locust counts all HTTP responses as success regardless of status code.",
|
|
108
|
+
"files": [],
|
|
109
|
+
"assertions": [
|
|
110
|
+
{
|
|
111
|
+
"id": "catch-response-cause",
|
|
112
|
+
"text": "Identifies missing catch_response=True as the root cause"
|
|
113
|
+
},
|
|
114
|
+
{
|
|
115
|
+
"id": "failure-call",
|
|
116
|
+
"text": "Explains that r.failure() must be called explicitly inside the catch_response context manager"
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
"id": "code-example",
|
|
120
|
+
"text": "Provides a corrected code snippet using catch_response=True"
|
|
121
|
+
}
|
|
122
|
+
]
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"id": 7,
|
|
126
|
+
"prompt": "What are good starting threshold values for a REST API and a checkout payment endpoint in k6?",
|
|
127
|
+
"expected_output": "Provides differentiated thresholds: REST API p95<500ms, checkout p95<1000ms with stricter error rate. Includes both http_req_duration and http_req_waiting. Notes these are baselines to adjust.",
|
|
128
|
+
"files": [],
|
|
129
|
+
"assertions": [
|
|
130
|
+
{
|
|
131
|
+
"id": "differentiated-thresholds",
|
|
132
|
+
"text": "Gives different threshold values for REST API vs checkout - not the same value for both"
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"id": "http-req-waiting",
|
|
136
|
+
"text": "Includes http_req_waiting as a separate threshold alongside http_req_duration"
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
"id": "checks-threshold",
|
|
140
|
+
"text": "Includes checks: rate>0.99 threshold"
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
"id": "adjust-caveat",
|
|
144
|
+
"text": "Notes that values are baselines and should be adjusted to actual SLA requirements"
|
|
145
|
+
}
|
|
146
|
+
]
|
|
147
|
+
}
|
|
148
|
+
]
|
|
149
|
+
}
|