strobengine 0.5.0__tar.gz → 0.5.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {strobengine-0.5.0 → strobengine-0.5.2}/CHANGELOG.md +40 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/Cargo.lock +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/Cargo.toml +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/PKG-INFO +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/pyproject.toml +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/src/lib.rs +15 -13
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/http.rs +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/websocket.rs +22 -9
- {strobengine-0.5.0 → strobengine-0.5.2}/src/report/schema.rs +1 -1
- {strobengine-0.5.0 → strobengine-0.5.2}/src/report/writer.rs +31 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporter.py +7 -1
- strobengine-0.5.2/src/strobengine/reporting/__init__.py +8 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/baseline.py +26 -4
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/html_report.py +6 -4
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/junit_report.py +6 -4
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/markdown_report.py +5 -61
- strobengine-0.5.2/tests/test_reporting.py +419 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/uv.lock +1 -1
- strobengine-0.5.0/tests/e2e/__init__.py +0 -0
- strobengine-0.5.0/tests/test_reporting.py +0 -252
- {strobengine-0.5.0 → strobengine-0.5.2}/.dockerignore +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.github/workflows/ci.yml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.github/workflows/docker-release.yml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.github/workflows/publish-test.yml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.github/workflows/publish.yml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.gitignore +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.pre-commit-config.yaml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/.python-version +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/Dockerfile +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/LICENSE +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/Makefile +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/README.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/cliff.toml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/benchmarks.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/cli.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/dependencies.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/docker.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/grpc.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/http3.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/http_methods.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/infrastructure.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/install.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/quickstart.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/release.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/reports.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/roadmap.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/sse.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/table_of_contents.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/testing.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/docs/websockets.md +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/grpc.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/http3.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/http_methods.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/load_test.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/spike_test.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/sse.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/stress_test.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/examples/websocket.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/rust-toolchain.toml +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/scripts/release.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/chaos.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/config.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/logging.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/metrics.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/progress.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/grpc.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/grpc_parser.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/grpc_reflection.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/http3.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/mod.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/protocols/sse.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/report/mod.rs +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/__init__.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/_strobengine.pyi +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/cli.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/constants.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/engine.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/py.typed +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/assets/chart.min.js +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/src/strobengine/reporting/csv_report.py +0 -0
- {strobengine-0.5.0/src/strobengine/reporting → strobengine-0.5.2/tests}/__init__.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/conftest.py +0 -0
- {strobengine-0.5.0/tests → strobengine-0.5.2/tests/e2e}/__init__.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/conftest.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/mock_server.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_cli_interface.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_grpc_e2e.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_http3_e2e.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_http_scenarios.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_performance_sanity.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_persistence_e2e.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_sse_e2e.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/e2e/test_websocket_e2e.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/fixtures/service.proto +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/test_cli.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/test_engine.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/test_logging.py +0 -0
- {strobengine-0.5.0 → strobengine-0.5.2}/tests/test_reporter.py +0 -0
|
@@ -4,6 +4,45 @@ All notable changes to `strobengine` will be documented in this file.
|
|
|
4
4
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
5
5
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
6
6
|
|
|
7
|
+
## [0.5.2] - 2026-09-02
|
|
8
|
+
|
|
9
|
+
### Bug Fixes
|
|
10
|
+
|
|
11
|
+
- *(report)* Remove forced 2-decimal rounding of RPS in report schema
|
|
12
|
+
- *(lib)* Log report persistence errors instead of silently discarding
|
|
13
|
+
- *(reporter)* Replace hardcoded version string with dynamic resolution
|
|
14
|
+
- *(reporting)* Validate baseline artifact structure in compute_comparison
|
|
15
|
+
|
|
16
|
+
### Refactoring
|
|
17
|
+
|
|
18
|
+
- *(lib)* Remove unnecessary clones on config.body, config.form, config.headers
|
|
19
|
+
- *(websocket)* Remove unnecessary payload_bytes.clone() in publisher iteration
|
|
20
|
+
- *(reporting)* Extract us_to_ms helper for latency conversion
|
|
21
|
+
|
|
22
|
+
### Styling
|
|
23
|
+
|
|
24
|
+
- *(tests)* Move duplicate inline imports to top of test_reporting.py
|
|
25
|
+
## [0.5.1] - 2026-09-01
|
|
26
|
+
|
|
27
|
+
### Bug Fixes
|
|
28
|
+
|
|
29
|
+
- *(websocket)* Use user-configured timeout for persistent session read
|
|
30
|
+
- *(http)* Use user-configured method for CorruptedPayload chaos fault
|
|
31
|
+
- *(report)* Handle Windows fs::rename target collision in writer
|
|
32
|
+
- *(reporting)* Move rich import inside print_cli_comparison
|
|
33
|
+
- *(websocket)* Warn on invalid WebSocket headers instead of silent drop
|
|
34
|
+
|
|
35
|
+
### Miscellaneous Tasks
|
|
36
|
+
|
|
37
|
+
- *(release)* Bump version to 0.5.1
|
|
38
|
+
|
|
39
|
+
### Refactoring
|
|
40
|
+
|
|
41
|
+
- *(reporting)* Remove dead render_markdown_report function
|
|
42
|
+
|
|
43
|
+
### Testing
|
|
44
|
+
|
|
45
|
+
- *(reporting)* Add unit tests for baseline, html_report, and cli helpers
|
|
7
46
|
## [0.5.0] - 2026-08-30
|
|
8
47
|
|
|
9
48
|
### Documentation
|
|
@@ -51,6 +90,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
51
90
|
|
|
52
91
|
- *(pyproject)* Simplify readme config and update documentation URL
|
|
53
92
|
- *(scripts)* Run uv lock during release version bump
|
|
93
|
+
- *(release)* Bump version to 0.5.0
|
|
54
94
|
|
|
55
95
|
### Styling
|
|
56
96
|
|
|
@@ -72,11 +72,11 @@ fn parse_method(method_str: &str) -> PyResult<Method> {
|
|
|
72
72
|
})
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
fn parse_body(body: Option
|
|
76
|
-
body.map(bytes::Bytes::
|
|
75
|
+
fn parse_body(body: Option<&str>) -> Option<bytes::Bytes> {
|
|
76
|
+
body.map(|b| bytes::Bytes::copy_from_slice(b.as_bytes()))
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
-
fn parse_form(form: Option
|
|
79
|
+
fn parse_form(form: Option<&[(String, String)]>) -> Option<bytes::Bytes> {
|
|
80
80
|
form.map(|pairs| {
|
|
81
81
|
let encoded: String = pairs
|
|
82
82
|
.iter()
|
|
@@ -87,7 +87,7 @@ fn parse_form(form: Option<Vec<(String, String)>>) -> Option<bytes::Bytes> {
|
|
|
87
87
|
})
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
fn parse_headers(headers: Option
|
|
90
|
+
fn parse_headers(headers: Option<&[(String, String)]>) -> PyResult<HeaderMap> {
|
|
91
91
|
let mut header_map = HeaderMap::new();
|
|
92
92
|
|
|
93
93
|
if let Some(h) = headers {
|
|
@@ -96,7 +96,7 @@ fn parse_headers(headers: Option<Vec<(String, String)>>) -> PyResult<HeaderMap>
|
|
|
96
96
|
pyo3::exceptions::PyValueError::new_err(format!("Invalid header name '{k}': {e}"))
|
|
97
97
|
})?;
|
|
98
98
|
|
|
99
|
-
let val = HeaderValue::from_str(
|
|
99
|
+
let val = HeaderValue::from_str(v).map_err(|e| {
|
|
100
100
|
pyo3::exceptions::PyValueError::new_err(format!(
|
|
101
101
|
"Invalid header value for '{k}': {e}"
|
|
102
102
|
))
|
|
@@ -479,9 +479,9 @@ fn run_load_test(py: Python<'_>, config: TestConfig) -> PyResult<metrics::TestSu
|
|
|
479
479
|
protocols::detect_protocol(&url, &config, chaos)
|
|
480
480
|
} else {
|
|
481
481
|
let method = parse_method(&config.method)?;
|
|
482
|
-
let body = parse_body(config.body.
|
|
483
|
-
let form = parse_form(config.form.
|
|
484
|
-
let header_map = parse_headers(config.headers.
|
|
482
|
+
let body = parse_body(config.body.as_deref());
|
|
483
|
+
let form = parse_form(config.form.as_deref());
|
|
484
|
+
let header_map = parse_headers(config.headers.as_deref())?;
|
|
485
485
|
|
|
486
486
|
// Resolve payload and auto-inject Content-Type
|
|
487
487
|
let is_form = form.is_some();
|
|
@@ -533,11 +533,13 @@ fn run_load_test(py: Python<'_>, config: TestConfig) -> PyResult<metrics::TestSu
|
|
|
533
533
|
if !config.no_save {
|
|
534
534
|
let artifact =
|
|
535
535
|
report::schema::ReportArtifact::from_summary_and_config(&summary, &config);
|
|
536
|
-
let
|
|
536
|
+
if let Err(e) = report::writer::save_report_json(
|
|
537
537
|
&artifact,
|
|
538
538
|
config.output_dir.as_deref().map(std::path::Path::new),
|
|
539
539
|
false,
|
|
540
|
-
)
|
|
540
|
+
) {
|
|
541
|
+
tracing::warn!(error = %e, "failed to save report artifact");
|
|
542
|
+
}
|
|
541
543
|
}
|
|
542
544
|
|
|
543
545
|
Ok(summary)
|
|
@@ -575,9 +577,9 @@ fn run_load_profiles(
|
|
|
575
577
|
let chaos_engine = ChaosEngine::new(chaos, chaos_rate);
|
|
576
578
|
|
|
577
579
|
let method = parse_method(method)?;
|
|
578
|
-
let raw_body = parse_body(body);
|
|
579
|
-
let raw_form = parse_form(form);
|
|
580
|
-
let mut header_map = parse_headers(headers.
|
|
580
|
+
let raw_body = parse_body(body.as_deref());
|
|
581
|
+
let raw_form = parse_form(form.as_deref());
|
|
582
|
+
let mut header_map = parse_headers(headers.as_deref())?;
|
|
581
583
|
|
|
582
584
|
// Resolve payload and auto-inject Content-Type
|
|
583
585
|
let is_form = raw_form.is_some();
|
|
@@ -97,7 +97,7 @@ impl ProtocolEngine for HttpEngine {
|
|
|
97
97
|
Some(ChaosFault::CorruptedPayload) => {
|
|
98
98
|
tracing::trace!("chaos: corrupted payload injected");
|
|
99
99
|
self.client
|
|
100
|
-
.
|
|
100
|
+
.request(self.method.clone(), target_url)
|
|
101
101
|
.header(CHAOS_HEADER, "corrupted-payload")
|
|
102
102
|
.body(CORRUPTED_BODY)
|
|
103
103
|
}
|
|
@@ -97,11 +97,16 @@ impl PersistentWsSession {
|
|
|
97
97
|
.map_err(|e| EngineError::ConnectionFailed(e.to_string()))?;
|
|
98
98
|
|
|
99
99
|
for (key, value) in &self.headers {
|
|
100
|
-
|
|
100
|
+
match (
|
|
101
101
|
key.as_str().parse::<http::header::HeaderName>(),
|
|
102
102
|
value.as_str().parse(),
|
|
103
103
|
) {
|
|
104
|
-
|
|
104
|
+
(Ok(name), Ok(val)) => {
|
|
105
|
+
request.headers_mut().insert(name, val);
|
|
106
|
+
}
|
|
107
|
+
_ => {
|
|
108
|
+
tracing::warn!(key = %key, "invalid WebSocket header skipped");
|
|
109
|
+
}
|
|
105
110
|
}
|
|
106
111
|
}
|
|
107
112
|
|
|
@@ -136,7 +141,11 @@ impl PersistentWsSession {
|
|
|
136
141
|
self.messages_sent += 1;
|
|
137
142
|
|
|
138
143
|
// Read response with timeout
|
|
139
|
-
let timeout = Duration::from_secs(
|
|
144
|
+
let timeout = Duration::from_secs(if self.timeout_secs > 0 {
|
|
145
|
+
self.timeout_secs
|
|
146
|
+
} else {
|
|
147
|
+
5
|
|
148
|
+
});
|
|
140
149
|
let read_result = tokio::time::timeout(timeout, async {
|
|
141
150
|
let mut response_bytes = Vec::new();
|
|
142
151
|
while let Some(Ok(msg)) = stream.next().await {
|
|
@@ -283,10 +292,13 @@ impl WebSocketEngine {
|
|
|
283
292
|
.map_err(|e| EngineError::ConnectionFailed(e.to_string()))?;
|
|
284
293
|
|
|
285
294
|
for (key, value) in headers {
|
|
286
|
-
|
|
287
|
-
(
|
|
288
|
-
|
|
289
|
-
|
|
295
|
+
match (key.as_str().parse::<HeaderName>(), value.as_str().parse()) {
|
|
296
|
+
(Ok(name), Ok(val)) => {
|
|
297
|
+
request.headers_mut().insert(name, val);
|
|
298
|
+
}
|
|
299
|
+
_ => {
|
|
300
|
+
tracing::warn!(key = %key, "invalid WebSocket header skipped");
|
|
301
|
+
}
|
|
290
302
|
}
|
|
291
303
|
}
|
|
292
304
|
|
|
@@ -343,8 +355,9 @@ impl WebSocketEngine {
|
|
|
343
355
|
Some(w) => w,
|
|
344
356
|
None => return RequestMetric::error(req_start.elapsed().as_micros()),
|
|
345
357
|
};
|
|
358
|
+
let payload_len = payload_bytes.len() as u64;
|
|
346
359
|
let send_result = write
|
|
347
|
-
.send(Message::Binary(Bytes::from(payload_bytes
|
|
360
|
+
.send(Message::Binary(Bytes::from(payload_bytes)))
|
|
348
361
|
.await;
|
|
349
362
|
|
|
350
363
|
let latency_micros = req_start.elapsed().as_micros();
|
|
@@ -355,7 +368,7 @@ impl WebSocketEngine {
|
|
|
355
368
|
RequestMetric {
|
|
356
369
|
latency_micros,
|
|
357
370
|
status_code: 200,
|
|
358
|
-
bytes_received:
|
|
371
|
+
bytes_received: payload_len,
|
|
359
372
|
is_reconnect: false,
|
|
360
373
|
connection_latency_us: None,
|
|
361
374
|
timestamp_sent_ns: Some(wallclock_ns()),
|
|
@@ -115,7 +115,7 @@ impl ReportArtifact {
|
|
|
115
115
|
total_requests: summary.total_requests as u64,
|
|
116
116
|
successful_requests: successful,
|
|
117
117
|
failed_requests: summary.total_errors as u64,
|
|
118
|
-
rps
|
|
118
|
+
rps,
|
|
119
119
|
bytes_transferred: summary.total_bytes_received,
|
|
120
120
|
},
|
|
121
121
|
latency_percentiles: LatencyPercentiles {
|
|
@@ -57,6 +57,10 @@ pub fn save_report_json(
|
|
|
57
57
|
// Atomic write: serialize → write tmp → rename to final
|
|
58
58
|
let json_bytes = serde_json::to_vec_pretty(artifact)?;
|
|
59
59
|
fs::write(&tmp_path, &json_bytes)?;
|
|
60
|
+
|
|
61
|
+
// Pre-emptively clear target for cross-platform compatibility (Windows)
|
|
62
|
+
let _ = fs::remove_file(&target_path);
|
|
63
|
+
|
|
60
64
|
fs::rename(&tmp_path, &target_path)?;
|
|
61
65
|
|
|
62
66
|
// Update latest.json pointer (also atomic)
|
|
@@ -74,6 +78,10 @@ fn update_latest_pointer(dir: &Path, filename: &str) -> Result<(), io::Error> {
|
|
|
74
78
|
let json_bytes = serde_json::to_vec_pretty(&payload)?;
|
|
75
79
|
|
|
76
80
|
fs::write(&latest_tmp, &json_bytes)?;
|
|
81
|
+
|
|
82
|
+
// Pre-emptively clear existing latest.json for cross-platform compatibility (Windows)
|
|
83
|
+
let _ = fs::remove_file(&latest_path);
|
|
84
|
+
|
|
77
85
|
fs::rename(&latest_tmp, &latest_path)?;
|
|
78
86
|
|
|
79
87
|
Ok(())
|
|
@@ -242,4 +250,27 @@ mod tests {
|
|
|
242
250
|
);
|
|
243
251
|
fs::remove_dir_all("./.strobengine").ok();
|
|
244
252
|
}
|
|
253
|
+
|
|
254
|
+
#[test]
|
|
255
|
+
fn test_save_report_json_overwrites_existing_target() {
|
|
256
|
+
let dir = std::env::temp_dir().join("strobengine_test_overwrite");
|
|
257
|
+
let artifact = sample_artifact();
|
|
258
|
+
|
|
259
|
+
// First write — creates the file
|
|
260
|
+
let path1 = save_report_json(&artifact, Some(&dir), false).unwrap();
|
|
261
|
+
assert!(path1.exists());
|
|
262
|
+
let content1 = fs::read_to_string(&path1).unwrap();
|
|
263
|
+
assert!(content1.contains("metadata"));
|
|
264
|
+
|
|
265
|
+
// Second write — should overwrite (simulates same-second collision)
|
|
266
|
+
let path2 = save_report_json(&artifact, Some(&dir), false).unwrap();
|
|
267
|
+
assert!(path2.exists());
|
|
268
|
+
let content2 = fs::read_to_string(&path2).unwrap();
|
|
269
|
+
assert!(content2.contains("metadata"));
|
|
270
|
+
|
|
271
|
+
// Both paths should be the same (same filename)
|
|
272
|
+
assert_eq!(path1, path2);
|
|
273
|
+
|
|
274
|
+
fs::remove_dir_all(&dir).ok();
|
|
275
|
+
}
|
|
245
276
|
}
|
|
@@ -230,11 +230,17 @@ def _slugify_url(url: str) -> str:
|
|
|
230
230
|
def _get_system_info() -> dict:
|
|
231
231
|
"""Collect basic system information for report metadata."""
|
|
232
232
|
import socket
|
|
233
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
234
|
+
|
|
235
|
+
try:
|
|
236
|
+
pkg_version = version("strobengine")
|
|
237
|
+
except PackageNotFoundError:
|
|
238
|
+
pkg_version = "0.0.0-dev"
|
|
233
239
|
|
|
234
240
|
return {
|
|
235
241
|
"hostname": socket.gethostname(),
|
|
236
242
|
"platform": sys.platform,
|
|
237
|
-
"version":
|
|
243
|
+
"version": pkg_version,
|
|
238
244
|
}
|
|
239
245
|
|
|
240
246
|
|
|
@@ -6,8 +6,7 @@ import json
|
|
|
6
6
|
import sys
|
|
7
7
|
from pathlib import Path
|
|
8
8
|
|
|
9
|
-
from
|
|
10
|
-
from rich.table import Table
|
|
9
|
+
from strobengine.reporting import us_to_ms
|
|
11
10
|
|
|
12
11
|
|
|
13
12
|
def load_baseline_artifact(
|
|
@@ -47,6 +46,8 @@ def compute_comparison(current: dict, baseline: dict) -> dict:
|
|
|
47
46
|
|
|
48
47
|
Returns a dict with baseline metadata, current values, and percentage/point
|
|
49
48
|
deltas for RPS, P95 latency, and error rate.
|
|
49
|
+
|
|
50
|
+
Raises ValueError if either artifact is missing required keys.
|
|
50
51
|
"""
|
|
51
52
|
|
|
52
53
|
def pct_delta(curr_val: float, base_val: float) -> float:
|
|
@@ -54,6 +55,24 @@ def compute_comparison(current: dict, baseline: dict) -> dict:
|
|
|
54
55
|
return 0.0 if curr_val == 0 else 100.0
|
|
55
56
|
return round(((curr_val - base_val) / base_val) * 100, 2)
|
|
56
57
|
|
|
58
|
+
# Validate required structure
|
|
59
|
+
for label, artifact in [("current", current), ("baseline", baseline)]:
|
|
60
|
+
for key in ["summary", "latency_percentiles", "metadata"]:
|
|
61
|
+
if key not in artifact:
|
|
62
|
+
raise ValueError(
|
|
63
|
+
f"Invalid {label} artifact: missing required key '{key}'"
|
|
64
|
+
)
|
|
65
|
+
for subkey in ["rps", "total_requests", "failed_requests"]:
|
|
66
|
+
if subkey not in artifact["summary"]:
|
|
67
|
+
raise ValueError(
|
|
68
|
+
f"Invalid {label} artifact: missing 'summary.{subkey}'"
|
|
69
|
+
)
|
|
70
|
+
for subkey in ["p95_us"]:
|
|
71
|
+
if subkey not in artifact["latency_percentiles"]:
|
|
72
|
+
raise ValueError(
|
|
73
|
+
f"Invalid {label} artifact: missing 'latency_percentiles.{subkey}'"
|
|
74
|
+
)
|
|
75
|
+
|
|
57
76
|
curr_summary = current["summary"]
|
|
58
77
|
base_summary = baseline["summary"]
|
|
59
78
|
curr_lp = current["latency_percentiles"]
|
|
@@ -77,16 +96,19 @@ def compute_comparison(current: dict, baseline: dict) -> dict:
|
|
|
77
96
|
"rps_delta": pct_delta(curr_summary["rps"], base_summary["rps"]),
|
|
78
97
|
"error_rate_delta": round(curr_error_rate - base_error_rate, 2),
|
|
79
98
|
"baseline_rps": base_summary["rps"],
|
|
80
|
-
"baseline_p95_ms":
|
|
99
|
+
"baseline_p95_ms": us_to_ms(base_lp["p95_us"]),
|
|
81
100
|
"baseline_error_rate": round(base_error_rate, 2),
|
|
82
101
|
"current_rps": curr_summary["rps"],
|
|
83
|
-
"current_p95_ms":
|
|
102
|
+
"current_p95_ms": us_to_ms(curr_lp["p95_us"]),
|
|
84
103
|
"current_error_rate": round(curr_error_rate, 2),
|
|
85
104
|
}
|
|
86
105
|
|
|
87
106
|
|
|
88
107
|
def print_cli_comparison(comparison: dict) -> None:
|
|
89
108
|
"""Print baseline comparison table to terminal using Rich."""
|
|
109
|
+
from rich.console import Console
|
|
110
|
+
from rich.table import Table
|
|
111
|
+
|
|
90
112
|
console = Console()
|
|
91
113
|
table = Table(title="Historical Comparison", show_lines=True, padding=(0, 1))
|
|
92
114
|
table.add_column("Metric", style="bold cyan", no_wrap=True)
|
|
@@ -192,12 +192,14 @@ def render_html_report(summary, config, comparison=None) -> str:
|
|
|
192
192
|
artifact = build_artifact_dict(summary, config)
|
|
193
193
|
|
|
194
194
|
# Convert us -> ms for chart display
|
|
195
|
+
from strobengine.reporting import us_to_ms
|
|
196
|
+
|
|
195
197
|
lp = artifact["latency_percentiles"]
|
|
196
198
|
latency_ms = {
|
|
197
|
-
"p50":
|
|
198
|
-
"p90":
|
|
199
|
-
"p95":
|
|
200
|
-
"p99":
|
|
199
|
+
"p50": us_to_ms(lp.get("p50_us")),
|
|
200
|
+
"p90": us_to_ms(lp.get("p90_us")),
|
|
201
|
+
"p95": us_to_ms(lp.get("p95_us")),
|
|
202
|
+
"p99": us_to_ms(lp.get("p99_us")),
|
|
201
203
|
}
|
|
202
204
|
|
|
203
205
|
# Group status codes by class
|
|
@@ -24,7 +24,9 @@ def generate_junit_report(artifact: dict) -> str:
|
|
|
24
24
|
failed = s["failed_requests"]
|
|
25
25
|
total = s["total_requests"]
|
|
26
26
|
error_rate = (failed / total * 100) if total > 0 else 0.0
|
|
27
|
-
|
|
27
|
+
from strobengine.reporting import us_to_ms
|
|
28
|
+
|
|
29
|
+
p95_ms = us_to_ms(lp["p95_us"])
|
|
28
30
|
|
|
29
31
|
testsuites = Element("testsuites")
|
|
30
32
|
testsuite = SubElement(
|
|
@@ -48,9 +50,9 @@ def generate_junit_report(artifact: dict) -> str:
|
|
|
48
50
|
so1 = SubElement(tc1, "system-out")
|
|
49
51
|
so1.text = (
|
|
50
52
|
f"RPS: {s['rps']:.2f}, "
|
|
51
|
-
f"P50: {lp['p50_us']
|
|
52
|
-
f"P95: {p95_ms
|
|
53
|
-
f"P99: {lp['p99_us']
|
|
53
|
+
f"P50: {us_to_ms(lp['p50_us'])}ms, "
|
|
54
|
+
f"P95: {p95_ms}ms, "
|
|
55
|
+
f"P99: {us_to_ms(lp['p99_us'])}ms, "
|
|
54
56
|
f"Errors: {error_rate:.2f}%, "
|
|
55
57
|
f"Total: {total:,}"
|
|
56
58
|
)
|
|
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|
|
5
5
|
from pathlib import Path
|
|
6
6
|
|
|
7
7
|
from strobengine.reporter import build_artifact_dict
|
|
8
|
+
from strobengine.reporting import us_to_ms
|
|
8
9
|
|
|
9
10
|
|
|
10
11
|
def generate_markdown_summary(artifact: dict) -> str:
|
|
@@ -43,10 +44,10 @@ def generate_markdown_summary(artifact: dict) -> str:
|
|
|
43
44
|
f"| Successful | {s['successful_requests']:,} |",
|
|
44
45
|
f"| Failed | {failed:,} |",
|
|
45
46
|
f"| Requests/sec | {s['rps']:.2f} |",
|
|
46
|
-
f"| P50 Latency | {lp['p50_us']
|
|
47
|
-
f"| P90 Latency | {lp['p90_us']
|
|
48
|
-
f"| P95 Latency | {lp['p95_us']
|
|
49
|
-
f"| P99 Latency | {lp['p99_us']
|
|
47
|
+
f"| P50 Latency | {us_to_ms(lp['p50_us'])} ms |",
|
|
48
|
+
f"| P90 Latency | {us_to_ms(lp['p90_us'])} ms |",
|
|
49
|
+
f"| P95 Latency | {us_to_ms(lp['p95_us'])} ms |",
|
|
50
|
+
f"| P99 Latency | {us_to_ms(lp['p99_us'])} ms |",
|
|
50
51
|
f"| Error Rate | {error_rate:.2f}% |",
|
|
51
52
|
"",
|
|
52
53
|
]
|
|
@@ -77,63 +78,6 @@ def generate_markdown_summary(artifact: dict) -> str:
|
|
|
77
78
|
return "\n".join(lines)
|
|
78
79
|
|
|
79
80
|
|
|
80
|
-
def render_markdown_report(summary, config, duration_secs: float) -> str:
|
|
81
|
-
"""Render a Markdown report from TestSummary (full format with all details)."""
|
|
82
|
-
artifact = build_artifact_dict(summary, config)
|
|
83
|
-
meta = artifact["metadata"]
|
|
84
|
-
s = artifact["summary"]
|
|
85
|
-
lp = artifact["latency_percentiles"]
|
|
86
|
-
|
|
87
|
-
duration = duration_secs or meta.get("duration_secs", 0)
|
|
88
|
-
error_rate = (
|
|
89
|
-
(s["failed_requests"] / s["total_requests"] * 100)
|
|
90
|
-
if s["total_requests"] > 0
|
|
91
|
-
else 0.0
|
|
92
|
-
)
|
|
93
|
-
bytes_kb = s["bytes_transferred"] / 1024
|
|
94
|
-
|
|
95
|
-
lines = [
|
|
96
|
-
"# Load Test Results",
|
|
97
|
-
"",
|
|
98
|
-
"| Metric | Value |",
|
|
99
|
-
"|--------|-------|",
|
|
100
|
-
f"| Target URL | `{meta['target_url']}` |",
|
|
101
|
-
f"| Timestamp | {meta['timestamp']} |",
|
|
102
|
-
f"| Duration | {duration:.1f}s |",
|
|
103
|
-
f"| Concurrency | {meta['cli_options']['concurrency']} |",
|
|
104
|
-
f"| Method | {meta['cli_options']['method']} |",
|
|
105
|
-
f"| Total Requests | {s['total_requests']:,} |",
|
|
106
|
-
f"| Successful | {s['successful_requests']:,} |",
|
|
107
|
-
f"| Failed | {s['failed_requests']:,} |",
|
|
108
|
-
f"| Requests/sec | {s['rps']:.2f} |",
|
|
109
|
-
f"| P50 Latency | {lp['p50_us'] / 1000:.2f} ms |",
|
|
110
|
-
f"| P90 Latency | {lp['p90_us'] / 1000:.2f} ms |",
|
|
111
|
-
f"| P95 Latency | {lp['p95_us'] / 1000:.2f} ms |",
|
|
112
|
-
f"| P99 Latency | {lp['p99_us'] / 1000:.2f} ms |",
|
|
113
|
-
f"| Min Latency | {lp['min_us'] / 1000:.2f} ms |",
|
|
114
|
-
f"| Max Latency | {lp['max_us'] / 1000:.2f} ms |",
|
|
115
|
-
f"| Mean Latency | {lp['mean_us'] / 1000:.2f} ms |",
|
|
116
|
-
f"| Error Rate | {error_rate:.2f}% |",
|
|
117
|
-
f"| Bytes Transferred | {bytes_kb:.1f} KB |",
|
|
118
|
-
"",
|
|
119
|
-
]
|
|
120
|
-
|
|
121
|
-
# Status code breakdown
|
|
122
|
-
if artifact["error_breakdown"]:
|
|
123
|
-
lines.append("## Status Codes")
|
|
124
|
-
lines.append("")
|
|
125
|
-
lines.append("| Code | Count |")
|
|
126
|
-
lines.append("|------|-------|")
|
|
127
|
-
for code, count in sorted(
|
|
128
|
-
artifact["error_breakdown"].items(),
|
|
129
|
-
key=lambda x: int(x[0]) if str(x[0]).isdigit() else 0,
|
|
130
|
-
):
|
|
131
|
-
lines.append(f"| {code} | {count:,} |")
|
|
132
|
-
lines.append("")
|
|
133
|
-
|
|
134
|
-
return "\n".join(lines)
|
|
135
|
-
|
|
136
|
-
|
|
137
81
|
def save_markdown_report(summary, config, filepath: str, duration_secs: float) -> str:
|
|
138
82
|
"""Render and write Markdown report to disk. Returns filepath."""
|
|
139
83
|
filepath = str(Path(filepath).expanduser().resolve())
|