benchset 0.0.3__tar.gz → 0.0.4__tar.gz

{benchset-0.0.3 → benchset-0.0.4}/Cargo.lock

@@ -4,4 +4,4 @@ version = 3
 
 [[package]]
 name = "benchset"
-version = "0.0.3"
+version = "0.0.4"

{benchset-0.0.3 → benchset-0.0.4}/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "benchset"
 # Generated from [project].version in pyproject.toml by scripts/version.py.
-version = "0.0.3"
+version = "0.0.4"
 edition = "2021"
 rust-version = "1.74"
 description = "Embeddable sustained scalar CPU benchmarks with a C ABI"

{benchset-0.0.3 → benchset-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchset
-Version: 0.0.3
+Version: 0.0.4
 Summary: Sustained scalar CPU benchmarks with an embeddable Rust core
 Author-email: Evgeniy Blinov <zheni-b@yandex.ru>
 Project-URL: Source, https://github.com/mutating/benchset
@@ -356,7 +356,7 @@ settings are comparable.
 
 ```toml
 [dependencies]
-benchset = "0.0.3"
+benchset = "0.0.4"
 # For changes from a local source checkout, use:
 # benchset = { path = "../benchset" }
 ```
@@ -471,7 +471,7 @@ the same repository. Rust is not required by the consumer.
 For a Swift manifest, add:
 
 ```swift
-.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.3"))
+.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.4"))
 ```
 
 Add `.product(name: "Benchset", package: "benchset")` to your target's

{benchset-0.0.3 → benchset-0.0.4}/Package.swift

@@ -9,8 +9,8 @@ let package = Package(
     products: [.library(name: "Benchset", targets: ["Benchset"])],
     targets: [
         .binaryTarget(name: "CBenchset",
-                      url: "https://github.com/mutating/benchset/releases/download/v0.0.3/Benchset.xcframework.zip",
-                      checksum: "f50ee204c5a636497bdae1f0a2bb8a7987042f58bd84c5153cae6201e1e35aea"),
+                      url: "https://github.com/mutating/benchset/releases/download/v0.0.4/Benchset.xcframework.zip",
+                      checksum: "0e6643b1daa58773a93aec6b4228b2d81580f058dc5348609457363a9b5ec4ae"),
         .target(name: "Benchset", dependencies: ["CBenchset"]),
         .testTarget(name: "BenchsetTests", dependencies: ["Benchset"], path: "tests/swift")
     ]

{benchset-0.0.3 → benchset-0.0.4}/README.md

@@ -322,7 +322,7 @@ settings are comparable.
 
 ```toml
 [dependencies]
-benchset = "0.0.3"
+benchset = "0.0.4"
 # For changes from a local source checkout, use:
 # benchset = { path = "../benchset" }
 ```
@@ -437,7 +437,7 @@ the same repository. Rust is not required by the consumer.
 For a Swift manifest, add:
 
 ```swift
-.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.3"))
+.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.4"))
 ```
 
 Add `.product(name: "Benchset", package: "benchset")` to your target's

{benchset-0.0.3 → benchset-0.0.4}/benchset/cli.py

@@ -53,7 +53,7 @@ class _Parser(argparse.ArgumentParser):
 
 
 def settings(argv: List[str]) -> Settings:
-    source = _Arguments(named_arguments=['bench', 'warmup', 'duration'])
+    source = _Arguments(named_arguments=[])
     source.parser = _Parser(
         prog='benchset', argument_default=argparse.SUPPRESS,
         description='Sustained scalar CPU throughput and speed relative to Raspberry Pi 5.',

{benchset-0.0.3 → benchset-0.0.4}/benchset.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: benchset
-Version: 0.0.3
+Version: 0.0.4
 Summary: Sustained scalar CPU benchmarks with an embeddable Rust core
 Author-email: Evgeniy Blinov <zheni-b@yandex.ru>
 Project-URL: Source, https://github.com/mutating/benchset
@@ -356,7 +356,7 @@ settings are comparable.
 
 ```toml
 [dependencies]
-benchset = "0.0.3"
+benchset = "0.0.4"
 # For changes from a local source checkout, use:
 # benchset = { path = "../benchset" }
 ```
@@ -471,7 +471,7 @@ the same repository. Rust is not required by the consumer.
 For a Swift manifest, add:
 
 ```swift
-.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.3"))
+.package(url: "https://github.com/mutating/benchset.git", .exact("0.0.4"))
 ```
 
 Add `.product(name: "Benchset", package: "benchset")` to your target's

{benchset-0.0.3 → benchset-0.0.4}/native/src/kernel_tests.rs

@@ -45,6 +45,22 @@ fn golden_workload_vectors() {
                 8459165180143766608,
             ],
         ),
+        (
+            2,
+            [
+                6438600776746354456,
+                496348541366349225,
+                10204253244531584854,
+            ],
+        ),
+        (
+            3,
+            [
+                8423339373471318280,
+                5810453192619338186,
+                1329735283187018493,
+            ],
+        ),
         (
             usize::MAX,
             [
@@ -78,6 +94,42 @@ fn crafted_parser_inputs_and_wrapping_states() {
     }
 }
 
+#[test]
+fn parser_preserves_initial_state_before_any_separator() {
+    // A leading separator resets parser state. Exercise prefixes before that
+    // reset, including invalid tokens, while preserving the calibrated workload.
+    for (state, expected) in [
+        (
+            0,
+            [412704983584011650, 9308271035056677818, 3612450324633344040],
+        ),
+        (
+            1,
+            [
+                9009395174570800451,
+                17241420472984297642,
+                8483684467978543156,
+            ],
+        ),
+        (
+            u64::MAX,
+            [
+                5890298480000480609,
+                1796841914283376693,
+                15172682340896178227,
+            ],
+        ),
+    ] {
+        for (prefix, checksum) in b"+.e".iter().zip(expected) {
+            let mut kernel = Kernel::new(0).unwrap();
+            kernel.state = state;
+            kernel.input.fill(b' ');
+            kernel.input[0] = *prefix;
+            assert_eq!(kernel.step(), checksum);
+        }
+    }
+}
+
 #[test]
 fn capacity_failure_is_reported_without_real_oom() {
     assert!(allocate_table(usize::MAX).unwrap_err().contains("capacity"));

{benchset-0.0.3 → benchset-0.0.4}/native/src/lib.rs

@@ -169,6 +169,20 @@ struct Timing {
     end: Instant,
 }
 
+impl Timing {
+    fn is_warmup(self, now: Instant) -> bool {
+        now < self.warm_end
+    }
+
+    fn sample_elapsed(self, now: Instant, last_elapsed: f64) -> Option<f64> {
+        if self.is_warmup(now) || now >= self.end {
+            return None;
+        }
+        let elapsed = now.duration_since(self.warm_end).as_secs_f64();
+        (elapsed - last_elapsed >= 1.0).then_some(elapsed)
+    }
+}
+
 struct Workers {
     handles: Vec<JoinHandle<()>>,
     stop: Arc<AtomicBool>,
@@ -319,7 +333,7 @@ fn run_with(
                     {
                         break;
                     }
-                    let warmup = now < times.warm_end;
+                    let warmup = times.is_warmup(now);
                     // Publish once per eight iterations, never contend on a shared counter.
                     for _ in 0..8 {
                         checksum = kernel.step();
@@ -387,24 +401,21 @@ fn run_with(
     let mut last_sample_iterations = 0;
     loop {
         let now = Instant::now();
-        let warmup = now < times.warm_end;
+        let warmup = times.is_warmup(now);
         let (phase, phase_start, phase_duration) = if warmup {
             (Phase::Warmup, times.start, options.warmup)
         } else {
             (Phase::Measurement, times.warm_end, options.measurement)
         };
         let iterations = sum(&counters, warmup);
-        if phase == Phase::Measurement {
-            let elapsed = now.saturating_duration_since(times.warm_end).as_secs_f64();
-            if elapsed - last_sample_elapsed >= 1.0 && now < times.end {
-                append_sample(
-                    &mut result.samples,
-                    elapsed,
-                    iterations,
-                    &mut last_sample_elapsed,
-                    &mut last_sample_iterations,
-                );
-            }
+        if let Some(elapsed) = times.sample_elapsed(now, last_sample_elapsed) {
+            append_sample(
+                &mut result.samples,
+                elapsed,
+                iterations,
+                &mut last_sample_elapsed,
+                &mut last_sample_iterations,
+            );
         }
         if last_phase != phase || now.duration_since(last_progress) >= Duration::from_millis(100) {
             on_progress(Progress {
@@ -433,16 +444,7 @@ fn run_with(
     for handle in workers.handles.drain(..) {
         panicked |= handle.join().is_err();
     }
-    result.iterations = sum(&counters, false);
-    result.elapsed_seconds = counters
-        .iter()
-        .map(|c| c.finish_ns.load(Ordering::Relaxed))
-        .max()
-        .unwrap_or(0) as f64
-        / 1e9;
-    result.checksum = counters
-        .iter()
-        .fold(0, |value, c| value ^ c.checksum.load(Ordering::Relaxed));
+    collect_counters(&mut result, &counters);
     finish_samples(&mut result);
     finish_result(&mut result, panicked, cancellation.is_cancelled());
     on_progress(Progress {
@@ -469,6 +471,19 @@ fn release(gate: &(std::sync::Mutex<bool>, std::sync::Condvar)) {
     gate.1.notify_all();
 }
 
+fn collect_counters(result: &mut BenchmarkResult, counters: &[Arc<Counters>]) {
+    result.iterations = sum(counters, false);
+    result.elapsed_seconds = counters
+        .iter()
+        .map(|c| c.finish_ns.load(Ordering::Relaxed))
+        .max()
+        .unwrap_or(0) as f64
+        / 1e9;
+    result.checksum = counters
+        .iter()
+        .fold(0, |value, c| value ^ c.checksum.load(Ordering::Relaxed));
+}
+
 fn sum(counters: &[Arc<Counters>], warmup: bool) -> u64 {
     counters
         .iter()

{benchset-0.0.3 → benchset-0.0.4}/native/src/runtime_tests.rs

@@ -49,6 +49,8 @@ fn assert_recovered() {
 
 #[test]
 fn public_defaults_names_and_shared_cancellation() {
+    assert_eq!(DEFAULT_WARMUP_SECONDS, 120.0);
+    assert_eq!(DEFAULT_MEASUREMENT_SECONDS, 60.0);
     assert_eq!(Benchmark::SingleCore.name(), "single_core");
     assert_eq!(Benchmark::MultiCore.name(), "multi_core");
     assert_eq!(
@@ -141,6 +143,13 @@ fn real_runs_account_for_work_and_progress_on_caller() {
             assert!((0.0..=1.0).contains(&p.fraction));
             if matches!(p.phase, Phase::Warmup | Phase::Measurement) {
                 assert!(p.phase_elapsed_seconds <= p.phase_duration_seconds);
+                let elapsed = p.phase_elapsed_seconds
+                    + if p.phase == Phase::Measurement {
+                        0.02
+                    } else {
+                        0.0
+                    };
+                assert!((p.fraction - (elapsed / 0.06).min(1.0)).abs() < 1e-12);
             } else {
                 assert_eq!(p.phase_elapsed_seconds, result.elapsed_seconds);
             }
@@ -807,3 +816,177 @@ fn final_sample_repairs_snapshot_after_worker_finish() {
         }
     }
 }
+
+#[test]
+fn timing_uses_half_open_phases_and_periodic_measurement_samples() {
+    let start = Instant::now();
+    let times = Timing {
+        start,
+        warm_end: start + Duration::from_secs(1),
+        end: start + Duration::from_secs(5),
+    };
+    assert!(times.is_warmup(times.warm_end - Duration::from_nanos(1)));
+    assert!(!times.is_warmup(times.warm_end));
+    assert!(!times.is_warmup(times.end));
+    assert!(!Timing {
+        warm_end: start,
+        ..times
+    }
+    .is_warmup(start));
+    for (offset_ns, last_elapsed, expected) in [
+        (0, 0.0, None),
+        (1_000_000_000, 0.0, None),
+        (1_999_999_999, 0.0, None),
+        (2_000_000_000, 0.0, Some(1.0)),
+        (2_100_000_000, 1.0, None),
+        (2_500_000_000, 0.5, Some(1.5)),
+        (3_000_000_000, 1.0, Some(2.0)),
+        (5_000_000_000, 0.0, None),
+        (5_000_000_001, 0.0, None),
+    ] {
+        assert_eq!(
+            times.sample_elapsed(start + Duration::from_nanos(offset_ns), last_elapsed),
+            expected
+        );
+    }
+}
+
+#[test]
+fn cancelled_warmup_excludes_work_and_reports_elapsed_fraction() {
+    let _serial = TEST_LOCK.lock().unwrap();
+    let token = Cancellation::default();
+    let before = Instant::now();
+    let mut events = Vec::new();
+    let mut observed_work = false;
+    let result = run(
+        Benchmark::SingleCore,
+        Options {
+            warmup: Duration::from_secs(10),
+            measurement: Duration::from_secs(20),
+        },
+        &token,
+        |p| {
+            events.push(p);
+            if p.phase == Phase::Warmup && p.iterations > 0 {
+                observed_work = true;
+                token.cancel();
+            } else if p.phase == Phase::Measurement {
+                token.cancel();
+            }
+        },
+    );
+    assert!(observed_work, "warmup must publish its completed work");
+    assert_eq!(result.status, Status::Cancelled);
+    assert_eq!(
+        result.iterations, 0,
+        "warmup work must not enter measurement"
+    );
+    assert_eq!(result.elapsed_seconds, 0.0);
+    let final_event = events.last().unwrap();
+    assert_eq!(final_event.phase, Phase::Cancelled);
+    assert!(final_event.fraction > 0.0);
+    assert!(final_event.fraction <= before.elapsed().as_secs_f64() / 30.0 + 1e-12);
+    assert!(result.score.is_none() && result.pi_equivalent.is_none());
+}
+
+#[test]
+fn observed_measurement_time_produces_periodic_samples_before_the_tail() {
+    let _serial = TEST_LOCK.lock().unwrap();
+    let token = Cancellation::default();
+    let mut observed_snapshot = None;
+    let mut previous_progress = None;
+    let result = run(
+        Benchmark::SingleCore,
+        Options {
+            warmup: Duration::ZERO,
+            measurement: Duration::from_secs(10),
+        },
+        &token,
+        |p| {
+            if p.phase == Phase::Measurement {
+                if let Some(previous) = previous_progress {
+                    assert!(
+                        p.phase_elapsed_seconds - previous >= 0.1 - 1e-9,
+                        "same-phase progress must be throttled to 10 Hz"
+                    );
+                }
+                previous_progress = Some(p.phase_elapsed_seconds);
+            }
+            if p.phase == Phase::Measurement && p.phase_elapsed_seconds >= 1.0 {
+                let first = *observed_snapshot.get_or_insert(p.phase_elapsed_seconds);
+                if p.phase_elapsed_seconds - first >= 0.2 {
+                    token.cancel();
+                }
+            }
+        },
+    );
+    assert_eq!(result.status, Status::Cancelled);
+    assert!(
+        result.samples.len() >= 2,
+        "an observed snapshot must precede the cancellation tail"
+    );
+    assert!(result.samples[0].elapsed_seconds <= observed_snapshot.unwrap());
+    assert!(result.samples[..result.samples.len() - 1]
+        .iter()
+        .all(|s| s.duration_seconds >= 1.0));
+}
+
+#[test]
+fn callback_panic_stops_already_running_workers_before_the_deadline() {
+    let _serial = TEST_LOCK.lock().unwrap();
+    let token = Cancellation::default();
+    let cleanup = token.clone();
+    let (done_tx, done_rx) = mpsc::channel();
+    let worker = thread::spawn(move || {
+        let outcome = std::panic::catch_unwind(|| {
+            run(
+                Benchmark::SingleCore,
+                Options {
+                    warmup: Duration::from_secs(60),
+                    measurement: Duration::from_secs(60),
+                },
+                &token,
+                |p| {
+                    if p.phase == Phase::Warmup && p.iterations > 0 {
+                        panic!("callback fault after observed work");
+                    }
+                },
+            )
+        });
+        done_tx.send(outcome.is_err()).unwrap();
+    });
+    let stopped = done_rx.recv_timeout(Duration::from_secs(5));
+    // Always unblock a broken worker before asserting, including on a mutant.
+    cleanup.cancel();
+    worker.join().unwrap();
+    assert!(stopped.unwrap());
+    assert_recovered();
+}
+
+#[test]
+fn counter_aggregation_preserves_measurement_work_max_finish_and_xor() {
+    let counters: Vec<_> = [
+        (50, 8, 2_000_000_000, 0b1100),
+        (500, 16, 1_500_000_000, 0b1010),
+    ]
+    .into_iter()
+    .map(|(warmup, measurement, finish, checksum)| {
+        Arc::new(Counters {
+            warmup: AtomicU64::new(warmup),
+            measurement: AtomicU64::new(measurement),
+            finish_ns: AtomicU64::new(finish),
+            checksum: AtomicU64::new(checksum),
+        })
+    })
+    .collect();
+    let mut result = fresh_result();
+    collect_counters(&mut result, &counters);
+    assert_eq!(result.iterations, 24);
+    assert_eq!(result.elapsed_seconds, 2.0);
+    assert_eq!(result.checksum, 0b0110);
+    collect_counters(&mut result, &[]);
+    assert_eq!(
+        (result.iterations, result.elapsed_seconds, result.checksum),
+        (0, 0.0, 0)
+    );
+}

{benchset-0.0.3 → benchset-0.0.4}/pyproject.toml

@@ -5,7 +5,7 @@ backend-path = ["."]
 
 [project]
 name = "benchset"
-version = "0.0.3"
+version = "0.0.4"
 authors = [
   { name="Evgeniy Blinov", email="zheni-b@yandex.ru" },
 ]

{benchset-0.0.3 → benchset-0.0.4}/tests/swift/BenchsetTests.swift

@@ -371,6 +371,59 @@ struct BenchsetTests {
         #expect(stub.creates == 2 && stub.frees == 2 && stub.cancels == 0)
     }
 
+    @Test func cleanupWaitsForAnInFlightNativeCancellation() throws {
+        let stub = NativeStub()
+        let running = DispatchSemaphore(value: 0)
+        let releaseRun = DispatchSemaphore(value: 0)
+        let cancelling = DispatchSemaphore(value: 0)
+        let releaseCancel = DispatchSemaphore(value: 0)
+        let freed = DispatchSemaphore(value: 0)
+        let finished = DispatchSemaphore(value: 0)
+        let cancelDone = DispatchSemaphore(value: 0)
+        let cancelInFlight = LockedBox(false)
+        let output = LockedBox<Result<BenchmarkResult, Error>?>(nil)
+        stub.action = { _, _ in
+            running.signal()
+            #expect(releaseRun.wait(timeout: .now() + 5) == .success)
+        }
+        var native = stub.api
+        native.cancel = { handle in
+            #expect(handle == stub.pointer)
+            cancelInFlight.value = true
+            cancelling.signal()
+            #expect(releaseCancel.wait(timeout: .now() + 5) == .success)
+            cancelInFlight.value = false
+        }
+        native.free = { handle in
+            #expect(handle == stub.pointer)
+            #expect(!cancelInFlight.value, "free must not overlap a native call using the handle")
+            stub.frees += 1
+            freed.signal()
+        }
+        let runner = BenchmarkRunner(native: native)
+        defer { releaseRun.signal(); releaseCancel.signal() }
+        DispatchQueue.global().async {
+            output.value = Result { try runner.run(.singleCore) }
+            finished.signal()
+        }
+        #expect(running.wait(timeout: .now() + 5) == .success)
+        DispatchQueue.global().async {
+            runner.cancel()
+            cancelDone.signal()
+        }
+        #expect(cancelling.wait(timeout: .now() + 5) == .success)
+        releaseRun.signal()
+        // Give cleanup a chance to attempt free while cancellation owns the
+        // handle. The assertion in free checks actual overlap, not elapsed time.
+        _ = freed.wait(timeout: .now() + 0.2)
+        releaseCancel.signal()
+        #expect(cancelDone.wait(timeout: .now() + 5) == .success)
+        #expect(finished.wait(timeout: .now() + 5) == .success)
+        let result = try #require(output.value)
+        #expect(try result.get().success)
+        #expect(stub.frees == 1)
+    }
+
     @Test func terminalCancellationKeepsCompletedResult() throws {
         let runner = BenchmarkRunner()
         let result = try runner.run(.singleCore, warmupSeconds: 0, measurementSeconds: 0.03) {

{benchset-0.0.3 → benchset-0.0.4}/tests/test_cli.py

@@ -26,6 +26,10 @@ from benchset import (
     cli,
 )
 
+# Content checks allow slow interpreter startup on shared runners. Cancellation
+# and terminal-disconnect responsiveness retain their separate 5-second bounds.
+CLI_PROCESS_TIMEOUT = 30
+
 
 def assert_short_cli_completed_or_no_work(
     result: subprocess.CompletedProcess, names: List[str], warmup: float, encoding: str = 'utf-8',
@@ -57,6 +61,7 @@ def assert_short_cli_completed_or_no_work(
         assert float(fields['Measurement seconds']) == 0.03
         assert fields['Unit'] == 'iterations/s'
         if index < len(completed):
+            assert fields['Reason'] in ('—', '-', '\\u2014')
             assert fields['Status'] == 'completed'
             number, unit = fields['Pi equivalent'].split(' \u00d7 ' if encoding == 'utf-8' else ' x ')
             assert float(number) > 0
@@ -163,10 +168,8 @@ def test_information_is_complete_and_has_no_stderr(
     invocation: Tuple[List[str], Dict[str, str]], option: str,
 ) -> None:
     command, environment = invocation
-    # Shared CI runners can delay interpreter startup; this checks content,
-    # while cancellation responsiveness has its own stricter deadlines.
     result = subprocess.run([*command, option], env=environment, capture_output=True,
-                            encoding='utf-8', check=False, timeout=30)
+                            encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == 0, result.stderr
     assert not result.stderr
     if option == '--version':
@@ -200,7 +203,7 @@ def test_invalid_arguments_are_actionable_without_a_traceback(
 ) -> None:
     command, environment = invocation
     result = subprocess.run([*command, *arguments], env=environment, capture_output=True,
-                            encoding='utf-8', check=False, timeout=10)
+                            encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == 2
     assert not result.stdout
     assert 'usage: benchset' in result.stderr
@@ -223,13 +226,13 @@ def test_real_entrypoints_accept_selection_order_numeric_forms_and_redirected_ou
         [f'--{name}={value}'] if equals else ['--' + name, value]
     )]
     result = subprocess.run([*command, *arguments], env=environment, capture_output=True,
-                            check=False, timeout=10)
+                            check=False, timeout=CLI_PROCESS_TIMEOUT)
     names = ['single_core', 'multi_core'] if benchmark is None or benchmark == 'all' else [benchmark]
     assert_short_cli_completed_or_no_work(result, names, float(warmup))
 
 
 def test_defaults_repeated_options_and_environment_independence(
-    monkeypatch: pytest.MonkeyPatch,
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str],
 ) -> None:
     calls: List[Tuple[str, float, float]] = []
 
@@ -243,12 +246,94 @@ def test_defaults_repeated_options_and_environment_independence(
     monkeypatch.setenv('BENCHSET_DURATION', '0')
     assert cli.main([]) == 0
     assert calls == [('single_core', 120, 60), ('multi_core', 120, 60)]
+    output = capsys.readouterr()
+    assert not output.err
+    sections = output.out.rstrip('\n').split('\n\n')
+    assert len(sections) == 2
+    assert sections[0].startswith('Benchmark')
+    assert sections[1].startswith('Benchmark')
+    assert output.out.endswith('  —\n')
     calls.clear()
     assert cli.main(['--bench=all', '--bench', 'multi_core', '--warmup=0.5', '--warmup=-0.0',
                      '--duration=2', '--duration', '+.03']) == 0
     assert calls == [('multi_core', -0.0, 0.03)]
 
 
+@pytest.mark.parametrize(('arguments', 'message'), [
+    (['--warmup=oops'], 'argument --warmup: must be a number of seconds'),
+    (['--duration=oops'], 'argument --duration: must be a number of seconds'),
+    (['--warmup', '-1e-3'], '--warmup must be finite and nonnegative'),
+    (['--warmup=nan'], '--warmup must be finite and nonnegative'),
+    (['--duration=0'], '--duration must be finite and positive'),
+    (['--duration', '-NaN'], '--duration must be finite and positive'),
+])
+def test_programmatic_cli_rejects_invalid_durations_with_option_diagnostic(
+    arguments: List[str], message: str, capsys: pytest.CaptureFixture[str],
+) -> None:
+    assert cli.main(arguments) == 2
+    captured = capsys.readouterr()
+    assert not captured.out
+    assert captured.err.startswith('usage: benchset ')
+    assert captured.err.splitlines()[-1] == 'benchset: error: ' + message
+
+
+def test_cli_help_keeps_explanations_and_copyable_examples_on_separate_lines(
+    capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setenv('COLUMNS', '80')
+    # A calling application's argv must not change the CLI's displayed name.
+    monkeypatch.setattr(sys, 'argv', ['embedding-application'])
+    assert cli.main(['--help']) == 0
+    captured = capsys.readouterr()
+    assert not captured.err
+    paragraphs = captured.out.strip().split('\n\n')
+    assert paragraphs[0].startswith('usage: benchset ')
+    assert paragraphs[1] == 'Sustained scalar CPU throughput and speed relative to Raspberry Pi 5.'
+    assert paragraphs[-2].splitlines() == [
+        'Higher iterations/s is faster. Pi equivalent: 0.5 = half the Pi speed; 2 = twice.',
+        'With no options, runs single_core then multi_core (about 6 minutes total).',
+        'Ctrl+C cancels the run.',
+        'Exit codes: 0 success, 1 failed, 2 invalid arguments, 130 cancelled.',
+    ]
+    assert paragraphs[-1].splitlines() == [
+        'Examples:',
+        '  benchset --bench single_core',
+        '  benchset --bench multi_core --warmup 120 --duration 60',
+        '  benchset --warmup 0 --duration 1  # quick smoke test',
+    ]
+    options = ' '.join(paragraphs[2].split())
+    for entry in (
+        '--bench {all,single_core,multi_core} Benchmark to run (default: all).',
+        '--warmup SECONDS Warmup excluded from the score (default: 120; zero allowed).',
+        '--duration SECONDS Measurement duration, must be positive (default: 60).',
+    ):
+        assert entry in options
+
+
+@pytest.mark.parametrize('benchmark', ['single_core', 'multi_core'])
+def test_cli_worker_exception_retains_benchmark_and_stops_the_suite(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], benchmark: str,
+) -> None:
+    calls: List[str] = []
+
+    def fail(_runner: BenchmarkRunner, name: str, **_options: object) -> BenchmarkResult:
+        calls.append(name)
+        raise RuntimeError('backend unavailable')
+
+    monkeypatch.setattr(BenchmarkRunner, 'run', fail)
+    assert cli.main(['--bench', benchmark]) == 1
+    captured = capsys.readouterr()
+    assert not captured.out
+    fields = {key.strip(): value for key, value in (
+        line.split(' : ', 1) for line in captured.err.split('Samples:\n')[0].splitlines()
+    )}
+    assert fields['Benchmark'] == benchmark
+    assert fields['Status'] == 'failed'
+    assert fields['Reason'] == 'backend unavailable'
+    assert fields['Score'] == fields['Pi equivalent'] == '—'
+    assert calls == [benchmark]
+
+
 @pytest.mark.parametrize('arguments', [
     ['--warmup', '.0', '--duration', ' .03 '],
     ['--war=0', '--dur=3e-2', '--ben=single_core'],
@@ -268,7 +353,7 @@ def test_negative_zero_has_equivalent_cli_spellings(
     command, environment = invocation
     option = ['--warmup=' + value] if equals else ['--warmup', value]
     result = subprocess.run([*command, '--bench=single_core', '--duration=.03', *option],
-                            env=environment, capture_output=True, encoding='utf-8', check=False, timeout=10)
+                            env=environment, capture_output=True, encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert_short_cli_completed_or_no_work(result, ['single_core'], float(value))
 
 
@@ -281,7 +366,7 @@ def test_negative_numeric_values_reach_their_option_validator(
 ) -> None:
     command, environment = invocation
     result = subprocess.run([*command, option, value], env=environment, capture_output=True,
-                            encoding='utf-8', check=False, timeout=10)
+                            encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == 2
     assert not result.stdout
     assert option in result.stderr
@@ -295,7 +380,7 @@ def test_real_redirected_output_supports_non_unicode_encodings(
     command, environment = invocation
     environment['PYTHONIOENCODING'] = encoding + ':strict'
     result = subprocess.run([*command, '--bench=single_core', '--warmup=0', '--duration=.03'],
-                            env=environment, capture_output=True, check=False, timeout=10)
+                            env=environment, capture_output=True, check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert_short_cli_completed_or_no_work(result, ['single_core'], 0, encoding)
 
 
@@ -307,7 +392,7 @@ def test_native_duration_limits_report_failure_and_stop_the_suite(
 ) -> None:
     command, environment = invocation
     result = subprocess.run([*command, '--warmup=0', '--duration=.03', option, value],
-                            env=environment, capture_output=True, encoding='utf-8', check=False, timeout=10)
+                            env=environment, capture_output=True, encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == 1
     assert not result.stdout
     assert 'failed' in result.stderr
@@ -358,6 +443,12 @@ def test_result_calibration_links_keep_cli_output_compact(status: str) -> None:
     assert text.count('Reference machine') == 1
     assert f'{result.reference_machine.name} ({result.reference_machine.short_name})' in text
     assert 'Pi equivalent' in text
+    fields = {key.strip(): value for key, value in (
+        line.split(' : ', 1) for line in text.split('Samples:\n')[0].splitlines()
+    )}
+    for name in ('Reason', 'Score', 'Pi equivalent', 'Variation percent'):
+        assert fields[name] == '—'
+    assert text.endswith('Samples:\n  —\n')
     for nested_metadata in ('reference_machine', 'reference_run', 'ReferenceMachine(', 'ReferenceRun(',
                             'started_at_unix_seconds', 'start_temperature_c', 'warmup_seconds=1248'):
         assert nested_metadata not in text
@@ -503,7 +594,7 @@ def test_real_terminal_disconnect_has_execution_failure_exit_without_traceback(
     os.close(slave)
     try:
         observed = b''
-        deadline = time.monotonic() + 10
+        deadline = time.monotonic() + CLI_PROCESS_TIMEOUT
         while b'\rsingle_core: measurement' not in observed and time.monotonic() < deadline:
             readable, _writeable, _errors = select.select([master], [], [], 0.1)
             if readable:
@@ -552,7 +643,7 @@ def test_closed_output_pipe_exits_without_traceback_or_shutdown_warning(
                                 stdout=writer if destination != 'stderr' else subprocess.PIPE,
                                 stderr=writer if destination == 'stderr' else (
                                     subprocess.STDOUT if destination == 'merged' else subprocess.PIPE
-                                ), check=False, timeout=10)
+                                ), check=False, timeout=CLI_PROCESS_TIMEOUT)
     finally:
         os.close(writer)
     assert result.returncode == 1
@@ -610,7 +701,7 @@ else:
     command = [sys.executable, '-c', program, ','.join(map(str, closed)),
                'module' if len(command) > 1 else command[0], str(completed_fixture)]
     result = subprocess.run([*command, *arguments], env=environment, capture_output=True,
-                            encoding='utf-8', check=False, timeout=10)
+                            encoding='utf-8', check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == status, (result.returncode, result.stdout, result.stderr)
     assert 'Traceback' not in result.stderr
     assert 'Exception ignored' not in result.stderr
@@ -697,7 +788,7 @@ raise SystemExit(main(sys.argv[1:]))
                                  '--unknown' if destination == 'stderr' else '--version'],
                                 env=environment, stdout=sink if destination != 'stderr' else subprocess.PIPE,
                                 stderr=sink if destination != 'stdout' else subprocess.PIPE,
-                                text=True, check=False, timeout=10)
+                                text=True, check=False, timeout=CLI_PROCESS_TIMEOUT)
     assert result.returncode == 1
     assert 'Traceback' not in (result.stderr or '')
     assert 'Exception ignored' not in (result.stderr or '')
@@ -716,6 +807,24 @@ def test_error_report_write_failure_does_not_escape(monkeypatch: pytest.MonkeyPa
     assert cli.main([]) == 1
 
 
+def test_missing_stdout_still_redirects_a_broken_stderr_at_shutdown(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    class Broken(io.StringIO):
+        def flush(self) -> None:
+            raise OSError('destination is unavailable')
+
+        def fileno(self) -> int:
+            return 42
+
+    redirects: List[int] = []
+    monkeypatch.setattr(cli.sys, 'stdout', None)
+    monkeypatch.setattr(cli.sys, 'stderr', Broken())
+    monkeypatch.setattr(cli.os, 'dup2', lambda _source, target: redirects.append(target))
+    assert cli.main([]) == 1
+    assert redirects == [42]
+
+
 @pytest.mark.skipif(os.name == 'nt', reason='POSIX terminal signals; Windows cancellation has direct tests')
 @pytest.mark.parametrize('phase', ['warmup', 'measurement'])
 def test_real_ctrl_c_cancels_each_phase_and_stops_the_suite(
@@ -733,7 +842,7 @@ def test_real_ctrl_c_cancels_each_phase_and_stops_the_suite(
     os.close(slave)
     try:
         observed = b''
-        deadline = time.monotonic() + 10
+        deadline = time.monotonic() + CLI_PROCESS_TIMEOUT
         while phase.encode() not in observed and time.monotonic() < deadline:
             readable, _writeable, _errors = select.select([master], [], [], 0.1)
             if readable:

{benchset-0.0.3 → benchset-0.0.4}/tests/test_semantics.py

@@ -198,23 +198,28 @@ class ManagedNative(Native):
 
 
 @pytest.mark.parametrize('entrypoint', ['runner', 'run', 'run_all'])
-def test_public_api_forwards_default_durations_without_running_long_benchmarks(
-    monkeypatch: pytest.MonkeyPatch, entrypoint: str,
+@pytest.mark.parametrize('durations', [None, (1.2345678901234567, 9.876543210987654)])
+def test_public_api_forwards_durations_and_callback_without_running_long_benchmarks(
+    monkeypatch: pytest.MonkeyPatch, entrypoint: str, durations: Optional[Tuple[float, float]],
 ) -> None:
     backend = ManagedNative()
     monkeypatch.setattr(api, 'Native', lambda: backend)
+    options = {} if durations is None else {'warmup_seconds': durations[0], 'measurement_seconds': durations[1]}
+    expected_durations = (120.0, 60.0) if durations is None else durations
+    events: List[Progress] = []
     results: Tuple[BenchmarkResult, ...]
     if entrypoint == 'runner':
-        results = (BenchmarkRunner().run('single_core'),)
+        results = (BenchmarkRunner().run('single_core', on_progress=events.append, **options),)
     elif entrypoint == 'run':
-        results = (run('single_core'),)
+        results = (run('single_core', on_progress=events.append, **options),)
     else:
-        results = run_all()
+        results = run_all(on_progress=events.append, **options)
     for result in results:
         assert_result_references(result)
     benchmarks = ['single_core', 'multi_core'] if entrypoint == 'run_all' else ['single_core']
     assert [(result.benchmark, result.warmup_seconds, result.measurement_seconds)
-            for result in results] == [(benchmark, 120.0, 60.0) for benchmark in benchmarks]
+            for result in results] == [(benchmark, *expected_durations) for benchmark in benchmarks]
+    assert [event.benchmark for event in events] == benchmarks
     assert all(result.success for result in results)
     assert backend.calls == ['create', 'execute', 'free'] * len(benchmarks)