flowproof 0.2.3__tar.gz → 0.2.4__tar.gz

{flowproof-0.2.3 → flowproof-0.2.4}/Cargo.lock

@@ -722,7 +722,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-adapters"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "anyhow",
  "flowproof-driver",
@@ -738,7 +738,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-agent"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "chrono",
  "flowproof-driver",
@@ -753,7 +753,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-cli"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "ab_glyph",
  "clap",
@@ -771,7 +771,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-driver"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "image",
  "postgres",
@@ -785,7 +785,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-python"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "flowproof-agent",
  "flowproof-cli",
@@ -797,7 +797,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-replay"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "chrono",
  "flowproof-agent",
@@ -811,7 +811,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-trace"
-version = "0.2.3"
+version = "0.2.4"
 dependencies = [
  "jsonschema",
  "serde",

{flowproof-0.2.3 → flowproof-0.2.4}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/flowproof-driver", "crates/flowproof-trace", "crates/flowproof-replay", "crates/flowproof-agent", "crates/flowproof-adapters", "crates/flowproof-cli", "crates/flowproof-python"]
 
 [workspace.package]
-version = "0.2.3"
+version = "0.2.4"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/automators-com/flowproof"

{flowproof-0.2.3 → flowproof-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowproof
-Version: 0.2.3
+Version: 0.2.4
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3

{flowproof-0.2.3 → flowproof-0.2.4}/crates/flowproof-adapters/src/web.rs

@@ -19,8 +19,18 @@ use crate::AdapterError;
 
 const FIND_TIMEOUT: Duration = Duration::from_secs(5);
 
+/// Wrap a browser-driver failure. Transport faults (a dead CDP websocket,
+/// a dropped event) are classified apart from app observations: an
+/// assertion polling inside its recorded wait budget tolerates the former
+/// as a miss, because a call that never reached the page learned nothing
+/// about it.
 fn web_err(context: &str, err: impl std::fmt::Display) -> DriverError {
-    DriverError::Uia(format!("web: {context}: {err}"))
+    let message = format!("{context}: {err}");
+    if is_transport_fault(&message) {
+        DriverError::Transport(message)
+    } else {
+        DriverError::Browser(message)
+    }
 }
 
 /// Launch a fresh headless Chromium (`CHROME` env var overrides the
@@ -210,7 +220,7 @@ impl WebAppDriver {
     fn tab(&self) -> Result<&Arc<Tab>, DriverError> {
         self.tab
             .as_ref()
-            .ok_or_else(|| DriverError::Uia("web: no page open: call launch first".into()))
+            .ok_or_else(|| DriverError::Browser("no page open: call launch first".into()))
     }
 
     fn locator_of(selector: &UiaSelector) -> Option<WebLocator> {
@@ -234,8 +244,8 @@ impl WebAppDriver {
 
     fn locator(selector: &UiaSelector) -> Result<WebLocator, DriverError> {
         Self::locator_of(selector).ok_or_else(|| {
-            DriverError::Uia(format!(
-                "web: selector [{selector}] has no css, automation_id, or text"
+            DriverError::Browser(format!(
+                "selector [{selector}] has no css, automation_id, or text"
             ))
         })
     }
@@ -279,7 +289,7 @@ impl WebAppDriver {
                 return Ok(element);
             }
             if std::time::Instant::now() >= deadline {
-                return Err(DriverError::Uia(format!("web: no element for {locator}")));
+                return Err(DriverError::Browser(format!("no element for {locator}")));
             }
             std::thread::sleep(Duration::from_millis(100));
         }
@@ -473,8 +483,8 @@ impl AppDriver for WebAppDriver {
         // isolated), paying its cold start instead of sharing.
         if let Some(config) = &staged_browser {
             if !config.args.is_empty() {
-                self.browser =
-                    launch_browser(&config.args).map_err(|e| DriverError::Uia(e.to_string()))?;
+                self.browser = launch_browser(&config.args)
+                    .map_err(|e| DriverError::Browser(e.to_string()))?;
                 self.context_id = None;
             }
         }
@@ -1035,7 +1045,7 @@ impl AppDriver for WebAppDriver {
         let json = value
             .value
             .and_then(|v| v.as_str().map(str::to_string))
-            .ok_or_else(|| DriverError::Uia("web: scene script returned no value".into()))?;
+            .ok_or_else(|| DriverError::Browser("scene script returned no value".into()))?;
         Ok(Some(json))
     }
 

{flowproof-0.2.3 → flowproof-0.2.4}/crates/flowproof-agent/src/recorder.rs

@@ -896,11 +896,28 @@ fn poll_oob(
 }
 
 fn assert_holds(actual: &str, expected: &str, matcher: TextMatch) -> bool {
+    // Case-insensitive FALLBACK, mirroring element anchors: an exact
+    // match always wins; when it misses, lowercased comparison decides
+    // ("page shows Close Account" against a page reading "Close
+    // account").
+    //
+    // Widening-only, and it must stay byte-identical to replay's
+    // `text_matches`: if record and replay disagree about a single
+    // assertion, record mints a trace that cannot replay - the exact
+    // failure mode the round-3 field run was built to catch. The negative
+    // form therefore does NOT take the fallback (widening it would fail
+    // traces that used to pass), and a nonzero case-sensitive count is
+    // the count.
+    let (actual_ci, expected_ci) = (actual.to_lowercase(), expected.to_lowercase());
     match matcher {
-        TextMatch::Contains => actual.contains(expected),
+        TextMatch::Contains => actual.contains(expected) || actual_ci.contains(&expected_ci),
         TextMatch::NotContains => !actual.contains(expected),
-        TextMatch::CountEquals(n) => actual.matches(expected).count() as u64 == n,
-        TextMatch::Equals => actual == expected,
+        TextMatch::CountEquals(n) => {
+            let sensitive = actual.matches(expected).count() as u64;
+            sensitive == n
+                || (sensitive == 0 && actual_ci.matches(&expected_ci).count() as u64 == n)
+        }
+        TextMatch::Equals => actual == expected || actual_ci == expected_ci,
         TextMatch::NumericEquals => matches!(
             (flowproof_driver::numeric_value(actual), expected.parse::<f64>()),
             (Some(a), Ok(e)) if a == e

{flowproof-0.2.3 → flowproof-0.2.4}/crates/flowproof-agent/src/rules.rs

@@ -175,11 +175,13 @@ pub const WAIT_STEP_TIMEOUT_MS: u64 = 60_000;
 /// Parse a trailing `within <N>s` / `within <N> seconds` qualifier off a
 /// step, returning (rest, timeout override).
 fn split_within(text: &str) -> (&str, Option<u64>) {
-    let lower = text.to_lowercase();
-    let Some(pos) = lower.rfind(" within ") else {
+    // rfind_ci returns indices valid in `text` itself — never positions
+    // from a lowercased copy, whose byte offsets can differ.
+    let Some(pos) = rfind_ci(text, " within ") else {
         return (text, None);
     };
-    let qualifier = lower[pos + " within ".len()..].trim();
+    let qualifier = text[pos + " within ".len()..].trim().to_lowercase();
+    let qualifier = qualifier.as_str();
     let digits = qualifier
         .strip_suffix(" seconds")
         .or_else(|| qualifier.strip_suffix(" second"))
@@ -207,29 +209,68 @@ fn unresolvable(step: &str, reason: impl Into<String>) -> RulesError {
 }
 
 /// Case-insensitively strip an ASCII `prefix`, returning the rest of the
-/// ORIGINAL string (case preserved).
+/// ORIGINAL string (case preserved). Boundary-safe: `get` returns None
+/// when the cut would split a multibyte char, and an ASCII prefix cannot
+/// match across one — so "no match" is the correct answer, never a panic.
 fn strip_prefix_ci<'a>(text: &'a str, prefix: &str) -> Option<&'a str> {
-    if text.len() >= prefix.len() && text[..prefix.len()].eq_ignore_ascii_case(prefix) {
-        Some(&text[prefix.len()..])
-    } else {
-        None
-    }
+    let head = text.get(..prefix.len())?;
+    head.eq_ignore_ascii_case(prefix)
+        .then(|| &text[prefix.len()..])
 }
 
 /// Case-insensitively strip an ASCII `suffix`, returning the front of the
-/// ORIGINAL string (case preserved).
+/// ORIGINAL string (case preserved). Boundary-safe like `strip_prefix_ci`:
+/// `text.len() - suffix.len()` is a byte offset that can land inside a
+/// multibyte char (`4 × 5=20` with suffix ` times`), which must mean "no
+/// match", not a slice panic — the 0.2.3 escape.
 fn strip_suffix_ci<'a>(text: &'a str, suffix: &str) -> Option<&'a str> {
-    if text.len() >= suffix.len() && text[text.len() - suffix.len()..].eq_ignore_ascii_case(suffix)
-    {
-        Some(&text[..text.len() - suffix.len()])
-    } else {
-        None
-    }
+    let split = text.len().checked_sub(suffix.len())?;
+    let tail = text.get(split..)?;
+    tail.eq_ignore_ascii_case(suffix).then(|| &text[..split])
+}
+
+/// Byte index of the last case-insensitive occurrence of ASCII `needle`
+/// in `text`, ALWAYS valid for slicing `text` itself. The
+/// `text.to_lowercase().find(…)` idiom is the sibling of the slice-panic
+/// bug: lowercasing can change byte lengths (e.g. 'İ' grows), so an index
+/// found in the copy is not an index into the original.
+fn rfind_ci(text: &str, needle: &str) -> Option<usize> {
+    ci_positions(text, needle).last()
+}
+
+fn ci_positions<'a>(text: &'a str, needle: &'a str) -> impl Iterator<Item = usize> + 'a {
+    text.char_indices().map(|(i, _)| i).filter(move |&i| {
+        text.get(i..i + needle.len())
+            .is_some_and(|window| window.eq_ignore_ascii_case(needle))
+    })
 }
 
 /// Resolve one spec step into concrete actions for `app`. Out-of-band
 /// assertions are app-independent — they never touch the UI at all.
+///
+/// NEVER panics: a panic inside parsing (the 0.2.2/0.2.3 multibyte
+/// slice bugs were this class) is caught and degraded to a clean
+/// `Unresolvable` error naming the step — one bad step must fail one
+/// step, not abort the whole suite. The specific known sites are fixed;
+/// this is the backstop for the variant nobody has written yet.
 pub fn resolve_step(app: &str, step: &SpecStep) -> Result<Vec<ResolvedAction>, RulesError> {
+    std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+        resolve_step_inner(app, step)
+    }))
+    .unwrap_or_else(|panic| {
+        let detail = panic
+            .downcast_ref::<String>()
+            .map(String::as_str)
+            .or_else(|| panic.downcast_ref::<&str>().copied())
+            .unwrap_or("unknown panic");
+        Err(unresolvable(
+            &step.intent(),
+            format!("internal parser error ({detail}) — please report this step text"),
+        ))
+    })
+}
+
+fn resolve_step_inner(app: &str, step: &SpecStep) -> Result<Vec<ResolvedAction>, RulesError> {
     match step {
         SpecStep::AssertSql { assert_sql } => {
             return Ok(vec![ResolvedAction::AssertSql {
@@ -796,8 +837,7 @@ mod web {
         // `css:` selector) target; `Type <text> into the <id> field` →
         // `#<id>`; bare `Type <text>` → the focused element.
         if let Some(rest) = strip_prefix_ci(trimmed, "type ") {
-            let lower = rest.to_lowercase();
-            let Some(pos) = lower.rfind(" into the ") else {
+            let Some(pos) = rfind_ci(rest, " into the ") else {
                 let value = rest.trim();
                 if value.is_empty() {
                     return Err(unresolvable(trimmed, "nothing to type"));
@@ -842,8 +882,7 @@ mod web {
         // file-chooser input. The path is stored as written; relative paths
         // resolve against the working directory at execution time.
         if let Some(rest) = strip_prefix_ci(trimmed, "upload ") {
-            let lower = rest.to_lowercase();
-            if let Some(pos) = lower.rfind(" into the ") {
+            if let Some(pos) = rfind_ci(rest, " into the ") {
                 let path = rest[..pos].trim().trim_matches('"');
                 let field = rest[pos + " into the ".len()..].trim();
                 if path.is_empty() {
@@ -882,11 +921,9 @@ mod web {
         // option its own way (the web driver goes through the select's
         // native value setter and fires input+change).
         if let Some(rest) = strip_prefix_ci(trimmed, "select ") {
-            let lower = rest.to_lowercase();
-            let split = lower
-                .rfind(" from the ")
+            let split = rfind_ci(rest, " from the ")
                 .map(|p| (p, " from the ".len()))
-                .or_else(|| lower.rfind(" in the ").map(|p| (p, " in the ".len())));
+                .or_else(|| rfind_ci(rest, " in the ").map(|p| (p, " in the ".len())));
             if let Some((pos, sep_len)) = split {
                 let value = rest[..pos].trim();
                 let (nth, field) = split_ordinal(rest[pos + sep_len..].trim());
@@ -1930,3 +1967,105 @@ mod tests {
         );
     }
 }
+
+#[cfg(test)]
+mod multibyte_tests {
+    use super::*;
+
+    /// The 0.2.2 and 0.2.3 field escapes were both char-boundary slice
+    /// panics on multibyte step text. Sweep every grammar form with a
+    /// zoo of multibyte content — with and without `=`, which is what
+    /// pushed the 0.2.3 variant past the suffix-strip boundary — and
+    /// require that EVERY step either resolves or fails cleanly. A panic
+    /// here is the bug, regardless of which slice site regressed.
+    #[test]
+    fn multibyte_step_text_never_panics_across_the_grammar() {
+        const ZOO: &[&str] = &["×", "÷", "é", "café", "測試", "🚀", "İstanbul"];
+        let mut cases: Vec<(String, String)> = Vec::new();
+        for chunk in ZOO {
+            for tail in ["", "=20", "=x", " = 5"] {
+                let payload = format!("4 {chunk} 5{tail}");
+                for step in [
+                    format!("page shows {payload}"),
+                    format!("page shows {payload} 2 times"),
+                    format!("page does not show {payload}"),
+                    format!("Wait until page shows {payload} within 5s"),
+                    format!("Type {payload} into the \"Name\" field"),
+                    format!("Type {payload}"),
+                    format!("Replace the \"Name\" field with {payload}"),
+                    format!("Select {payload} from the \"Kind\" field"),
+                    format!("Upload {payload} into the \"File\" field"),
+                    format!("Click \"{payload}\""),
+                    format!("Right-click \"{payload}\""),
+                    format!("Press the \"{payload}\" button"),
+                    format!("the \"{payload}\" shows {payload}"),
+                    format!("the \"Name\" field contains {payload}"),
+                    payload.clone(),
+                ] {
+                    cases.push((step, payload.clone()));
+                }
+            }
+        }
+        for (step_text, _payload) in &cases {
+            for kind in [
+                SpecStep::Plain(step_text.clone()),
+                SpecStep::Assert {
+                    assert: step_text.clone(),
+                },
+            ] {
+                // Ok or Err are both fine — only a panic is a failure.
+                let _ = resolve_step("web", &kind);
+                let _ = resolve_step("vision", &kind);
+            }
+        }
+    }
+
+    /// The exact 0.2.3 field repro, pinned: multibyte char followed
+    /// later by `=` must resolve as a surface-text assertion.
+    #[test]
+    fn multibyte_with_equals_resolves_as_page_shows() {
+        for text in ["page shows 4 × 5=20", "page shows a × b=c"] {
+            let step = SpecStep::Assert {
+                assert: text.to_string(),
+            };
+            let actions = resolve_step("web", &step).expect("resolves cleanly");
+            match &actions[0] {
+                ResolvedAction::AssertText {
+                    expected, matcher, ..
+                } => {
+                    assert_eq!(
+                        expected,
+                        text.strip_prefix("page shows ").expect("test constant")
+                    );
+                    assert_eq!(*matcher, TextMatch::Contains);
+                }
+                other => panic!("expected surface AssertText, got {other:?}"),
+            }
+        }
+        // The counting form survives multibyte too (its ` times` suffix
+        // strip was the 0.2.3 panic site).
+        let step = SpecStep::Assert {
+            assert: "page shows 4 × 5=20 2 times".into(),
+        };
+        let actions = resolve_step("web", &step).expect("resolves cleanly");
+        assert!(matches!(
+            &actions[0],
+            ResolvedAction::AssertText {
+                matcher: TextMatch::CountEquals(2),
+                ..
+            }
+        ));
+    }
+
+    /// The backstop: even if a future slice bug panics inside parsing,
+    /// the caller gets a clean error, not a dead suite.
+    #[test]
+    fn parser_panics_degrade_to_clean_errors() {
+        // No known panicking input exists anymore (that is the point of
+        // the fixes), so prove the mechanism directly.
+        let caught = std::panic::catch_unwind(|| {
+            resolve_step("web", &SpecStep::Plain("page shows anything".into()))
+        });
+        assert!(caught.is_ok(), "resolve_step must never unwind");
+    }
+}

{flowproof-0.2.3 → flowproof-0.2.4}/crates/flowproof-cli/src/lib.rs

@@ -496,6 +496,30 @@ fn record_one(
 /// Run every recorded flow under `dir` as one suite: per-flow bundles as
 /// usual, plus a merged `junit.xml` for CI, and a non-zero exit if ANY flow
 /// fails. A failing flow does not stop the suite.
+/// Record one flow as ERRORED and keep the suite going. A driver fault, an
+/// unreadable trace or a failing seed hook is one flow's problem: before
+/// this existed, the first such fault aborted the whole run and no merged
+/// junit was written at all, so CI saw nothing (field report, round 3).
+fn errored_flow(
+    spec_path: &Path,
+    name: &str,
+    message: String,
+    json: bool,
+    flows: &mut Vec<serde_json::Value>,
+    reports: &mut Vec<flowproof_replay::RunReport>,
+) {
+    let report = flowproof_replay::RunReport::errored(name, &message);
+    if !json {
+        println!("[ERROR] {name} ({message})");
+    }
+    flows.push(serde_json::json!({
+        "spec": spec_path,
+        "report": report,
+        "report_path": null,
+    }));
+    reports.push(report);
+}
+
 pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) -> Result<u8, String> {
     let mut specs = Vec::new();
     discover_specs(dir, &mut specs)?;
@@ -520,7 +544,22 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
         // missing-trace error): a deliberately gated flow with no trace
         // is a skip, not a failure. Loading here also surfaces spec parse
         // errors for every suite member.
-        let gated_spec = FlowSpec::load(spec_path).map_err(|e| e.to_string())?;
+        let gated_spec = match FlowSpec::load(spec_path).map_err(|e| e.to_string()) {
+            Ok(spec) => spec,
+            // A spec that will not parse is one broken flow, not a broken
+            // suite: record it and keep going.
+            Err(e) => {
+                errored_flow(
+                    spec_path,
+                    &spec_path.display().to_string(),
+                    e,
+                    json,
+                    &mut flows,
+                    &mut reports,
+                );
+                continue;
+            }
+        };
         if let Some(reason) = gated_spec.skip_reason() {
             let report = flowproof_replay::RunReport::skipped(&gated_spec.name, &reason);
             if !json {
@@ -538,17 +577,35 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
         if !trace_path.exists() {
             match missing {
                 MissingTrace::Error => {
-                    return Err(format!(
-                        "trace {} not found — run `flowproof record {}` first",
-                        trace_path.display(),
-                        spec_path.display()
-                    ));
+                    errored_flow(
+                        spec_path,
+                        &gated_spec.name,
+                        format!(
+                            "trace {} not found — run `flowproof record {}` first",
+                            trace_path.display(),
+                            spec_path.display()
+                        ),
+                        json,
+                        &mut flows,
+                        &mut reports,
+                    );
+                    continue;
                 }
                 MissingTrace::Record => {
                     if !json {
                         println!("[RECORD] {} (no trace yet)", spec_path.display());
                     }
-                    record_one(spec_path, &trace_path, manifest.browser.as_ref())?;
+                    if let Err(e) = record_one(spec_path, &trace_path, manifest.browser.as_ref()) {
+                        errored_flow(
+                            spec_path,
+                            &gated_spec.name,
+                            e,
+                            json,
+                            &mut flows,
+                            &mut reports,
+                        );
+                        continue;
+                    }
                     // Fall through to the normal replay below.
                 }
                 MissingTrace::Skip => {
@@ -575,17 +632,61 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
         }
         // Seed before the flow; a failing hook fails the flow, not the run.
         if let Some(cmd) = &manifest.before_each {
-            run_hook(cmd, spec_path, "before_each")?;
-        }
-        let (header, _) = flowproof_replay::load_trace(&trace_path).map_err(|e| e.to_string())?;
-        // A fresh driver per flow: full isolation, like Playwright contexts.
-        let (report, run_dir, attempts) =
-            replay_with_retries(&trace_path, &header.app.name, retries, !json)?;
-        // Cleanup always runs, pass or fail.
-        if let Some(cmd) = &manifest.after_each {
-            run_hook(cmd, spec_path, "after_each")?;
+            if let Err(e) = run_hook(cmd, spec_path, "before_each") {
+                errored_flow(
+                    spec_path,
+                    &gated_spec.name,
+                    e,
+                    json,
+                    &mut flows,
+                    &mut reports,
+                );
+                continue;
+            }
         }
-        let result_path = report.write_into(&run_dir).map_err(|e| e.to_string())?;
+        let replayed = flowproof_replay::load_trace(&trace_path)
+            .map_err(|e| e.to_string())
+            // A fresh driver per flow: full isolation, like Playwright
+            // contexts. A driver fault here ends THIS flow only.
+            .and_then(|(header, _)| {
+                replay_with_retries(&trace_path, &header.app.name, retries, !json)
+            });
+        // Cleanup always runs, pass, fail or error.
+        let cleanup = match &manifest.after_each {
+            Some(cmd) => run_hook(cmd, spec_path, "after_each"),
+            None => Ok(()),
+        };
+        // Replay first, cleanup second, and only then decide the outcome:
+        // whichever failed, the cleanup has already run.
+        let (report, run_dir, attempts) = match replayed.and_then(|triple| cleanup.map(|()| triple))
+        {
+            Ok(triple) => triple,
+            Err(e) => {
+                errored_flow(
+                    spec_path,
+                    &gated_spec.name,
+                    e,
+                    json,
+                    &mut flows,
+                    &mut reports,
+                );
+                continue;
+            }
+        };
+        let result_path = match report.write_into(&run_dir).map_err(|e| e.to_string()) {
+            Ok(path) => path,
+            Err(e) => {
+                errored_flow(
+                    spec_path,
+                    &gated_spec.name,
+                    e,
+                    json,
+                    &mut flows,
+                    &mut reports,
+                );
+                continue;
+            }
+        };
         if !json {
             println!(
                 "[{}] {} ({} ms){}{}",
@@ -628,6 +729,7 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
     .map_err(|e| e.to_string())?;
 
     let skipped = reports.iter().filter(|r| r.trace_id == "skipped").count();
+    let errored = reports.iter().filter(|r| r.trace_id == "errored").count();
     let passed = reports.iter().filter(|r| r.passed).count() - skipped;
     let ran = reports.len() - skipped;
     let all_passed = reports.iter().all(|r| r.passed);
@@ -636,6 +738,7 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
             "flows": flows,
             "passed": all_passed,
             "skipped": skipped,
+            "errored": errored,
             "junit_path": junit_path,
         });
         println!(
@@ -644,20 +747,31 @@ pub fn run_suite(dir: &Path, json: bool, retries: u8, missing: MissingTrace) ->
         );
     } else {
         println!(
-            "{}: {passed}/{ran} flows passed{} -> {}",
+            "{}: {passed}/{ran} flows passed{}{} -> {}",
             if all_passed { "PASS" } else { "FAIL" },
             if skipped > 0 {
                 format!(", {skipped} skipped")
             } else {
                 String::new()
             },
+            if errored > 0 {
+                format!(", {errored} errored")
+            } else {
+                String::new()
+            },
             junit_path.display()
         );
         if reports.iter().any(|r| r.degraded) {
             println!("DEGRADED: fallback selectors were needed in some flows — heal them");
         }
     }
-    Ok(if all_passed { EXIT_PASS } else { EXIT_FAIL })
+    Ok(if errored > 0 {
+        EXIT_ERROR
+    } else if all_passed {
+        EXIT_PASS
+    } else {
+        EXIT_FAIL
+    })
 }
 
 fn cmd_run(
@@ -731,6 +845,13 @@ fn cmd_run(
                         .unwrap_or_default(),
                 ),
                 StepStatus::Skipped => ("SKIP", String::new()),
+                StepStatus::Errored => (
+                    "ERROR",
+                    step.detail
+                        .as_deref()
+                        .map(|d| format!(" — {d}"))
+                        .unwrap_or_default(),
+                ),
             };
             if step.degraded {
                 let tier = step.selector_tier.as_deref().unwrap_or("fallback");