flowproof 0.9.0__tar.gz → 0.9.1__tar.gz

{flowproof-0.9.0 → flowproof-0.9.1}/Cargo.lock

@@ -760,7 +760,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-adapters"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "anyhow",
  "flowproof-driver",
@@ -779,7 +779,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-agent"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "chrono",
  "flowproof-driver",
@@ -795,7 +795,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-cli"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "ab_glyph",
  "chrono",
@@ -816,7 +816,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-driver"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "image",
  "postgres",
@@ -830,7 +830,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-python"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "flowproof-agent",
  "flowproof-cli",
@@ -842,7 +842,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-replay"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "chrono",
  "flowproof-agent",
@@ -856,7 +856,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-trace"
-version = "0.9.0"
+version = "0.9.1"
 dependencies = [
  "jsonschema",
  "regex",

{flowproof-0.9.0 → flowproof-0.9.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/flowproof-driver", "crates/flowproof-trace", "crates/flowproof-replay", "crates/flowproof-agent", "crates/flowproof-adapters", "crates/flowproof-cli", "crates/flowproof-python"]
 
 [workspace.package]
-version = "0.9.0"
+version = "0.9.1"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/automators-com/flowproof"

{flowproof-0.9.0 → flowproof-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowproof
-Version: 0.9.0
+Version: 0.9.1
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Testing
 Requires-Dist: mcp>=1.2 ; python_full_version >= '3.10' and extra == 'mcp'
 Provides-Extra: mcp
-Summary: Deterministic tests for AI agents: record a run once, replay it with zero LLM calls. Assert which tools an agent called, with which arguments, in which order.
+Summary: Deterministic evidence and control-regression gating for AI agents and the SAP, web and Citrix systems they drive. Record once, replay with zero LLM calls.
 Author-email: Automators <hello@automators.com>
 License-Expression: Apache-2.0
 Requires-Python: >=3.9

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-adapters/src/agent_runner.rs

@@ -36,6 +36,31 @@ use crate::egress::{AllowSet, EgressLog};
 /// upstream.
 const BASE_URL_VARS: [&str; 3] = ["OPENAI_BASE_URL", "OPENAI_API_BASE", "OPENAI_BASE"];
 
+/// Every variable that can carry a REAL model credential, and which the agent
+/// must therefore never see.
+///
+/// The agent is third-party code. flowproof holds the upstream key and attaches
+/// it only on the outbound hop, so the agent is handed a placeholder — that was
+/// always the design for `OPENAI_API_KEY` and `ANTHROPIC_API_KEY`, which are
+/// overwritten below.
+///
+/// What was missing is that overwriting two names is not the same as masking
+/// the credential. `Command` inherits the parent environment, and the *first*
+/// name `flowproof record` looks in is `FLOWPROOF_AGENT_KEY` — so the
+/// recommended way to supply the key was also the one way it reached the agent
+/// verbatim. `FLOWPROOF_AI_API_KEY`, which the LLM-authoring backend reads, did
+/// the same.
+///
+/// Removed rather than placeheld: a variable flowproof does not hand out has no
+/// business having a value, and an agent reading one is asking a question it
+/// should not get an answer to.
+pub const CREDENTIAL_VARS: [&str; 4] = [
+    "FLOWPROOF_AGENT_KEY",
+    "FLOWPROOF_AI_API_KEY",
+    "ANTHROPIC_API_KEY",
+    "OPENAI_API_KEY",
+];
+
 /// Split a command line into argv, honouring double quotes so a path
 /// with spaces survives.
 ///
@@ -366,6 +391,13 @@ fn configure(
     for var in BASE_URL_VARS {
         child.env(var, base);
     }
+    // Drop every credential name BEFORE handing out placeholders. `Command`
+    // inherits this process's environment, so a name flowproof does not
+    // explicitly set is passed through — and the record path runs with a real
+    // key in exactly these variables.
+    for var in CREDENTIAL_VARS {
+        child.env_remove(var);
+    }
     child.env("OPENAI_API_KEY", "flowproof-replay-no-key-needed");
     // The Anthropic SDK reads its own base URL and appends `/v1/messages`
     // itself, so it wants the origin WITHOUT the `/v1` the OpenAI vars keep;
@@ -707,6 +739,91 @@ mod tests {
         assert_eq!(out.len(), 200_000, "every byte the child wrote is captured");
     }
 
+    /// What `configure` decided for one variable: `Some` to set it, `None`
+    /// if it is explicitly removed, and `None` from the outer option if it
+    /// was never mentioned — which, since `Command` inherits, means the
+    /// child gets whatever this process has.
+    fn decided(cmd: &Command, key: &str) -> Option<Option<String>> {
+        cmd.get_envs()
+            .find(|(k, _)| *k == std::ffi::OsStr::new(key))
+            .map(|(_, v)| v.map(|v| v.to_string_lossy().into_owned()))
+    }
+
+    /// The agent is third-party code and flowproof holds the real key, so the
+    /// key must not be in the agent's environment. Two names were: the record
+    /// path reads `FLOWPROOF_AGENT_KEY` FIRST, so the recommended way to
+    /// supply the credential was also the one way it reached the agent intact.
+    ///
+    /// Asserted on the `Command` rather than by spawning. Proving it by
+    /// spawning means putting a real value in this process's environment, and
+    /// `set_var` is process-global — with tests running in parallel that is a
+    /// race, which is exactly what #236 was. `env_remove` guarantees the child
+    /// sees nothing regardless of the parent, so reading the decision is both
+    /// deterministic and a stronger claim than one spawn happening to be clean.
+    #[test]
+    fn a_real_credential_is_never_handed_to_the_agent() {
+        let cmd =
+            configure("echo hi", "http://127.0.0.1:9/v1", &BTreeMap::new()).expect("configure");
+        for var in ["FLOWPROOF_AGENT_KEY", "FLOWPROOF_AI_API_KEY"] {
+            assert_eq!(
+                decided(&cmd, var),
+                Some(None),
+                "{var} must be REMOVED from the agent's environment; \
+                 inheriting it hands third-party code the real upstream key"
+            );
+        }
+    }
+
+    /// The other direction, and it is not implied by the one above: a client
+    /// that refuses to start without a key must still get one, or masking the
+    /// credential would break every agent it protects. Placeholders, not
+    /// removal, for exactly these two.
+    #[test]
+    fn the_agent_still_gets_a_key_shaped_placeholder() {
+        let cmd =
+            configure("echo hi", "http://127.0.0.1:9/v1", &BTreeMap::new()).expect("configure");
+        for var in ["OPENAI_API_KEY", "ANTHROPIC_API_KEY"] {
+            assert_eq!(
+                decided(&cmd, var),
+                Some(Some("flowproof-replay-no-key-needed".to_string())),
+                "{var} must be a placeholder: present, and worthless"
+            );
+        }
+    }
+
+    /// Every name that can hold a real credential has to be covered, not just
+    /// the two that were noticed. This fails when someone adds a fifth.
+    #[test]
+    fn every_credential_variable_is_accounted_for() {
+        let cmd =
+            configure("echo hi", "http://127.0.0.1:9/v1", &BTreeMap::new()).expect("configure");
+        for var in CREDENTIAL_VARS {
+            let d = decided(&cmd, var);
+            assert!(
+                d == Some(None) || d == Some(Some("flowproof-replay-no-key-needed".to_string())),
+                "{var} is in CREDENTIAL_VARS but configure() neither removes it \
+                 nor replaces it with the placeholder; it would be inherited"
+            );
+        }
+    }
+
+    /// A spec that deliberately passes a credential through — `agent.env` with
+    /// `ANTHROPIC_API_KEY: ${SECRET}` — is a documented escape hatch, and spec
+    /// env is applied last precisely so it wins. Masking must not quietly take
+    /// that away; an adopter relying on it would see their agent lose its key
+    /// with no error.
+    #[test]
+    fn a_spec_can_still_pass_a_key_through_on_purpose() {
+        let mut env = BTreeMap::new();
+        env.insert("ANTHROPIC_API_KEY".to_string(), "sk-deliberate".to_string());
+        let cmd = configure("echo hi", "http://127.0.0.1:9/v1", &env).expect("configure");
+        assert_eq!(
+            decided(&cmd, "ANTHROPIC_API_KEY"),
+            Some(Some("sk-deliberate".to_string())),
+            "the spec's own env is applied last and must still override"
+        );
+    }
+
     /// The gap a real adopter hit: their client reads AI_GATEWAY_URL, and
     /// the proxy's port is not known when the spec is written, so a static
     /// `agent.env` value could not reach it.

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-adapters/src/sap_com.rs

@@ -605,6 +605,39 @@ pub mod com {
                 .map(Disp)
         }
 
+        /// Log in on SAP's own logon screen, using the standard fields every
+        /// SAP system ships (client/user/password - the same ones a VBScript
+        /// recording of a manual login would drive). A no-op unless both
+        /// SAP_USER and SAP_PASSWORD are set: attach-only and an explicit
+        /// OpenConnection-without-login stay exactly as they were before this
+        /// existed. Values are read fresh from the environment and used only
+        /// as this one property-put each - never logged, never traced, never
+        /// stored anywhere by this function.
+        fn try_auto_login(session: &Disp) {
+            let (Ok(user), Ok(password)) =
+                (std::env::var("SAP_USER"), std::env::var("SAP_PASSWORD"))
+            else {
+                return;
+            };
+            eprintln!("sap-com: not logged in - attempting auto-login");
+            if let Some(client) = std::env::var("SAP_CLIENT").ok().filter(|c| !c.is_empty()) {
+                if let Some(field) = Self::find_in(session, "wnd[0]/usr/txtRSYST-MANDT") {
+                    let _ = field.put("Text", VARIANT::from(BSTR::from(client.as_str())));
+                }
+            }
+            if let Some(field) = Self::find_in(session, "wnd[0]/usr/txtRSYST-BNAME") {
+                let _ = field.put("Text", VARIANT::from(BSTR::from(user.as_str())));
+            }
+            if let Some(field) = Self::find_in(session, "wnd[0]/usr/pwdRSYST-BCODE") {
+                let _ = field.put("Text", VARIANT::from(BSTR::from(password.as_str())));
+            }
+            // Submit - Enter on the window, exactly how navigate() confirms
+            // any other screen.
+            if let Some(window) = Self::find_in(session, "wnd[0]") {
+                let _ = window.call("SendVKey", vec![VARIANT::from(0i32)]);
+            }
+        }
+
         fn walk_into(element: &Disp, depth: u32, out: &mut Vec<SapElement>) {
             if depth > 14 || out.len() >= 400 {
                 return;
@@ -698,8 +731,9 @@ pub mod com {
             }
             let deadline = Instant::now() + timeout;
             let mut opened = false;
+            let mut login_attempted = false;
             loop {
-                let attempt = (|| -> Result<Option<Disp>, DriverError> {
+                let attempt = (|| -> Result<(Option<Disp>, bool), DriverError> {
                     let sapgui = Disp(attach_to_sapgui()?);
                     let engine = sapgui.call_disp("GetScriptingEngine", vec![])?;
                     let connections = engine.get_disp("Children")?;
@@ -710,12 +744,12 @@ pub mod com {
                                 vec![VARIANT::from(BSTR::from(connection)), VARIANT::from(true)],
                             )?;
                         }
-                        return Ok(None); // keep waiting for a session
+                        return Ok((None, false)); // keep waiting for a session
                     }
                     let conn = connections.call_disp("ElementAt", vec![VARIANT::from(0)])?;
                     let sessions = conn.get_disp("Children")?;
                     if sessions.get_i32("Count").unwrap_or(0) == 0 {
-                        return Ok(None);
+                        return Ok((None, false));
                     }
                     let session = sessions.call_disp("ElementAt", vec![VARIANT::from(0)])?;
                     // A session object exists as soon as its window opens -
@@ -732,17 +766,28 @@ pub mod com {
                         .map(|info| !info.get_string("User").is_empty())
                         .unwrap_or(false);
                     if !logged_in {
-                        return Ok(None); // keep waiting - not logged in yet
+                        // #226 follow-up: an unattended nightly run has
+                        // nobody to log back in when a session expires.
+                        // SAP_USER/SAP_PASSWORD let us do that ourselves,
+                        // once per attach - a no-op when they're unset, so
+                        // attach-only and interactive use are unchanged.
+                        if !login_attempted {
+                            Self::try_auto_login(&session);
+                        }
+                        return Ok((None, true)); // keep waiting - not logged in yet
                     }
-                    Ok(Some(session))
+                    Ok((Some(session), false))
                 })();
 
                 match attempt {
-                    Ok(Some(session)) => {
+                    Ok((Some(session), _)) => {
                         self.session = Some(session);
                         return Ok(());
                     }
-                    Ok(None) => opened = !connection.is_empty(),
+                    Ok((None, at_login_screen)) => {
+                        opened = !connection.is_empty();
+                        login_attempted = login_attempted || at_login_screen;
+                    }
                     Err(e) if Instant::now() >= deadline => {
                         return Err(DriverError::Uia(format!(
                             "{e} — start SAP Logon, log in, and enable scripting \

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-agent/src/recorder.rs

@@ -37,6 +37,19 @@ const STEP_TIMEOUT_MS: u64 = 5000;
 /// Poll cadence while an auto-waiting assertion is pending.
 const ASSERT_POLL_INTERVAL: Duration = Duration::from_millis(250);
 
+/// The precondition wait baked into every targeted step at record time.
+/// `STEP_TIMEOUT_MS` stays the default for everyone - override only when
+/// recording somewhere with real reason to expect more latency (e.g. a
+/// self-hosted CI runner reaching its app over a slower network hop than a
+/// developer's own machine). The replayed value lives in the trace, not
+/// this constant, so this only matters at record time.
+fn step_timeout_ms() -> u64 {
+    std::env::var("FLOWPROOF_STEP_TIMEOUT_MS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(STEP_TIMEOUT_MS)
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum RecordError {
     #[error(transparent)]
@@ -812,7 +825,7 @@ fn step_for(id: usize, intent: &str, app: &str, action: &ResolvedAction) -> Step
         vec![]
     } else {
         vec![Condition::ElementExists {
-            timeout_ms: STEP_TIMEOUT_MS,
+            timeout_ms: step_timeout_ms(),
             selector_ref: None,
         }]
     };

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-cli/src/agent_flow.rs

@@ -1462,10 +1462,17 @@ mod tests {
 
     /// Write a trace file the way `record` would, but hand-built so the test
     /// needs no real model - a driver-blind cassette on disk.
+    ///
+    /// The name carries a nonce as well as the pid. Cargo runs tests as threads
+    /// in ONE process, so a pid-only name is shared by every caller: two of them
+    /// wrote the same file while a third read it, and the reader got
+    /// `EOF while parsing`. That flake failed CI on two unrelated pull requests
+    /// before anyone traced it. Same reason `setup` pairs the pid with
+    /// `mcp_nonce()` a few hundred lines up.
     fn write_trace(cassette: Cassette) -> std::path::PathBuf {
         let dir = std::env::temp_dir().join("flowproof-agent-flow");
         std::fs::create_dir_all(&dir).expect("temp dir");
-        let path = dir.join(format!("trace-{}.json", std::process::id()));
+        let path = dir.join(format!("trace-{}-{}.json", std::process::id(), mcp_nonce()));
         let trace = AgentTrace {
             app: "agent".into(),
             mocks: BTreeMap::new(),
@@ -1481,6 +1488,56 @@ mod tests {
         path
     }
 
+    /// Concurrent `write_trace` calls must not collide.
+    ///
+    /// This asserts the property directly rather than hoping a race shows up:
+    /// with a pid-only name every thread returns the SAME path, so the
+    /// distinctness assertion fails every time instead of one run in twenty.
+    /// Running the suite repeatedly proved nothing - eight consecutive green
+    /// full-suite runs on a four-core box failed to reproduce the flake that had
+    /// already broken CI twice.
+    #[test]
+    fn concurrent_write_trace_calls_get_their_own_files() {
+        use std::sync::{Arc, Barrier};
+
+        const THREADS: usize = 8;
+        let barrier = Arc::new(Barrier::new(THREADS));
+        let handles: Vec<_> = (0..THREADS)
+            .map(|i| {
+                let barrier = Arc::clone(&barrier);
+                std::thread::spawn(move || {
+                    // Start together, to overlap the write as much as possible.
+                    barrier.wait();
+                    write_trace(neutral_cassette(
+                        &format!("prompt {i}"),
+                        &format!("reply {i}"),
+                    ))
+                })
+            })
+            .collect();
+
+        let paths: Vec<_> = handles
+            .into_iter()
+            .map(|h| h.join().expect("thread"))
+            .collect();
+
+        let unique: std::collections::BTreeSet<_> = paths.iter().collect();
+        assert_eq!(
+            unique.len(),
+            THREADS,
+            "every concurrent write_trace needs its own path; got {} distinct of {THREADS}",
+            unique.len()
+        );
+
+        // And each file must be complete: a shared path leaves a reader parsing
+        // a half-written file, which is the `EOF while parsing` the flake showed.
+        for path in &paths {
+            let text = std::fs::read_to_string(path).expect("trace readable");
+            serde_json::from_str::<AgentTrace>(&text)
+                .unwrap_or_else(|e| panic!("{} is not a complete trace: {e}", path.display()));
+        }
+    }
+
     /// A free localhost port: bind ephemeral, read the number, release it.
     /// A tiny race, acceptable in a test - the proxy re-binds it immediately.
     fn free_port() -> u16 {

flowproof-0.9.1/crates/flowproof-cli/tests/falsifiability_audit_since.rs

@@ -0,0 +1,135 @@
+//! Falsifiability proof for the `audit --since` regression gate (issue #254).
+//!
+//! `audit_record_e2e.rs` proves the gate FIRES: a removed control and a
+//! pass->fail change both exit non-zero. Nothing proves it can DECLINE to
+//! fire. A gate wired to exit non-zero unconditionally would satisfy every
+//! assertion in that file, and CI would go red on every clean run until
+//! somebody stopped believing it.
+//!
+//! That is the shape of a false signal in a gate: not a missed regression, but
+//! a gate that says "regression" so often it stops meaning anything. So the
+//! property under proof here is the discriminating half --
+//! `audit --since` exits ZERO when nothing regressed -- which is what makes
+//! its non-zero exit worth acting on.
+//!
+//! Both layers, as everywhere in this suite:
+//!   1. the diff CONTENT (`audit --since --json`), and
+//!   2. the process EXIT CODE a CI run would branch on.
+//!
+//! A gate that reported an empty diff while still exiting non-zero would pass
+//! a content-only check, and would still break every pipeline that uses it.
+
+use std::path::{Path, PathBuf};
+
+const FLOWPROOF_BIN: &str = env!("CARGO_BIN_EXE_flowproof");
+
+fn fixture(name: &str) -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../tests/falsifiability/fixtures/audit-since")
+        .join(name)
+}
+
+fn work_dir(name: &str) -> PathBuf {
+    let dir = std::env::temp_dir().join(format!("flowproof-fals-since-{name}"));
+    std::fs::remove_dir_all(&dir).ok();
+    std::fs::create_dir_all(&dir).expect("work dir");
+    dir
+}
+
+/// Install a committed record fixture as a real run record, in the layout
+/// `flowproof run` would have written. The directory name is taken from the
+/// record's own `run_id`, so the two can never drift apart.
+fn install(dir: &Path, fixture_name: &str) -> String {
+    let body = std::fs::read_to_string(fixture(fixture_name)).expect("read fixture");
+    let value: serde_json::Value = serde_json::from_str(&body).expect("fixture is JSON");
+    let run_id = value["run_id"].as_str().expect("run_id").to_string();
+    let run_dir = dir.join(".flowproof").join("runs").join(&run_id);
+    std::fs::create_dir_all(&run_dir).expect("run dir");
+    std::fs::write(run_dir.join("report.json"), body).expect("install record");
+    run_id
+}
+
+fn audit_since(dir: &Path, base: &str) -> std::process::Output {
+    std::process::Command::new(FLOWPROOF_BIN)
+        .args([
+            "audit",
+            dir.to_str().expect("utf8"),
+            "--since",
+            base,
+            "--json",
+        ])
+        .output()
+        .expect("audit --since")
+}
+
+/// Two runs with identical controls are not a regression. The gate must stay
+/// silent, and it must exit zero.
+#[test]
+fn an_unchanged_control_set_is_not_a_regression() {
+    let dir = work_dir("unchanged");
+    let base = install(&dir, "base.report.json");
+    install(&dir, "head-unchanged.report.json");
+
+    let out = audit_since(&dir, &base);
+
+    // Layer 2: the exit code CI branches on.
+    assert!(
+        out.status.success(),
+        "an unchanged control set must exit zero, or the gate cries wolf on \
+         every clean run; stdout={} stderr={}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+
+    // Layer 1: and it must say nothing changed, rather than exiting zero while
+    // reporting a diff -- which would be the same defect wearing the other face.
+    let diff: serde_json::Value =
+        serde_json::from_slice(&out.stdout).expect("audit --since --json is valid JSON");
+    for section in ["added", "removed", "changed"] {
+        assert_eq!(
+            diff[section].as_array().map(Vec::len).unwrap_or(0),
+            0,
+            "{section} must be empty for an unchanged pair: {diff}"
+        );
+    }
+
+    std::fs::remove_dir_all(&dir).ok();
+}
+
+/// Gaining a control is not a regression. It is reported, because a reviewer
+/// wants to see it, but it must not fail the build: a suite that could never
+/// grow without going red would teach its owners to stop adding controls.
+#[test]
+fn an_added_control_is_reported_but_does_not_fail_the_gate() {
+    let dir = work_dir("added");
+    let base = install(&dir, "base.report.json");
+    install(&dir, "head-added.report.json");
+
+    let out = audit_since(&dir, &base);
+
+    assert!(
+        out.status.success(),
+        "adding a control must not fail the gate; stdout={} stderr={}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+
+    let diff: serde_json::Value =
+        serde_json::from_slice(&out.stdout).expect("audit --since --json is valid JSON");
+    assert_eq!(
+        diff["added"][0]["id"], "falsifiability.since.gamma",
+        "the addition is still reported: {diff}"
+    );
+    assert_eq!(
+        diff["removed"].as_array().map(Vec::len).unwrap_or(0),
+        0,
+        "nothing was removed: {diff}"
+    );
+    assert_eq!(
+        diff["changed"].as_array().map(Vec::len).unwrap_or(0),
+        0,
+        "nothing changed verdict: {diff}"
+    );
+
+    std::fs::remove_dir_all(&dir).ok();
+}

flowproof-0.9.1/crates/flowproof-cli/tests/falsifiability_control_verdict.rs

@@ -0,0 +1,189 @@
+//! Falsifiability proof for `control:` verdict mapping (issue #253).
+//!
+//! `audit_record_e2e.rs` proves audit READS a record rather than re-replaying,
+//! and that `--since` exits non-zero on a regression. But every verdict it
+//! asserts is either `capability-error` (a flow skipped by an env gate) or
+//! hand-written into a record fixture. Nothing there runs a control-bearing
+//! flow that genuinely FAILS and checks what verdict the record receives.
+//!
+//! That is the gap this closes, and it is the highest-consequence false green
+//! available: the control map is what a reader trusts when they cannot read
+//! the trace, so a control reporting `pass` over a failed flow would misreport
+//! the one thing the artifact exists to say.
+//!
+//! The red path is built the way `api_pipeline.rs` builds its green one -- a
+//! real loopback server, with the host travelling as `${API_BASE}` so it never
+//! enters the trace. Record against a live endpoint (an honest trace is
+//! minted), then run with the port dead, so the assertion fails for real at
+//! replay rather than being simulated.
+//!
+//! Two layers, deliberately, because the streaming false green of 0.9.0 got
+//! through by being checked at the wrong one:
+//!   1. the VERDICT recorded in `.flowproof/runs/<id>/report.json`, and
+//!      re-rendered by `flowproof audit`;
+//!   2. the process EXIT CODE of the real binary.
+//!
+//! A run that failed loudly while recording `verdict: pass` would sail through
+//! a one-layer check.
+
+use std::path::{Path, PathBuf};
+
+const FLOWPROOF_BIN: &str = env!("CARGO_BIN_EXE_flowproof");
+
+/// A port with nothing on it. Chosen by binding and dropping, so the number is
+/// free at this instant rather than merely assumed to be.
+fn dead_base() -> String {
+    let probe = std::net::TcpListener::bind("127.0.0.1:0").expect("probe binds");
+    let addr = probe.local_addr().expect("probe addr");
+    drop(probe);
+    format!("http://{addr}")
+}
+
+/// Serve `GET /health` -> 200 for up to `requests` requests, then stop.
+///
+/// Deliberately DETACHED and never joined. Joining would block forever the
+/// moment the flow makes fewer probes than the count guessed here -- a hang is
+/// a far worse failure mode for a falsifiability test than an over-generous
+/// server, because a hung test reports nothing at all.
+fn serve(server: tiny_http::Server, requests: usize) {
+    std::thread::spawn(move || {
+        for _ in 0..requests {
+            let Ok(request) = server.recv() else { break };
+            let (code, body) = if request.url() == "/health" {
+                (200, r#"{"status":"ok"}"#)
+            } else {
+                (404, r#"{"error":"not found"}"#)
+            };
+            request
+                .respond(tiny_http::Response::from_string(body).with_status_code(code))
+                .ok();
+        }
+    });
+}
+
+/// The guilty flow, kept as a committed fixture so it is reviewable as
+/// evidence rather than buried in a string literal.
+fn fixture() -> PathBuf {
+    Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("../../tests/falsifiability/fixtures/control-verdict-fail.flow.yaml")
+}
+
+fn work_dir(name: &str) -> PathBuf {
+    let dir = std::env::temp_dir().join(format!("flowproof-fals-control-{name}"));
+    std::fs::remove_dir_all(&dir).ok();
+    std::fs::create_dir_all(&dir).expect("work dir");
+    dir
+}
+
+/// The single `report.json` written under `<dir>/.flowproof/runs`.
+fn find_record(dir: &Path) -> Option<PathBuf> {
+    let runs = dir.join(".flowproof").join("runs");
+    for entry in std::fs::read_dir(&runs).ok()?.filter_map(Result::ok) {
+        let candidate = entry.path().join("report.json");
+        if candidate.is_file() {
+            return Some(candidate);
+        }
+    }
+    None
+}
+
+/// A control-bearing flow that fails must be recorded as `fail` -- never
+/// `pass`, and never quietly downgraded to `capability-error`, which would
+/// read as "we could not tell" when in fact we could.
+#[test]
+fn a_failing_control_flow_is_recorded_as_fail() {
+    let dir = work_dir("verdict");
+    let spec = dir.join("control-verdict-fail.flow.yaml");
+    std::fs::copy(fixture(), &spec).expect("stage fixture");
+
+    // --- Record honestly, against a live endpoint. ---
+    let server = tiny_http::Server::http("127.0.0.1:0").expect("server binds");
+    let live = format!("http://{}", server.server_addr());
+    serve(server, 4);
+    std::env::set_var("API_BASE", &live);
+
+    let rec = std::process::Command::new(FLOWPROOF_BIN)
+        .arg("record")
+        .arg(&spec)
+        .current_dir(&dir)
+        .env("API_BASE", &live)
+        .output()
+        .expect("record against the live endpoint");
+    assert!(
+        rec.status.success(),
+        "the recording leg must be honest and green; stdout={} stderr={}",
+        String::from_utf8_lossy(&rec.stdout),
+        String::from_utf8_lossy(&rec.stderr)
+    );
+
+    // --- Now run it with the endpoint gone. The assertion fails for real. ---
+    let dead = dead_base();
+    let out = std::process::Command::new(FLOWPROOF_BIN)
+        .arg("run")
+        .arg(&spec)
+        .current_dir(&dir)
+        .env("API_BASE", &dead)
+        .output()
+        .expect("run against the dead endpoint");
+
+    // Layer 2: the exit code CI would see.
+    assert!(
+        !out.status.success(),
+        "a flow whose assertion cannot pass must exit non-zero; stdout={} stderr={}",
+        String::from_utf8_lossy(&out.stdout),
+        String::from_utf8_lossy(&out.stderr)
+    );
+
+    // Layer 1: the verdict the record actually carries.
+    let record = find_record(&dir).unwrap_or_else(|| {
+        panic!(
+            "a failing run must still write a run record, or the control \
+             vanishes from the audit map instead of reporting fail; stdout={} stderr={}",
+            String::from_utf8_lossy(&out.stdout),
+            String::from_utf8_lossy(&out.stderr)
+        )
+    });
+    let body = std::fs::read_to_string(&record).expect("read record");
+    let value: serde_json::Value = serde_json::from_str(&body).expect("record is JSON");
+    let verdict = value["flows"]
+        .as_array()
+        .expect("flows array")
+        .iter()
+        .find_map(|f| {
+            let c = &f["control"];
+            (c["id"] == "falsifiability.control.verdict").then(|| c["verdict"].clone())
+        })
+        .unwrap_or_else(|| panic!("the record folds the flow's control: {body}"));
+
+    assert_eq!(
+        verdict, "fail",
+        "a control whose flow failed must be recorded as fail, not {verdict} -- \
+         this is the false green the audit map cannot afford: {body}"
+    );
+
+    // And `audit` must re-render that verdict rather than soften it.
+    let audit = std::process::Command::new(FLOWPROOF_BIN)
+        .args(["audit", dir.to_str().expect("utf8"), "--json"])
+        .output()
+        .expect("audit --json");
+    let map: serde_json::Value =
+        serde_json::from_slice(&audit.stdout).expect("audit --json is valid JSON");
+    let rendered = map["controls"]
+        .as_array()
+        .expect("controls array")
+        .iter()
+        .find(|c| c["id"] == "falsifiability.control.verdict")
+        .unwrap_or_else(|| {
+            panic!(
+                "audit names the control: {}",
+                String::from_utf8_lossy(&audit.stdout)
+            )
+        })
+        .clone();
+    assert_eq!(
+        rendered["verdict"], "fail",
+        "audit renders the recorded verdict unchanged"
+    );
+
+    std::fs::remove_dir_all(&dir).ok();
+}

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-cli/tests/sap_e2e.rs

@@ -1,6 +1,7 @@
-//! End-to-end against a REAL SAP GUI session. No CI runner has one, so
-//! this is maintainer-run: Windows, SAP GUI installed, logged in, scripting
-//! enabled (`sapgui/user_scripting = TRUE`), and FLOWPROOF_E2E_SAP=1.
+//! End-to-end against a REAL SAP GUI session: on the self-hosted `sap`
+//! runner via .github/workflows/sap-e2e.yml (see #32), or maintainer-run
+//! locally with Windows, SAP GUI installed, logged in, scripting enabled
+//! (`sapgui/user_scripting = TRUE`), and FLOWPROOF_E2E_SAP=1.
 //!
 //! The flow only navigates and reads — it types a transaction code and
 //! asserts on the status/title, touching no business data.
@@ -10,13 +11,19 @@
 use flowproof_adapters::sap_com::SapAppDriver;
 use flowproof_agent::FlowSpec;
 
-const SPEC: &str = "\
-name: Navigate to session status
-app: sap
-steps:
-  - Go to /nSESSION_MANAGER
-  - assert: page shows Session
-";
+/// A logged-in session already exists on every machine this test runs on
+/// today (attach-only, `connection` omitted). SAP_CONNECTION lets an
+/// unattended nightly run open one instead, should the runner ever come up
+/// without one — the value is carried as `${VAR}` and resolved at launch
+/// time, so it never reaches the trace.
+fn spec_yaml() -> String {
+    let mut spec = String::from("name: Navigate to session status\napp: sap\n");
+    if std::env::var("SAP_CONNECTION").is_ok() {
+        spec.push_str("connection: ${SAP_CONNECTION}\n");
+    }
+    spec.push_str("steps:\n  - Go to /nSESSION_MANAGER\n  - assert: page shows Session\n");
+    spec
+}
 
 #[test]
 fn navigates_a_real_sap_session() {
@@ -31,7 +38,7 @@ fn navigates_a_real_sap_session() {
     std::fs::create_dir_all(&dir).expect("temp dir");
     let trace_path = dir.join("session.trace.jsonl");
 
-    let spec = FlowSpec::parse(SPEC).expect("spec parses");
+    let spec = FlowSpec::parse(&spec_yaml()).expect("spec parses");
 
     let mut driver = SapAppDriver::new().expect("COM engine");
     flowproof_agent::record(&spec, &mut driver, &trace_path).expect("records against live SAP");

{flowproof-0.9.0 → flowproof-0.9.1}/crates/flowproof-cli/tests/support/sap_simulator.py

@@ -118,12 +118,45 @@ class Collection:
     Item = ElementAt
 
 
+class SessionInfo:
+    """`GuiSession.Info`, and specifically its `User`.
+
+    The engine uses a non-empty `User` to tell a logged-in session from one
+    sitting at the SAP login screen: a session object exists as soon as its
+    window opens, login screen included, so existence alone proves nothing.
+    Real SAP reports transaction `S000` and an empty user until someone
+    authenticates.
+
+    The simulator therefore has to present a session that is actually logged
+    in, because that is what the test claims to exercise - "record through the
+    production COM engine against a running, logged-in session". A simulator
+    that reported an empty user would be asking the engine to relax the check
+    rather than asking the test to be honest.
+    """
+
+    _public_methods_ = []
+    _public_attrs_ = ["User", "Client", "Transaction", "SystemName"]
+
+    def __init__(self, user="FLOWPROOF", client="001", transaction="VA01",
+                 system="SIM"):
+        self.User = user
+        self.Client = client
+        self.Transaction = transaction
+        self.SystemName = system
+
+
 class Session(Component):
     _public_methods_ = Component._public_methods_ + ["FindById"]
+    _public_attrs_ = Component._public_attrs_ + ["Info"]
 
     def __init__(self, screen):
         Component.__init__(self, screen, "ses", "GuiSession", "ses[0]")
         self.Id = "/app/con[0]/ses[0]"
+        # Wrapped, like every other nested object here (`Children`, `child`,
+        # `connection`). An unwrapped Python instance is not dispatchable, so
+        # the engine's `get_disp("Info")` would fail and the session would look
+        # exactly as "not logged in" as before.
+        self.Info = wrap(SessionInfo())
 
     def FindById(self, element_id):
         element = self._screen.by_id.get(str(element_id))

{flowproof-0.9.0 → flowproof-0.9.1}/flowproof/__init__.py

@@ -24,7 +24,7 @@ from flowproof.flow import (
     run,
 )
 
-__version__ = "0.9.0"
+__version__ = "0.9.1"
 
 __all__ = [
     "ClarificationNeeded",

{flowproof-0.9.0 → flowproof-0.9.1}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "maturin"
 
 [project]
 name = "flowproof"
-version = "0.9.0"
-description = "Deterministic tests for AI agents: record a run once, replay it with zero LLM calls. Assert which tools an agent called, with which arguments, in which order."
+version = "0.9.1"
+description = "Deterministic evidence and control-regression gating for AI agents and the SAP, web and Citrix systems they drive. Record once, replay with zero LLM calls."
 readme = "README.md"
 license = "Apache-2.0"
 requires-python = ">=3.9"