flowproof 0.6.0__tar.gz → 0.6.1__tar.gz

{flowproof-0.6.0 → flowproof-0.6.1}/Cargo.lock

@@ -760,7 +760,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-adapters"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "anyhow",
  "flowproof-driver",
@@ -779,7 +779,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-agent"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "chrono",
  "flowproof-driver",
@@ -795,7 +795,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-cli"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "ab_glyph",
  "chrono",
@@ -816,7 +816,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-driver"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "image",
  "postgres",
@@ -830,7 +830,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-python"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "flowproof-agent",
  "flowproof-cli",
@@ -842,7 +842,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-replay"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "chrono",
  "flowproof-agent",
@@ -856,7 +856,7 @@ dependencies = [
 
 [[package]]
 name = "flowproof-trace"
-version = "0.6.0"
+version = "0.6.1"
 dependencies = [
  "jsonschema",
  "regex",

{flowproof-0.6.0 → flowproof-0.6.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "2"
 members = ["crates/flowproof-driver", "crates/flowproof-trace", "crates/flowproof-replay", "crates/flowproof-agent", "crates/flowproof-adapters", "crates/flowproof-cli", "crates/flowproof-python"]
 
 [workspace.package]
-version = "0.6.0"
+version = "0.6.1"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/automators-com/flowproof"

{flowproof-0.6.0 → flowproof-0.6.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowproof
-Version: 0.6.0
+Version: 0.6.1
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -8,7 +8,7 @@ Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Testing
 Requires-Dist: mcp>=1.2 ; python_full_version >= '3.10' and extra == 'mcp'
 Provides-Extra: mcp
-Summary: Generic open-source automation framework for the AI-agent era: automated testing and agentic process automation across web, desktop, and Citrix.
+Summary: Deterministic tests for AI agents: record a run once, replay it with zero LLM calls. Assert which tools an agent called, with which arguments, in which order.
 Author-email: Automators <hello@automators.com>
 License-Expression: Apache-2.0
 Requires-Python: >=3.9
@@ -18,9 +18,12 @@ Project-URL: Repository, https://github.com/automators-com/flowproof
 
 # flowproof
 
-A generic open-source automation framework for the AI-agent era:
-automated testing and agentic process automation across web, desktop,
-and Citrix.
+Run your AI agent once, keep the recording, and assert against it from then
+on. flowproof captures the run at the model boundary - every request and
+every tool-call decision - and serves it back on later runs, so replay makes
+**zero LLM calls**. You assert which tools were called, with which arguments,
+in which order, and which were not. The same engine drives web, desktop and
+Citrix.
 
 **Agents author, a deterministic engine executes.** A flow is described in
 YAML with natural-language steps, recorded once against the live app, and

{flowproof-0.6.0 → flowproof-0.6.1}/README.md

@@ -1,8 +1,11 @@
 # flowproof
 
-A generic open-source automation framework for the AI-agent era:
-automated testing and agentic process automation across web, desktop,
-and Citrix.
+Run your AI agent once, keep the recording, and assert against it from then
+on. flowproof captures the run at the model boundary - every request and
+every tool-call decision - and serves it back on later runs, so replay makes
+**zero LLM calls**. You assert which tools were called, with which arguments,
+in which order, and which were not. The same engine drives web, desktop and
+Citrix.
 
 **Agents author, a deterministic engine executes.** A flow is described in
 YAML with natural-language steps, recorded once against the live app, and

flowproof-0.6.1/crates/flowproof-cli/tests/examples_resolve.rs

@@ -0,0 +1,154 @@
+//! Shipped examples must stay honest: every step parses and resolves
+//! through the deterministic rules — no live system needed, no model
+//! backend. This is the same role `documented_grammar_examples_all_
+//! resolve` plays for docs/authoring.md.
+
+use flowproof_agent::{FlowSpec, SuiteManifest};
+
+const FIORI_SPEC: &str = include_str!("../../../examples/fiori/manage-info-records.flow.yaml");
+const FIORI_SUITE: &str = include_str!("../../../examples/fiori/suite.yaml");
+const CONN_TEST_SPEC: &str = include_str!("../../../examples/api/connection-test.flow.yaml");
+/// The npm-path agent quickstart. It is the first example a reader coming
+/// from `npx flowproof` runs, so it has to keep parsing.
+const AGENT_NODE_SPEC: &str = include_str!("../../../examples/agent-demo/weather-node.flow.yaml");
+const AGENT_PY_SPEC: &str = include_str!("../../../examples/agent-demo/weather.flow.yaml");
+
+#[test]
+fn connection_test_example_resolves_with_body_and_headers() {
+    let spec = FlowSpec::parse(CONN_TEST_SPEC).expect("example parses");
+    assert_eq!(spec.app.id(), "api");
+    let actions =
+        flowproof_agent::rules::resolve_step(spec.app.id(), &spec.steps[0]).expect("step resolves");
+    let flowproof_agent::rules::ResolvedAction::AssertApi {
+        headers,
+        body,
+        status,
+        ..
+    } = &actions[0]
+    else {
+        panic!("expected AssertApi");
+    };
+    // Raw refs in the resolved action — resolution is probe-time only.
+    assert_eq!(
+        headers.get("Authorization").map(String::as_str),
+        Some("Bearer ${DM_SESSION_TOKEN}")
+    );
+    let body = body.as_ref().expect("body present");
+    assert_eq!(body["connectionString"], "${TEST_CONN_STRING}");
+    // Pinned to the real DataMaker /connections/test contract the example
+    // mirrors (the api_pipeline mock speaks the same shape): the field is
+    // `type`, and an unsupported provider answers 500. Drift fails here.
+    assert_eq!(body["type"], "postgres");
+    assert_eq!(*status, Some(500));
+}
+
+#[test]
+fn fiori_example_resolves_entirely_via_rules() {
+    let spec = FlowSpec::parse(FIORI_SPEC).expect("example parses");
+    assert_eq!(spec.app.id(), "web", "Fiori is a browser app");
+    assert!(
+        spec.url
+            .as_deref()
+            .unwrap_or_default()
+            .contains("${FIORI_BASE_URL}"),
+        "launch URL stays a ${{VAR}} reference"
+    );
+    for step in &spec.steps {
+        let actions = flowproof_agent::rules::resolve_step(spec.app.id(), step)
+            .unwrap_or_else(|e| panic!("step '{}' must resolve via rules: {e}", step.intent()));
+        assert!(
+            !actions.is_empty(),
+            "step '{}' yields actions",
+            step.intent()
+        );
+    }
+}
+
+#[test]
+fn fiori_suite_manifest_declares_the_data_leg() {
+    let manifest: SuiteManifest = serde_yaml::from_str(FIORI_SUITE).expect("suite.yaml parses");
+    let cmd = manifest.env_from.expect("env_from present");
+    assert!(
+        cmd.contains("datamaker"),
+        "data comes from the DataMaker CLI"
+    );
+    assert!(manifest.env.contains_key("FIORI_BASE_URL"));
+}
+
+/// The quickstart's two agent demos must stay runnable and stay TWINS: the
+/// docs present them as the same flow in two languages, so a change to one
+/// that is not mirrored in the other makes the documentation lie.
+#[test]
+fn both_agent_demos_resolve_and_assert_the_same_thing() {
+    let node = FlowSpec::parse(AGENT_NODE_SPEC).expect("node example parses");
+    let py = FlowSpec::parse(AGENT_PY_SPEC).expect("python example parses");
+
+    for spec in [&node, &py] {
+        assert_eq!(spec.app.id(), "agent");
+        // A mocked tool is what makes replay deterministic here; without a
+        // result the demo would depend on a live timestamp.
+        let mocked = spec
+            .tools
+            .iter()
+            .find(|t| t.name == "get_weather")
+            .expect("get_weather declared");
+        assert!(!mocked.result.is_null(), "get_weather must carry a result:");
+    }
+
+    // Same trajectory, same assertions - only the runtime differs.
+    assert_eq!(
+        node.steps.len(),
+        py.steps.len(),
+        "the two demos must stay the same flow"
+    );
+
+    // The Node demo must not need Python, which is the entire reason it
+    // exists: the npm install path has to stand on its own.
+    let command = node
+        .agent
+        .as_ref()
+        .and_then(|a| a.command.clone())
+        .expect("node demo has a command");
+    assert!(
+        command.starts_with("node "),
+        "the npm-path demo must run under node, got: {command}"
+    );
+    assert!(
+        !command.contains("python"),
+        "the npm-path demo must not need Python: {command}"
+    );
+}
+
+/// The quickstart is the most-read page we have and it is now the front
+/// door for the npm audience, so its YAML must not drift from the file it
+/// claims to quote. A reader who copies a block that no longer parses is
+/// the worst possible first experience.
+#[test]
+fn the_quickstart_quotes_the_shipped_agent_example_verbatim() {
+    const DOC: &str = include_str!("../../../docs/getting-started.md");
+
+    // Pull the fenced block that names the example file.
+    let marker = "```yaml\n# examples/agent-demo/weather-node.flow.yaml\n";
+    let start = DOC
+        .find(marker)
+        .expect("quickstart quotes the node example")
+        + marker.len();
+    let block = &DOC[start..];
+    let block = &block[..block.find("```").expect("fence closes")];
+
+    // The shipped file, minus its comment header.
+    let shipped: String = AGENT_NODE_SPEC
+        .lines()
+        .filter(|l| !l.starts_with('#'))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    assert_eq!(
+        block.trim(),
+        shipped.trim(),
+        "docs/getting-started.md has drifted from examples/agent-demo/weather-node.flow.yaml"
+    );
+
+    // And it must still parse, so the block a reader copies actually runs.
+    FlowSpec::parse(block).expect("the quoted quickstart block parses");
+}

{flowproof-0.6.0 → flowproof-0.6.1}/crates/flowproof-trace/src/cassette.rs

@@ -203,11 +203,35 @@ fn message_divergence(recorded: &[Message], incoming: &[Message]) -> Option<Stri
                     .collect::<Vec<_>>()
                     .join(", ")
             };
+            // When the SAME tools were called and only their arguments
+            // moved, the two name lists are identical and printing them
+            // says nothing at all - the reader is told something changed
+            // and left to find it. Name the argument PATH instead.
+            let (want_names, got_names) = (names(&want.tool_calls), names(&got.tool_calls));
+            if want_names == got_names {
+                let changes: Vec<String> = want
+                    .tool_calls
+                    .iter()
+                    .zip(&got.tool_calls)
+                    .flat_map(|(a, b)| {
+                        crate::cassette_diff::argument_changes(a, b)
+                            .into_iter()
+                            .map(move |(path, before, after)| {
+                                format!("{}.{path}: recorded {before}, replayed {after}", a.name)
+                            })
+                    })
+                    .collect();
+                if !changes.is_empty() {
+                    return Some(format!(
+                        "message {i} ({}) tool call arguments changed\n  {}",
+                        want.role,
+                        changes.join("\n  "),
+                    ));
+                }
+            }
             return Some(format!(
                 "message {i} ({}) tool calls changed\n  recorded: [{}]\n  replayed: [{}]",
-                want.role,
-                names(&want.tool_calls),
-                names(&got.tool_calls),
+                want.role, want_names, got_names,
             ));
         }
         return Some(format!("message {i} ({}) changed", want.role));
@@ -380,6 +404,68 @@ mod tests {
         }
     }
 
+    /// An argument-only change used to print two IDENTICAL tool-name lists
+    /// ("recorded: [book] / replayed: [book]"), telling the reader that
+    /// something moved but not what. It names the PATH now.
+    #[test]
+    fn an_argument_only_change_names_the_path_that_moved() {
+        let recorded = assistant_with(vec![call("book", r#"{"flight":{"id":"KQ311"},"seats":2}"#)]);
+        let replayed = assistant_with(vec![call("book", r#"{"flight":{"id":"KQ999"},"seats":2}"#)]);
+        let detail =
+            message_divergence(&[recorded], &[replayed]).expect("a change is a divergence");
+        assert!(
+            detail.contains("book.flight.id"),
+            "names the path: {detail}"
+        );
+        assert!(
+            detail.contains("KQ311"),
+            "shows what was recorded: {detail}"
+        );
+        assert!(
+            detail.contains("KQ999"),
+            "shows what was replayed: {detail}"
+        );
+        // The unchanged argument must not be listed as noise.
+        assert!(!detail.contains("seats"), "only the moved path: {detail}");
+    }
+
+    /// An argument that APPEARS is drift too, and the most likely shape of
+    /// a real regression: an agent starts passing something extra.
+    #[test]
+    fn an_added_argument_is_reported_as_absent_before() {
+        let recorded = assistant_with(vec![call("book", r#"{"id":"KQ311"}"#)]);
+        let replayed = assistant_with(vec![call("book", r#"{"id":"KQ311","override":true}"#)]);
+        let detail =
+            message_divergence(&[recorded], &[replayed]).expect("an added argument diverges");
+        assert!(detail.contains("book.override"), "{detail}");
+        assert!(detail.contains("absent"), "{detail}");
+    }
+
+    /// A DIFFERENT tool is a different failure and must keep the name lists,
+    /// which are the useful thing in that case.
+    #[test]
+    fn a_different_tool_still_reports_the_names() {
+        let recorded = assistant_with(vec![call("book", "{}")]);
+        let replayed = assistant_with(vec![call("cancel", "{}")]);
+        let detail =
+            message_divergence(&[recorded], &[replayed]).expect("a different tool diverges");
+        assert!(detail.contains("tool calls changed"), "{detail}");
+        assert!(
+            detail.contains("book") && detail.contains("cancel"),
+            "{detail}"
+        );
+    }
+
+    /// Arguments that are not JSON cannot be diffed field by field. Say the
+    /// whole thing moved rather than pretending to be precise about it.
+    #[test]
+    fn unparseable_arguments_report_the_whole_payload() {
+        let recorded = assistant_with(vec![call("book", "not json")]);
+        let replayed = assistant_with(vec![call("book", "also not json")]);
+        let detail = message_divergence(&[recorded], &[replayed]).expect("still a divergence");
+        assert!(detail.contains("book.arguments"), "{detail}");
+    }
+
     /// A two-turn booking trajectory: the model asks for a tool, the tool
     /// result comes back, the model replies.
     fn booking() -> Cassette {

{flowproof-0.6.0 → flowproof-0.6.1}/crates/flowproof-trace/src/cassette_diff.rs

@@ -271,7 +271,10 @@ fn prompt_change(before: &[Message], after: &[Message]) -> Option<String> {
 ///
 /// Field-level on purpose. Two JSON blobs side by side make a reviewer do
 /// the diffing; `flight.id KQ311 -> KQ999` is the finding itself.
-fn argument_changes(before: &ToolCall, after: &ToolCall) -> Vec<(String, String, String)> {
+pub(crate) fn argument_changes(
+    before: &ToolCall,
+    after: &ToolCall,
+) -> Vec<(String, String, String)> {
     let (Some(old), Some(new)) = (before.arguments_json(), after.arguments_json()) else {
         // Unparseable arguments cannot be compared field by field. Say
         // the whole thing moved rather than pretending to be precise.

{flowproof-0.6.0 → flowproof-0.6.1}/flowproof/__init__.py

@@ -24,7 +24,7 @@ from flowproof.flow import (
     run,
 )
 
-__version__ = "0.6.0"
+__version__ = "0.6.1"
 
 __all__ = [
     "ClarificationNeeded",

{flowproof-0.6.0 → flowproof-0.6.1}/pyproject.toml

@@ -4,8 +4,8 @@ build-backend = "maturin"
 
 [project]
 name = "flowproof"
-version = "0.6.0"
-description = "Generic open-source automation framework for the AI-agent era: automated testing and agentic process automation across web, desktop, and Citrix."
+version = "0.6.1"
+description = "Deterministic tests for AI agents: record a run once, replay it with zero LLM calls. Assert which tools an agent called, with which arguments, in which order."
 readme = "README.md"
 license = "Apache-2.0"
 requires-python = ">=3.9"

flowproof-0.6.0/crates/flowproof-cli/tests/examples_resolve.rs

@@ -1,72 +0,0 @@
-//! Shipped examples must stay honest: every step parses and resolves
-//! through the deterministic rules — no live system needed, no model
-//! backend. This is the same role `documented_grammar_examples_all_
-//! resolve` plays for docs/authoring.md.
-
-use flowproof_agent::{FlowSpec, SuiteManifest};
-
-const FIORI_SPEC: &str = include_str!("../../../examples/fiori/manage-info-records.flow.yaml");
-const FIORI_SUITE: &str = include_str!("../../../examples/fiori/suite.yaml");
-const CONN_TEST_SPEC: &str = include_str!("../../../examples/api/connection-test.flow.yaml");
-
-#[test]
-fn connection_test_example_resolves_with_body_and_headers() {
-    let spec = FlowSpec::parse(CONN_TEST_SPEC).expect("example parses");
-    assert_eq!(spec.app.id(), "api");
-    let actions =
-        flowproof_agent::rules::resolve_step(spec.app.id(), &spec.steps[0]).expect("step resolves");
-    let flowproof_agent::rules::ResolvedAction::AssertApi {
-        headers,
-        body,
-        status,
-        ..
-    } = &actions[0]
-    else {
-        panic!("expected AssertApi");
-    };
-    // Raw refs in the resolved action — resolution is probe-time only.
-    assert_eq!(
-        headers.get("Authorization").map(String::as_str),
-        Some("Bearer ${DM_SESSION_TOKEN}")
-    );
-    let body = body.as_ref().expect("body present");
-    assert_eq!(body["connectionString"], "${TEST_CONN_STRING}");
-    // Pinned to the real DataMaker /connections/test contract the example
-    // mirrors (the api_pipeline mock speaks the same shape): the field is
-    // `type`, and an unsupported provider answers 500. Drift fails here.
-    assert_eq!(body["type"], "postgres");
-    assert_eq!(*status, Some(500));
-}
-
-#[test]
-fn fiori_example_resolves_entirely_via_rules() {
-    let spec = FlowSpec::parse(FIORI_SPEC).expect("example parses");
-    assert_eq!(spec.app.id(), "web", "Fiori is a browser app");
-    assert!(
-        spec.url
-            .as_deref()
-            .unwrap_or_default()
-            .contains("${FIORI_BASE_URL}"),
-        "launch URL stays a ${{VAR}} reference"
-    );
-    for step in &spec.steps {
-        let actions = flowproof_agent::rules::resolve_step(spec.app.id(), step)
-            .unwrap_or_else(|e| panic!("step '{}' must resolve via rules: {e}", step.intent()));
-        assert!(
-            !actions.is_empty(),
-            "step '{}' yields actions",
-            step.intent()
-        );
-    }
-}
-
-#[test]
-fn fiori_suite_manifest_declares_the_data_leg() {
-    let manifest: SuiteManifest = serde_yaml::from_str(FIORI_SUITE).expect("suite.yaml parses");
-    let cmd = manifest.env_from.expect("env_from present");
-    assert!(
-        cmd.contains("datamaker"),
-        "data comes from the DataMaker CLI"
-    );
-    assert!(manifest.env.contains_key("FIORI_BASE_URL"));
-}