calc-flow-python 2026.9.25__tar.gz → 2026.10.12__tar.gz

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/Cargo.lock

@@ -553,16 +553,20 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "calc-flow"
-version = "2026.9.25"
+version = "2026.10.12"
 dependencies = [
+ "ahash 0.8.12",
  "allocation-counter",
+ "arrow-data",
  "async-trait",
  "chrono",
  "command-group",
  "criterion",
  "datafusion",
+ "flatbuffers",
  "fs2",
  "futures",
+ "hashbrown 0.17.1",
  "hex",
  "parking_lot",
  "proptest",
@@ -582,7 +586,7 @@ dependencies = [
 
 [[package]]
 name = "calc-flow-connectors"
-version = "2026.9.25"
+version = "2026.10.12"
 dependencies = [
  "allocation-counter",
  "arrow",
@@ -622,7 +626,7 @@ dependencies = [
 
 [[package]]
 name = "calc-flow-python"
-version = "2026.9.25"
+version = "2026.10.12"
 dependencies = [
  "async-trait",
  "calc-flow",
@@ -2848,6 +2852,28 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
+[[package]]
+name = "openssl-src"
+version = "300.6.1+3.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.117"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
+dependencies = [
+ "cc",
+ "libc",
+ "openssl-src",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "ordered-float"
 version = "2.10.1"
@@ -3560,6 +3586,7 @@ dependencies = [
  "libc",
  "libz-sys",
  "num_enum",
+ "openssl-sys",
  "pkg-config",
 ]
 

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/Cargo.toml

@@ -3,13 +3,14 @@ members = ["crates/calc-flow", "crates/calc-flow-connectors", "crates/calc-flow-
 resolver = "3"
 
 [workspace.package]
-version = "2026.9.25"
+version = "2026.10.12"
 edition = "2024"
 rust-version = "1.88.0"
 license = "Apache-2.0"
 repository = "https://github.com/wegamekinglc/calc-flow"
 
 [workspace.dependencies]
+ahash = "0.8.12"
 async-trait = "0.1.89"
 chrono = { version = "0.4.44", features = ["serde"] }
 command-group = "5.0.1"
@@ -19,6 +20,7 @@ futures = "0.3.31"
 futures-util = "0.3.31"
 fs2 = "0.4.3"
 hex = "0.4.3"
+hashbrown = "=0.17.1"
 parking_lot = "0.12.5"
 proptest = "1.7.0"
 regex = "1.12.2"

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: calc-flow-python
-Version: 2026.9.25
+Version: 2026.10.12
 Requires-Dist: pydantic>=2.12.0,<3
 Requires-Dist: pyarrow>=21.0.0,<22 ; python_full_version < '3.10'
 Requires-Dist: pyarrow>=24.0.0,<25 ; python_full_version >= '3.10'
@@ -17,7 +17,6 @@ Requires-Dist: psutil>=6.1 ; extra == 'benchmark'
 Requires-Dist: py-cpuinfo>=9 ; extra == 'benchmark'
 Requires-Dist: pytest>=8 ; extra == 'benchmark'
 Requires-Dist: pytest-benchmark>=5.2 ; extra == 'benchmark'
-Requires-Dist: hypothesis>=6.100 ; extra == 'dev'
 Requires-Dist: jax>=0.4 ; extra == 'dev'
 Requires-Dist: maturin>=1.14.1,<2.0 ; extra == 'dev'
 Requires-Dist: numpy>=2.0.0 ; extra == 'dev'
@@ -169,8 +168,12 @@ Python package is not a second engine.
   deep-copied strict-JSON settings and a cooperative, timezone-aware deadline
   normalized to UTC.
 - `TableExpr.stream` and `Program.stream` own a single native job with bounded
-  backpressure. SQL accepts one alias in a stream and runs per native batch;
-  SQL aggregation, sorting, and limits do not span batches.
+  backpressure. SQL accepts one alias in a stream. Ordinary aggregates and
+  `GROUP BY` emit full cumulative snapshots; other SQL runs per batch.
+- `Program(name, engine="sql")` or `Program(name, engine="streaming")` fixes
+  the engine when the immutable program is created. `Program.execute` returns
+  finite named tables or owned streaming results, respectively. Calls for the
+  other engine fail.
 - The source-driven `StreamingRunner` consumes a `StreamExecutionPlan`, owns
   async source/sink bindings, and returns a one-owner `StreamingJob`.
 - Managed epoch checkpoints use `LocalStateBackend` segments and strict v3
@@ -248,12 +251,12 @@ Large Cargo and Maturin outputs should use the repository `target/` tree.
 The complete CI/full-verification command reference is below. Local changes use
 the smallest affected checks under [AGENTS.md Verification](AGENTS.md#verification);
 full regression, Rust 90% coverage, and Studio backend 85% coverage belong to
-CI; routine performance gates run on the benchmark schedule:
+CI; routine benchmark measurements run on their own schedule:
 
 ```bash
 uv sync --extra dev
 cargo fmt --all --check
-cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo clippy --workspace --lib --bins --tests --examples --all-features -- -D warnings
 uv run python scripts/run_rust_tests.py
 CALC_FLOW_CONNECTOR_CONTAINERS=1 \
   CALC_FLOW_KAFKA_BOOTSTRAP=localhost:9092 \

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/README.md

@@ -125,8 +125,12 @@ Python package is not a second engine.
   deep-copied strict-JSON settings and a cooperative, timezone-aware deadline
   normalized to UTC.
 - `TableExpr.stream` and `Program.stream` own a single native job with bounded
-  backpressure. SQL accepts one alias in a stream and runs per native batch;
-  SQL aggregation, sorting, and limits do not span batches.
+  backpressure. SQL accepts one alias in a stream. Ordinary aggregates and
+  `GROUP BY` emit full cumulative snapshots; other SQL runs per batch.
+- `Program(name, engine="sql")` or `Program(name, engine="streaming")` fixes
+  the engine when the immutable program is created. `Program.execute` returns
+  finite named tables or owned streaming results, respectively. Calls for the
+  other engine fail.
 - The source-driven `StreamingRunner` consumes a `StreamExecutionPlan`, owns
   async source/sink bindings, and returns a one-owner `StreamingJob`.
 - Managed epoch checkpoints use `LocalStateBackend` segments and strict v3
@@ -204,12 +208,12 @@ Large Cargo and Maturin outputs should use the repository `target/` tree.
 The complete CI/full-verification command reference is below. Local changes use
 the smallest affected checks under [AGENTS.md Verification](AGENTS.md#verification);
 full regression, Rust 90% coverage, and Studio backend 85% coverage belong to
-CI; routine performance gates run on the benchmark schedule:
+CI; routine benchmark measurements run on their own schedule:
 
 ```bash
 uv sync --extra dev
 cargo fmt --all --check
-cargo clippy --workspace --all-targets --all-features -- -D warnings
+cargo clippy --workspace --lib --bins --tests --examples --all-features -- -D warnings
 uv run python scripts/run_rust_tests.py
 CALC_FLOW_CONNECTOR_CONTAINERS=1 \
   CALC_FLOW_KAFKA_BOOTSTRAP=localhost:9092 \

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/crates/calc-flow/Cargo.toml

@@ -8,13 +8,22 @@ repository.workspace = true
 description = "Rust core for stateful DataFusion calculation pipelines"
 exclude = ["tests/**"]
 
+[features]
+benchmark-internals = []
+benchmark-checkpoint-off = ["benchmark-internals"]
+benchmark-checkpoint-diagnostic = ["benchmark-internals"]
+
 [dependencies]
+ahash.workspace = true
+arrow-data = "=58.3.0"
 async-trait.workspace = true
 chrono.workspace = true
 datafusion.workspace = true
 futures.workspace = true
 fs2.workspace = true
+flatbuffers = "=25.12.19"
 hex.workspace = true
+hashbrown.workspace = true
 parking_lot.workspace = true
 regex.workspace = true
 schemars.workspace = true
@@ -40,6 +49,10 @@ tokio = { workspace = true, features = ["test-util"] }
 name = "core"
 harness = false
 
+[[bench]]
+name = "stream_union"
+harness = false
+
 [[bench]]
 name = "allocation_regression"
 harness = false
@@ -66,3 +79,19 @@ workspace = true
 [[bench]]
 name = "stream_asof_perf"
 harness = false
+
+[[bench]]
+name = "stream_asof_e2e"
+harness = false
+
+[[bench]]
+name = "stream_sql_aggregate"
+harness = false
+
+[[bench]]
+name = "stream_asof_eviction"
+harness = false
+
+[[bench]]
+name = "stream_window_groups"
+harness = false

{calc_flow_python-2026.9.25 → calc_flow_python-2026.10.12}/crates/calc-flow/benches/m4_state_window.rs

@@ -206,7 +206,10 @@ fn window_execution_and_restore(c: &mut Criterion) {
                     .process_data("input", input.clone(), &context, collector)
                     .await
                     .unwrap();
-                black_box(());
+                // Drain every port so each iteration measures one fresh batch.
+                for port in operator.output_ports() {
+                    black_box(collector.drain(port.name()));
+                }
             });
         });
     }
@@ -230,9 +233,101 @@ fn window_execution_and_restore(c: &mut Criterion) {
     });
 }
 
+fn dense_schema() -> Arc<Schema> {
+    Arc::new(Schema::new(vec![
+        Field::new(
+            "event_time",
+            DataType::Timestamp(TimeUnit::Microsecond, None),
+            false,
+        ),
+        Field::new("group", DataType::Int64, false),
+        Field::new("value", DataType::Float64, true),
+        Field::new("ivalue", DataType::Int64, false),
+    ]))
+}
+
+fn dense_input(rows: usize) -> Batch {
+    let n = i64::try_from(rows).unwrap();
+    let record = RecordBatch::try_new(
+        dense_schema(),
+        vec![
+            Arc::new(TimestampMicrosecondArray::from_iter_values(0..n)) as ArrayRef,
+            Arc::new(Int64Array::from_iter_values((0..n).map(|i| i % 16))),
+            Arc::new(
+                (0..n)
+                    .map(|i| (i % 7 != 0).then_some(f64::from(i32::try_from(i).unwrap()) * 0.5))
+                    .collect::<datafusion::arrow::array::Float64Array>(),
+            ),
+            Arc::new(Int64Array::from_iter_values((0..n).map(|i| i * 3 - 7))),
+        ],
+    )
+    .unwrap();
+    Batch::table(vec![record], BatchMetadata::default()).unwrap()
+}
+
+fn dense_operator(hopping: bool) -> WindowAggregateOperator {
+    let spec = if hopping {
+        WindowSpec::hopping(
+            "event_time",
+            Duration::from_micros(1_000),
+            Duration::from_micros(100),
+        )
+        .unwrap()
+    } else {
+        WindowSpec::tumbling("event_time", Duration::from_micros(1_000)).unwrap()
+    }
+    .group_by(["group"])
+    .unwrap()
+    .aggregate(AggregateFunction::Sum, "value", "s")
+    .unwrap()
+    .aggregate(AggregateFunction::Count, "value", "c")
+    .unwrap()
+    .aggregate(AggregateFunction::Min, "ivalue", "mn")
+    .unwrap()
+    .aggregate(AggregateFunction::Max, "value", "mx")
+    .unwrap()
+    .aggregate(AggregateFunction::Avg, "ivalue", "a")
+    .unwrap();
+    WindowAggregateOperator::new("window", dense_schema(), spec).unwrap()
+}
+
+fn dense_window(c: &mut Criterion) {
+    let runtime = tokio::runtime::Runtime::new().unwrap();
+    let job = StreamJobContext::new(
+        1,
+        STATE_PIPELINE_FINGERPRINT,
+        JsonMap::new(),
+        None,
+        CancellationToken::new(),
+    );
+    let input = dense_input(65_536);
+    for (name, hopping) in [("tumbling", false), ("hopping", true)] {
+        let operator = dense_operator(hopping);
+        let collector = EdgeCollector::new(operator.output_ports().to_vec());
+        let state = tokio::sync::Mutex::new((operator, collector));
+        c.bench_function(&format!("dense/{name}_65536_rows_16_groups"), |b| {
+            b.to_async(&runtime).iter(|| async {
+                let context = StreamOperatorContext::new(&job, "window", None);
+                let mut state = state.lock().await;
+                let (operator, collector) = &mut *state;
+                operator.reset().unwrap();
+                operator
+                    .process_data("input", input.clone(), &context, collector)
+                    .await
+                    .unwrap();
+                // Drain every port so each iteration measures one fresh batch.
+                for port in operator.output_ports() {
+                    black_box(collector.drain(port.name()));
+                }
+            });
+        });
+    }
+}
+
 criterion_group!(
     m4_state_window,
     state_backend_io,
-    window_execution_and_restore
+    window_execution_and_restore,
+    dense_window
 );
 criterion_main!(m4_state_window);