calc-flow-python 4.0.0__tar.gz → 2026.9.24__tar.gz

{calc_flow_python-4.0.0 → calc_flow_python-2026.9.24}/Cargo.lock

@@ -553,7 +553,7 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "calc-flow"
-version = "4.0.0"
+version = "2026.9.24"
 dependencies = [
  "allocation-counter",
  "async-trait",
@@ -582,8 +582,9 @@ dependencies = [
 
 [[package]]
 name = "calc-flow-connectors"
-version = "4.0.0"
+version = "2026.9.24"
 dependencies = [
+ "allocation-counter",
  "arrow",
  "arrow-csv",
  "arrow-json",
@@ -591,6 +592,7 @@ dependencies = [
  "bytes",
  "calc-flow",
  "chrono",
+ "clap",
  "datafusion",
  "flate2",
  "futures-util",
@@ -599,6 +601,9 @@ dependencies = [
  "parquet",
  "pgwire-replication",
  "postgres-types",
+ "prost",
+ "prost-reflect",
+ "prost-types",
  "rdkafka",
  "reqwest",
  "rust_decimal",
@@ -617,7 +622,7 @@ dependencies = [
 
 [[package]]
 name = "calc-flow-python"
-version = "4.0.0"
+version = "2026.9.24"
 dependencies = [
  "async-trait",
  "calc-flow",
@@ -631,6 +636,8 @@ dependencies = [
  "pyo3",
  "pyo3-arrow",
  "pyo3-async-runtimes",
+ "pyo3-build-config",
+ "rdkafka",
  "serde_json",
  "tempfile",
  "tokio",
@@ -3157,6 +3164,48 @@ dependencies = [
  "unarray",
 ]
 
+[[package]]
+name = "prost"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1"
+dependencies = [
+ "bytes",
+ "prost-derive",
+]
+
+[[package]]
+name = "prost-derive"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf"
+dependencies = [
+ "anyhow",
+ "itertools 0.14.0",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.118",
+]
+
+[[package]]
+name = "prost-reflect"
+version = "0.16.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "01b80ea363c31af2de2b92e3c07ed1156628f7838c4afb4df75ee78a37fedbd1"
+dependencies = [
+ "prost",
+ "prost-types",
+]
+
+[[package]]
+name = "prost-types"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a"
+dependencies = [
+ "prost",
+]
+
 [[package]]
 name = "ptr_meta"
 version = "0.1.4"
@@ -3713,9 +3762,9 @@ dependencies = [
 
 [[package]]
 name = "rustls"
-version = "0.23.43"
+version = "0.23.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06"
+checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
 dependencies = [
  "aws-lc-rs",
  "log",

{calc_flow_python-4.0.0 → calc_flow_python-2026.9.24}/Cargo.toml

@@ -3,7 +3,7 @@ members = ["crates/calc-flow", "crates/calc-flow-connectors", "crates/calc-flow-
 resolver = "3"
 
 [workspace.package]
-version = "4.0.0"
+version = "2026.9.24"
 edition = "2024"
 rust-version = "1.88.0"
 license = "Apache-2.0"

{calc_flow_python-4.0.0 → calc_flow_python-2026.9.24}/PKG-INFO

@@ -1,9 +1,12 @@
 Metadata-Version: 2.4
 Name: calc-flow-python
-Version: 4.0.0
+Version: 2026.9.24
 Requires-Dist: pydantic>=2.12.0,<3
-Requires-Dist: pyarrow>=24.0.0,<25
-Requires-Dist: jax>=0.4 ; extra == 'array'
+Requires-Dist: pyarrow>=21.0.0,<22 ; python_full_version < '3.10'
+Requires-Dist: pyarrow>=24.0.0,<25 ; python_full_version >= '3.10'
+Requires-Dist: typing-extensions>=4.12
+Requires-Dist: jax>=0.4.30,<0.4.31 ; python_full_version < '3.10' and extra == 'array'
+Requires-Dist: jax>=0.4 ; python_full_version >= '3.10' and extra == 'array'
 Requires-Dist: numpy>=2.0.0 ; extra == 'array'
 Requires-Dist: ta-lib==0.7.1 ; extra == 'benchmark'
 Requires-Dist: datafusion==54.0.0 ; extra == 'benchmark'
@@ -24,7 +27,8 @@ Requires-Dist: pytest>=8 ; extra == 'dev'
 Requires-Dist: pytest-cov>=7.1 ; extra == 'dev'
 Requires-Dist: pytest-xdist>=3.6 ; extra == 'dev'
 Requires-Dist: ruff==0.16.0 ; extra == 'dev'
-Requires-Dist: jax>=0.4 ; extra == 'jax'
+Requires-Dist: jax>=0.4.30,<0.4.31 ; python_full_version < '3.10' and extra == 'jax'
+Requires-Dist: jax>=0.4 ; python_full_version >= '3.10' and extra == 'jax'
 Requires-Dist: numpy>=2.0.0 ; extra == 'numpy'
 Provides-Extra: array
 Provides-Extra: benchmark
@@ -32,9 +36,9 @@ Provides-Extra: dev
 Provides-Extra: jax
 Provides-Extra: numpy
 License-File: LICENSE
-Summary: Rust-native DataFusion batch and streaming calculation pipelines
+Summary: Python expression API for batch and streaming calculations on a Rust runtime
 License-Expression: Apache-2.0
-Requires-Python: >=3.13
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Repository, https://github.com/wegamekinglc/calc-flow
 
@@ -42,18 +46,19 @@ Project-URL: Repository, https://github.com/wegamekinglc/calc-flow
 
 [![Linux CI](https://github.com/wegamekinglc/calc-flow/actions/workflows/ci-linux.yml/badge.svg?branch=main)](https://github.com/wegamekinglc/calc-flow/actions/workflows/ci-linux.yml)
 [![Windows CI](https://github.com/wegamekinglc/calc-flow/actions/workflows/ci-windows.yml/badge.svg?branch=main)](https://github.com/wegamekinglc/calc-flow/actions/workflows/ci-windows.yml)
+[![Scheduled Benchmarks](https://github.com/wegamekinglc/calc-flow/actions/workflows/benchmark-suite.yml/badge.svg?branch=main)](https://github.com/wegamekinglc/calc-flow/actions/workflows/benchmark-suite.yml)
+[![Codacy Grade](https://app.codacy.com/project/badge/Grade/7416ea8873544e599089cf63aba3717d)](https://app.codacy.com/gh/wegamekinglc/calc-flow/dashboard)
 [![Coverage Status](https://coveralls.io/repos/github/wegamekinglc/calc-flow/badge.svg?branch=main)](https://coveralls.io/github/wegamekinglc/calc-flow?branch=main)
 
-Calc Flow 4.0 is a Rust-native calculation engine for immutable Arrow
-micro-batches and stateful streams. The core crate compiles typed calculation
-graphs, runs every table expression and query with Apache DataFusion, and owns
-checkpoint/recovery semantics. The Python package is a PyO3 binding to that
-engine; it is not a second implementation. Calc Flow Studio remains a separate
-local FastAPI and React application.
+Calc Flow is a Python calculation library for Arrow tables and stateful streams.
+Compose typed expressions, SQL, and reusable Python functions in one pipeline.
+Collect a dataset or iterate its results as data arrives. Rust provides the internal runtime:
+DataFusion table execution, graph compilation, state, checkpoints, and recovery.
+Calc Flow Studio is a separate local FastAPI and React application.
 
 ## Install
 
-Python 3.13 or newer:
+CPython 3.9 or newer:
 
 ```bash
 uv add calc-flow-python
@@ -66,125 +71,54 @@ uv add "calc-flow-python[numpy]"
 uv add "calc-flow-python[jax]"
 ```
 
-Rust:
-
-```toml
-[dependencies]
-calc-flow = "4.0.0"
-```
-
 ## Python quickstart
 
 ```python
-from datetime import UTC, datetime, timedelta
-
 import pyarrow as pa
+import calc_flow as cf
 
-from calc_flow import Batch, ExecutionOptions, PipelineBuilder
-
-batch = Batch.from_pyarrow(pa.table({"a": [1, 3], "b": [2, 4]}))
-plan = (
-    PipelineBuilder("totals").expression("calculate", "total = a + b").compile_batch()
-)
-result = plan.execute(
-    {"input": batch},
-    options=ExecutionOptions(
-        settings={"request": {"source": "readme"}},
-        deadline=datetime.now(UTC) + timedelta(seconds=30),
-    ),
-)
-
-assert result.outputs["output"].to_pyarrow()["total"].to_pylist() == [3, 7]
-```
-
-`PipelineBuilder` is functional: every method returns a new builder and leaves
-its input unchanged. Unconnected input ports become graph inputs; unconnected
-output ports become graph outputs. Use `execute_async()` inside an event loop.
-Both execution forms accept keyword-only, frozen `ExecutionOptions` carrying
-deep-copied strict-JSON settings and an optional timezone-aware deadline that
-is normalized to UTC. Settings may be nested mappings/lists; `settings=None`
-means empty settings.
-
-See [the Python API guide](docs/python-api.md) and the executable
-[examples](examples/README.md) for SQL, Python scalar UDFs, continuous
-execution and recovery, asyncio, and NumPy/JAX.
-The [symbolic workflow guide](docs/symbolic-workflows.md) covers composed
-financial features, checkpoint recovery, static matrices, bounded stream
-joins, capability errors, Studio inspection, and performance output.
-
-## Rust quickstart
-
-The Rust crate exposes the native data, operator, graph, runtime, project, and
-checkpoint types directly. A table `Batch` contains one or more Arrow
-`RecordBatch` values plus immutable metadata. Build a graph with
-`PipelineBuilder`, compile it against a `UdfRegistrySnapshot`, then await
-`BatchExecutionPlan::execute`. The canonical first example is
-[`crates/calc-flow/examples/expression_pipeline.rs`](crates/calc-flow/examples/expression_pipeline.rs),
-a true twin of the Python quickstart:
-
-```rust
-use std::{collections::BTreeMap, sync::Arc};
-
-use calc_flow::{
-    Batch, BatchMetadata, ExecutionOptions, ExpressionOperator, PipelineBuilder, UdfRegistry,
-};
-use datafusion::arrow::{array::Int64Array, record_batch::RecordBatch};
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let plan = PipelineBuilder::new("totals")?
-        .add_node(
-            "calculate",
-            Box::new(ExpressionOperator::new(
-                "calculate",
-                "total = a + b",
-                Vec::new(),
-                None,
-                Vec::new(),
-            )?),
-        )?
-        .compile_batch(&UdfRegistry::new().snapshot())?;
-    let input = RecordBatch::try_from_iter(vec![
-        (
-            "a",
-            Arc::new(Int64Array::from(vec![1, 3])) as Arc<dyn datafusion::arrow::array::Array>,
-        ),
-        ("b", Arc::new(Int64Array::from(vec![2, 4])) as _),
-    ])?;
-    let result = plan
-        .execute(
-            BTreeMap::from([(
-                "input".into(),
-                Batch::table(vec![input], BatchMetadata::default())?,
-            )]),
-            ExecutionOptions::default(),
-        )
-        .await?;
-    let output = result.outputs["output"].table_payload()?;
-    let totals = output.batches()[0]
-        .column_by_name("total")
-        .expect("expression output contains total")
-        .as_any()
-        .downcast_ref::<Int64Array>()
-        .expect("total is an Int64 column");
-
-    assert_eq!(totals.values(), &[3, 7]);
-    println!("calculated totals: {totals:?}");
-    Ok(())
-}
-```
-
-Run the checked examples:
-
-```bash
-cargo run -p calc-flow --example expression_pipeline
-cargo run -p calc-flow --example sql_join
-cargo run -p calc-flow --example continuous_runtime
-cargo run -p calc-flow --example windowed_streaming
+data = pa.table({"a": [1, 3], "b": [2, 4]})
+result = cf.compute(data, lambda t: t.select(total=t["a"] + t["b"]))
+assert result.to_pydict() == {"total": [3, 7]}
 ```
 
-See [the Rust API guide](docs/rust-api.md) for paired source examples and links
-to the public types, or the [Rust examples index](crates/calc-flow/examples/README.md).
+`cf.compute` infers the supported Arrow schema and returns a `pyarrow.Table`.
+The builder receives a `TableExpr`; indexing selects columns, operators compose
+calculations, and `select`, `with_columns`, and `filter` return new declarations.
+Use `await cf.compute_async(...)` in an event loop.
+
+Add a read-only SQL stage with `t.sql(query)`, using the local alias `input`.
+Use `cf.sql(query, orders=orders, fees=fees)` for explicit named table declarations.
+SQL returns another `TableExpr`, so `pipe`, column operators, and table methods
+compose before and after it. `expression.pipe(function, *args, **kwargs)` calls
+an ordinary synchronous function once while building the calculation.
+
+For streams, use `async with output.stream(batches()) as results`, then
+`async for table in results`. One native job retains state across batches and
+owns cleanup. Declared event-time inputs default to validated nondecreasing
+timestamps and watermarks that produce finalized results before EOF. Use
+`watermarks` for explicit disorder or source-provided progress; see
+[watermark policies](docs/streaming-guide.md#watermark-policies).
+A `Program` yields named `StreamOutput` events for multiple
+outputs. The convenience stream uses temporary checkpoints and best-effort
+iterable delivery; durable recovery uses explicit source/sink bindings and a
+managed checkpoint root.
+
+Continue with [SQL and reusable pipelines](docs/batch-guide.md#compose-sql-and-python-pipelines)
+and the [first streaming pipeline](docs/streaming-guide.md#first-python-continuous-job).
+The runnable examples cover [SQL composition](examples/19_sql_expression_pipeline.py),
+[stateful streaming](examples/20_streaming_pipeline.py), and
+[named streaming outputs](examples/21_streaming_outputs.py). The
+[connector examples](docs/connectors/README.md) cover all seven transports
+and every supported read/write direction, with service setup and output checks.
+The default wheel includes file; other transports require native build features.
+[Expression workflows](docs/symbolic-workflows.md) cover financial features,
+recovery, static matrices, and joins. Use explicit `Runtime`, plans, `Batch`,
+UDFs, and `PipelineBuilder` for runtime integrations and diagnostics.
+
+The [Rust runtime reference](docs/rust-api.md) and
+[native examples](crates/calc-flow/examples/README.md) cover implementation and
+extension work. The `calc-flow` crate's exports remain available to those users.
 
 ## Architecture
 
@@ -192,7 +126,7 @@ to the public types, or the [Rust examples index](crates/calc-flow/examples/READ
 crates/calc-flow  (Rust core: Batch, graph compiler, DataFusion, runners, stores)
   ├─ crates/calc-flow-connectors  (trusted transport implementations)
   └─ crates/calc-flow-python  (PyO3 _native binding + registered connectors)
-       └─ python/calc_flow  (pure-Python public API + functional adapters)
+       └─ python/calc_flow  (Python expressions, Arrow execution + integrations)
             └─ web-ui/backend  (calc-flow-studio FastAPI, /api/v3, loopback only)
                   └─ web-ui/src  (React + TypeScript + Vite + React Flow studio, via REST)
 ```
@@ -207,16 +141,17 @@ Python package is not a second engine.
 | `crates/calc-flow/`            | Native core: batches, ports/operators, graph compiler, DataFusion runtime, UDF/provider registries, runners, checkpoints, project stores |
 | `crates/calc-flow-connectors/` | Trusted file, Kafka, PostgreSQL, MySQL, ClickHouse, HTTP, and WebSocket connectors behind feature gates                                  |
 | `crates/calc-flow-python/`     | PyO3 binding exposing the core as `calc_flow._native`                                                                                    |
-| `python/calc_flow/`            | Pure-Python public API, functional `PipelineBuilder`, runner/store adapters, NumPy/JAX provider registration, exception hierarchy        |
+| `python/calc_flow/`            | Python expressions and SQL, `pipe`, Arrow collection, owned stream results, lowering, and runtime integrations                           |
 | `web-ui/backend/`              | `calc-flow-studio` FastAPI service under `/api/v3`, loopback-bound, spawned bounded continuous-job workers                               |
 | `web-ui/src/`                  | React + TypeScript + Vite + React Flow studio; API types generated from `web-ui/openapi.json`                                            |
 | `schemas/`                     | `project-v3.schema.json`, the canonical generated project contract                                                                       |
-| `examples/`                    | Executable v3 Python examples                                                                                                            |
-| `benchmarks/`                  | pytest-benchmark harness (informational)                                                                                                 |
+| `examples/`                    | Executable Python expression and integration examples                                                                                    |
+| `benchmarks/`                  | Benchmark workloads; scheduled regression gates and informational comparisons                                                            |
 
 ## Data and execution model
 
-- Table data is Arrow-backed and calculated only by DataFusion.
+- Table data is Arrow-backed. The Rust runtime executes row expressions and SQL
+  with DataFusion and owns native rolling, window, and cross-section operators.
 - NumPy and JAX are optional Python array providers. They are registered
   explicitly and evaluate a bounded, allowlisted expression language.
 - Raw tables or arrays never cross a graph or runner boundary; they are wrapped
@@ -228,16 +163,20 @@ Python package is not a second engine.
 - Table and mixed graph runs own one run-scoped DataFusion session. External-only
   NumPy/JAX runs own no DataFusion configuration, UDF state, or runtime and
   return an empty DataFusion metrics list.
-- Every graph run returns named outputs, per-node row counts/timings, and run
-  metadata; table work additionally reports DataFusion plans and timings.
+- Convenience batch calls return Arrow tables. Explicit plan execution returns
+  named `Batch` outputs, per-node timings, metadata, and DataFusion diagnostics.
 - Python executions accept reusable frozen `ExecutionOptions` with
   deep-copied strict-JSON settings and a cooperative, timezone-aware deadline
   normalized to UTC.
+- `TableExpr.stream` and `Program.stream` own a single native job with bounded
+  backpressure. SQL accepts one alias in a stream and runs per native batch;
+  SQL aggregation, sorting, and limits do not span batches.
 - The source-driven `StreamingRunner` consumes a `StreamExecutionPlan`, owns
   async source/sink bindings, and returns a one-owner `StreamingJob`.
 - Managed epoch checkpoints use `LocalStateBackend` segments and strict v3
   `CheckpointManifest` documents. Exactly-once compatibility is proved per
-  requested output; ordinary sinks remain at least once.
+  requested output; ordinary sinks can provide at-least-once delivery on a
+  lossless replayable route. Async iterable convenience inputs provide best effort.
 
 The capabilities and execution model are introduced in
 [docs/introduction.md](docs/introduction.md). The complete component and
@@ -252,7 +191,7 @@ input types, return type, and volatility. Graph nodes select registrations
 explicitly with `(provider, name, version)` references. Serialized projects
 contain references only.
 
-Rust applications use `UdfRegistry` for native DataFusion UDFs and
+Runtime extension authors use `UdfRegistry` for native DataFusion UDFs and
 `ProviderRegistry` for explicitly registered external operators.
 
 ## Studio
@@ -298,7 +237,7 @@ response types are in `web-ui/src/api/schema.d.ts`.
 
 ## Project contracts
 
-Calc Flow 4.0 accepts strict project-v3 documents and exposes the Studio
+Calc Flow accepts strict project-v3 documents and exposes the Studio
 `/api/v3` surface. Read [projects and persistence](docs/projects-guide.md)
 for validation, serialization, and reloading a graph. Historical changes are
 recorded in [CHANGELOG.md](CHANGELOG.md).
@@ -306,7 +245,10 @@ recorded in [CHANGELOG.md](CHANGELOG.md).
 ## Development
 
 Large Cargo and Maturin outputs should use the repository `target/` tree.
-A typical local verification sequence is:
+The complete CI/full-verification command reference is below. Local changes use
+the smallest affected checks under [AGENTS.md Verification](AGENTS.md#verification);
+full regression, Rust 90% coverage, and Studio backend 85% coverage belong to
+CI; routine performance gates run on the benchmark schedule:
 
 ```bash
 uv sync --extra dev
@@ -339,10 +281,10 @@ npm run test:e2e
 npm audit --omit=dev
 ```
 
-Release gates also run `cargo audit`, `cargo deny --locked check`, package
-inspectors, isolated wheel smoke tests, `cargo package`, and
-`cargo publish --dry-run`. See [AGENTS.md](AGENTS.md) for the maintained
-repository commands and constraints.
+Regular CI runs `cargo audit`, `cargo deny --locked check`, core package
+inspectors, and isolated wheel smoke tests. The Python release workflow builds,
+verifies, and publishes core package artifacts. See [AGENTS.md](AGENTS.md) for
+the maintained repository commands and constraints.
 
 ## Documentation
 
@@ -353,20 +295,20 @@ repository commands and constraints.
 - **[Batch calculations](docs/batch-guide.md)** — expressions, SQL, UDFs, and async execution
 - **[Arrays and matrices](docs/array-guide.md)** — NumPy/JAX and static weights
 - **[Continuous streaming](docs/streaming-guide.md)** — source-to-recovery tutorial
-- **[Connectors](docs/connectors.md)** — transport configuration and guarantees
+- **[Connectors](docs/connectors/README.md)** — transport configuration and guarantees
 - **[Projects and persistence](docs/projects-guide.md)** — JSON/YAML and file stores
 - **[Studio](docs/studio-guide.md)** — local editing, inspection, and job controls
 - **[Python API](docs/python-api.md)** — Python surface and examples
-- **[Symbolic workflows](docs/symbolic-workflows.md)** — declaration-to-Studio
+- **[Expression workflows](docs/symbolic-workflows.md)** — declaration-to-Studio
   batch, stream, recovery, static matrix, and performance workflows
-- **[Rust API](docs/rust-api.md)** — native surface and examples
+- **[Rust runtime reference](docs/rust-api.md)** — native surface and examples
 - **[API reference](docs/api-reference.md)** — supported surfaces at a glance
-- **[Symbolic API](docs/symbolic-api.md)** — declarations, analysis, and compile requirements
+- **[Expression API](docs/symbolic-api.md)** — declarations, analysis, and compile requirements
 - **[Design and architecture](docs/design.md)** — component ownership and execution design
 - **[Verification](docs/verification.md)** — documentation, examples, and implementation checks
 - **[Python release guide](docs/python-release.md)** — packaging, verification,
   Trusted Publishers, and the PyPI procedure
-- **[Benchmark suite](docs/benchmark-suite.md)** — complete CI tables, scale matrices,
+- **[Benchmark suite](docs/benchmark-suite.md)** — scheduled tables, scale matrices,
   external-engine comparisons and historical regression evidence
 - **[Changelog](CHANGELOG.md)** — the single history of changes