calc-flow-python 2026.9.25__cp310-abi3-win_amd64.whl

calc_flow/_native.pyi

@@ -0,0 +1,387 @@
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from datetime import datetime
+from typing import Literal, Protocol, TypedDict, final
+
+import pyarrow as pa
+from typing_extensions import Never, TypeAlias
+
+JSONValue: TypeAlias = (
+    None | bool | int | float | str | list["JSONValue"] | dict[str, "JSONValue"]
+)
+_JSONInput: TypeAlias = (
+    None | bool | int | float | str | list["_JSONInput"] | Mapping[str, "_JSONInput"]
+)
+StreamingFailureReasonCode: TypeAlias = Literal[
+    "join_state_limit_exceeded",
+    "join_match_limit_exceeded",
+    "join_counter_overflow",
+    "join_time_conversion_failed",
+    "asof_invalid_input",
+    "asof_duplicate_identity",
+    "asof_late_row",
+    "asof_state_limit_exceeded",
+    "asof_workspace_limit_exceeded",
+    "asof_output_limit_exceeded",
+    "asof_counter_overflow",
+    "asof_protocol_error",
+]
+
+class _ArrowCStream(Protocol):
+    def __arrow_c_stream__(
+        self, requested_schema: object | None = None, /
+    ) -> object: ...
+
+class CalcFlowError(Exception): ...
+
+class ConfigError(CalcFlowError):
+    @property
+    def issues(self) -> tuple[dict[str, str], ...]: ...
+
+class CompileError(CalcFlowError): ...
+class ExecutionError(CalcFlowError): ...
+class ProviderError(ExecutionError): ...
+class CheckpointError(CalcFlowError): ...
+class CancelledError(ExecutionError): ...
+
+class StreamingRuntimeError(CalcFlowError):
+    @property
+    def category(self) -> str: ...
+    @property
+    def reason_code(self) -> StreamingFailureReasonCode | None: ...
+    @property
+    def message(self) -> str: ...
+    @property
+    def job_id(self) -> int | None: ...
+    @property
+    def epoch(self) -> int | None: ...
+    @property
+    def checkpoint_phase(self) -> str | None: ...
+    @property
+    def component_kind(self) -> str | None: ...
+    @property
+    def component_id(self) -> str | None: ...
+    @property
+    def diagnostic_id(self) -> int | None: ...
+    @property
+    def position(self) -> int: ...
+
+class CheckpointPublicationUnknownError(StreamingRuntimeError): ...
+
+@final
+class ExecutionOptions:
+    def __init__(
+        self,
+        settings: Mapping[str, _JSONInput] | None = ...,
+        deadline: datetime | None = ...,
+    ) -> None: ...
+    @property
+    def settings(self) -> dict[str, JSONValue]: ...
+    @property
+    def deadline(self) -> datetime | None: ...
+
+@final
+class ProviderContext:
+    def __new__(cls) -> Never:
+        """Engine-created; application construction is not public API."""
+        ...
+    @property
+    def settings(self) -> dict[str, JSONValue]: ...
+    @property
+    def deadline(self) -> datetime | None: ...
+
+class _ExecutionCancellation(Protocol):
+    def cancel(self) -> None: ...
+
+class Batch:
+    @staticmethod
+    def from_pyarrow(
+        table: _ArrowCStream, metadata: Mapping[str, object] | None = None
+    ) -> Batch: ...
+    @staticmethod
+    def from_array(
+        array: object,
+        *,
+        backend: str,
+        metadata: Mapping[str, object] | None = None,
+    ) -> Batch: ...
+    @staticmethod
+    def _from_external(
+        object: object, backend: str, len: int, metadata: Mapping[str, object]
+    ) -> Batch:
+        """Build an internal Python-provider batch; this is not a stable public API."""
+        ...
+    @staticmethod
+    def _new_owned_numpy(shape: Sequence[int], dtype: str) -> tuple[object, object]:
+        """Allocate private Rust-owned NumPy storage."""
+        ...
+    @staticmethod
+    def _from_owned_array(
+        array: object,
+        *,
+        backend: str,
+        token: object | None,
+        metadata: Mapping[str, object],
+    ) -> Batch:
+        """Adopt a trusted provider result; this is not a stable public API."""
+        ...
+
+    def to_pyarrow(self) -> pa.Table: ...
+    @property
+    def array(self) -> object: ...
+    @property
+    def backend(self) -> str: ...
+    @property
+    def kind(self) -> str: ...
+    @property
+    def num_rows(self) -> int: ...
+    @property
+    def metadata(self) -> dict[str, object]: ...
+
+class Runtime:
+    def __init__(self) -> None: ...
+    def register_provider(
+        self,
+        provider: str,
+        name: str,
+        version: str,
+        callback: object,
+        *,
+        accepts_context: bool = False,
+    ) -> None: ...
+    def _register_mapping_provider(
+        self,
+        provider: str,
+        name: str,
+        version: str,
+        callback: object,
+        *,
+        input_ports: Sequence[tuple[str, str]],
+        output_ports: Sequence[tuple[str, str]],
+        accepts_context: bool = False,
+    ) -> None: ...
+    def _register_stateless_stream_provider(
+        self,
+        provider: str,
+        name: str,
+        version: str,
+        callback: object,
+        *,
+        microbatch_invariant: bool,
+        deterministic: bool,
+        replay_safe: bool,
+    ) -> None: ...
+    def _register_stateless_stream_mapping_provider(
+        self,
+        provider: str,
+        name: str,
+        version: str,
+        callback: object,
+        *,
+        input_ports: Sequence[tuple[str, str]],
+        output_ports: Sequence[tuple[str, str]],
+        microbatch_invariant: bool,
+        deterministic: bool,
+        replay_safe: bool,
+    ) -> None: ...
+    def register_scalar_udf(
+        self,
+        *,
+        provider: str,
+        name: str,
+        version: str,
+        input_types: Sequence[str],
+        return_type: str,
+        volatility: str,
+        function: Callable[..., object],
+    ) -> None: ...
+    def register_kafka_decoder(
+        self,
+        *,
+        name: str,
+        version: str,
+        function: Callable[[bytes], object],
+    ) -> None: ...
+    def catalog(self) -> list[dict[str, object]]: ...
+    def _infer_sql_schema(
+        self, query: str, schemas: Mapping[str, pa.Schema]
+    ) -> pa.Schema: ...
+    def validation_report(self, project_json: str) -> dict[str, object]: ...
+    def compile_project(self, project_json: str) -> ExecutionPlan: ...
+    def compile_stream_project(
+        self, project_json: str, delivery: Mapping[str, str]
+    ) -> StreamExecutionPlan: ...
+    def _compile_stream_graph_project(
+        self, project_json: str, delivery: Mapping[str, str]
+    ) -> StreamExecutionPlan: ...
+
+class ExecutionPlan:
+    @property
+    def name(self) -> str: ...
+    @property
+    def fingerprint(self) -> str: ...
+    def execute(
+        self,
+        inputs: dict[str, Batch],
+        *,
+        options: ExecutionOptions | None = None,
+    ) -> RunResult: ...
+    def execute_async(
+        self,
+        inputs: dict[str, Batch],
+        *,
+        options: ExecutionOptions | None = None,
+    ) -> Awaitable[RunResult]: ...
+    def _execute_async_cancellable(
+        self,
+        inputs: dict[str, Batch],
+        *,
+        options: ExecutionOptions | None = None,
+    ) -> tuple[asyncio.Future[RunResult], _ExecutionCancellation]: ...
+    def snapshot_async(self) -> Awaitable[dict[str, object]]: ...
+    def restore_async(self, state_json: str) -> Awaitable[None]: ...
+    def reset_async(self) -> Awaitable[None]: ...
+
+class StreamExecutionPlan:
+    @property
+    def name(self) -> str: ...
+    @property
+    def fingerprint(self) -> str: ...
+    @property
+    def source_binding_ids(self) -> tuple[str, ...]: ...
+    @property
+    def static_input_ids(self) -> tuple[str, ...]: ...
+    @property
+    def sink_binding_ids(self) -> tuple[str, ...]: ...
+    @property
+    def requirements(self) -> dict[str, str]: ...
+
+@final
+class _ManagedCheckpointRuntime:
+    def __init__(self, directory: str, /) -> None: ...
+
+@final
+class _StreamingRunner:
+    def __init__(
+        self,
+        plan: StreamExecutionPlan,
+        sources: Mapping[str, object],
+        sinks: Mapping[str, Sequence[object]],
+        checkpoints: _ManagedCheckpointRuntime,
+        config: Mapping[str, int],
+        static_inputs: Mapping[str, object],
+    ) -> None: ...
+    def start_async(self) -> Awaitable[_StreamingJob]: ...
+    def _wait_start_cleanup_async(self) -> Awaitable[None]: ...
+    def _release_roots(self) -> None: ...
+
+@final
+class _StreamingJob:
+    @property
+    def id(self) -> int: ...
+    def status(self) -> dict[str, object]: ...
+    def trigger_checkpoint_async(self) -> Awaitable[int]: ...
+    def shutdown_async(self) -> Awaitable[dict[str, object]]: ...
+    def cancel_async(self) -> Awaitable[dict[str, object]]: ...
+    def wait_async(self) -> Awaitable[dict[str, object]]: ...
+    def _release_roots(self) -> None: ...
+    def _enable_callback_profiling(self) -> None: ...
+    def _take_callback_profile(self) -> str: ...
+
+class _FileProjectStore:
+    def __init__(self, directory: str) -> None: ...
+    def create(self, project_json: str) -> Awaitable[None]: ...
+    def put(self, project_json: str) -> Awaitable[None]: ...
+    def get(self, project_id: str) -> Awaitable[str]: ...
+    def list(self) -> Awaitable[list[str]]: ...
+    def delete(self, project_id: str) -> Awaitable[None]: ...
+
+def import_project_json(document: bytes) -> str: ...
+def import_project_yaml(document: bytes) -> str: ...
+def export_project_json(project_json: str) -> str: ...
+def export_project_yaml(project_json: str) -> str: ...
+
+class _NodeTiming(TypedDict):
+    duration_ns: int
+    input_rows: dict[str, int]
+    output_rows: dict[str, int]
+
+class _DataFusionMetric(TypedDict):
+    query_id: int
+    node_id: str | None
+    runtime_acquire_ns: int
+    session_state_create_ns: int
+    input_adapter_ns: int
+    table_register_ns: int
+    sql_parse_ns: int
+    logical_planning_ns: int
+    physical_planning_ns: int
+    physical_planning_count: int
+    planning_ns: int
+    stream_open_ns: int
+    execution_to_first_batch_ns: int
+    execution_remaining_ns: int
+    execution_ns: int
+    collect_ns: int
+    output_arrow_wrap_ns: int
+    audit_ns: int
+    metrics_traversal_ns: int
+    logical_plan_string_ns: int
+    physical_plan_string_ns: int
+    batch_envelope_ns: int
+    run_result_ns: int
+    physical_metric_count: int
+    output_partition_count: int
+    output_partition_rows: list[int]
+    window_partition_count: int
+    window_partition_rows: list[int]
+    spill_bytes: int
+    elapsed_compute_ns: int
+    window_compute_ns: int
+    repartition_sort_compute_ns: int
+    window_operator_count: int
+    repartition_operator_count: int
+    sort_operator_count: int
+    coalesce_operator_count: int
+    output_rows: int
+    configured_batch_size: int
+    parallelism_mode: Literal["fixed", "auto"]
+    configured_target_partitions: int
+    requested_target_partitions: int
+    effective_target_partitions: int
+    available_parallelism: int
+    max_partitions: int
+    min_rows_per_partition: int
+    small_rows_threshold: int
+    parallelism_decision_reused: bool
+    decision_input_rows: int
+    decision_active_entities: int | None
+    decision_active_entities_source: str
+    input_rows: int
+    active_entities: int | None
+    active_entities_source: str
+    partition_limit_reason: str
+    rolling_rewrite_enabled: bool
+    diagnostics_collected: bool
+    rolling_candidate_windows: int
+    rolling_rewritten_windows: int
+    rolling_fallback_reasons: list[str]
+    logical_plan: str
+    physical_plan: str
+
+class RunResult:
+    @property
+    def outputs(self) -> dict[str, Batch]: ...
+    @property
+    def metadata(self) -> dict[str, str]: ...
+    @property
+    def node_timings(self) -> dict[str, _NodeTiming]: ...
+    @property
+    def datafusion_metrics(self) -> list[_DataFusionMetric]: ...
+
+def project_json_schema() -> str: ...
+def validate_project_json(project_json: str) -> str: ...
+def version() -> str: ...
+def registered_connectors() -> list[dict[str, object]]: ...

calc_flow/_stream_inputs.py

@@ -0,0 +1,233 @@
+"""Logical input policies and owned iterable adapters for expression streams."""
+
+from __future__ import annotations
+
+from collections.abc import AsyncIterable, AsyncIterator, Mapping
+from datetime import timedelta
+from typing import NoReturn, Union
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+from calc_flow._compat import TypeAliasType, dataclass
+from calc_flow.compute import TableData, _arrow_table, _table_batch
+from calc_flow.runtime import (
+    BoundedOutOfOrderness,
+    Cursor,
+    Data,
+    DisabledWatermarks,
+    EdgeBudget,
+    NativeWatermarkCapability,
+    ReplayPositioning,
+    SourceBinding,
+    SourceCapabilities,
+    SourceDeliveryCapability,
+    SourceProvidedWatermarks,
+    Watermark,
+    WatermarkPolicy,
+)
+from calc_flow.symbolic.analyzer import _schema_fields
+from calc_flow.symbolic.expr import Parameter, TableExpr
+from calc_flow.symbolic.lower.schema import _arrow_schema
+from calc_flow.symbolic.nodes import CStr
+
+StreamInput = TypeAliasType(
+    "StreamInput", Union[AsyncIterable[Union[TableData, Watermark]], SourceBinding]
+)
+
+_POLICIES = (SourceProvidedWatermarks, BoundedOutOfOrderness, DisabledWatermarks)
+
+
+def _capture_watermarks(
+    watermarks: WatermarkPolicy | Mapping[str, WatermarkPolicy] | None,
+) -> WatermarkPolicy | dict[str, WatermarkPolicy] | None:
+    return dict(watermarks) if isinstance(watermarks, Mapping) else watermarks
+
+
+def _policy_mapping(
+    watermarks: WatermarkPolicy | Mapping[str, WatermarkPolicy] | None,
+    dynamic: set[str],
+) -> Mapping[str, WatermarkPolicy]:
+    if watermarks is None:
+        return {}
+    if isinstance(watermarks, Mapping):
+        return watermarks
+    if not isinstance(watermarks, _POLICIES):
+        raise TypeError("stream.watermarks: expected supported watermark policy")
+    if len(dynamic) != 1:
+        raise ValueError("stream.watermarks: multiple inputs require a name mapping")
+    return {next(iter(dynamic)): watermarks}
+
+
+def _selected_policies(
+    inputs: Mapping[str, StreamInput | TableData],
+    expected: Mapping[str, TableExpr | Parameter],
+    watermarks: WatermarkPolicy | Mapping[str, WatermarkPolicy] | None,
+) -> Mapping[str, WatermarkPolicy]:
+    dynamic = {name for name, value in expected.items() if isinstance(value, TableExpr)}
+    selected = _policy_mapping(watermarks, dynamic)
+    for name, policy in selected.items():
+        if name not in dynamic:
+            raise ValueError(f"stream.watermarks.{name}: expected dynamic input name")
+        if not isinstance(policy, _POLICIES):
+            raise TypeError(
+                f"stream.watermarks.{name}: expected supported watermark policy"
+            )
+        if isinstance(inputs[name], SourceBinding):
+            raise ValueError(f"stream.watermarks.{name}: cannot override SourceBinding")
+    return selected
+
+
+@dataclass(frozen=True, slots=True)
+class _IterableConfig:
+    name: str
+    schema: pa.Schema
+    budget: EdgeBudget
+    policy: WatermarkPolicy
+    ordered_column: str | None
+
+
+def _source_config(
+    name: str,
+    value: TableExpr,
+    budget: EdgeBudget,
+    policy: WatermarkPolicy | None,
+) -> _IterableConfig:
+    schema = _arrow_schema(_schema_fields(value._node.attr("schema")))
+    event_time = value._node.attr("event_time")
+    ordered_column = None
+    if policy is None:
+        policy = DisabledWatermarks()
+        if isinstance(event_time, CStr):
+            ordered_column = event_time.value
+            policy = BoundedOutOfOrderness(
+                ordered_column, timedelta(microseconds=1), timedelta(milliseconds=100)
+            )
+    return _IterableConfig(name, schema, budget, policy, ordered_column)
+
+
+class _IterableSource:
+    def __init__(
+        self,
+        source: AsyncIterable[TableData | Watermark],
+        config: _IterableConfig,
+    ) -> None:
+        self._source = source
+        self._config = config
+        self._path = f"stream.inputs.{config.name}"
+        self._iterator: AsyncIterator[TableData | Watermark] | None = None
+        self._closed = False
+        self._position = 0
+        self._last_time: int | None = None
+        self.failure: str | None = None
+
+    def capabilities(self) -> SourceCapabilities:
+        watermarks = NativeWatermarkCapability.NEVER_EMITS
+        if isinstance(self._config.policy, SourceProvidedWatermarks):
+            watermarks = NativeWatermarkCapability.EMITS_NATIVE
+        return SourceCapabilities(
+            ReplayPositioning.UNSUPPORTED,
+            SourceDeliveryCapability.LOSSY,
+            max_batch_rows=self._config.budget.max_rows,
+            max_batch_bytes=self._config.budget.max_bytes,
+            schema=self._config.schema,
+            native_watermarks=watermarks,
+        )
+
+    async def open(self, cursor: Cursor | None) -> None:
+        self._iterator = self._source.__aiter__()
+
+    async def next(self) -> Data | Watermark | None:
+        if self._iterator is None:
+            raise RuntimeError(f"{self._path}: source is not open")
+        try:
+            data = await self._iterator.__anext__()
+        except StopAsyncIteration:
+            return None
+        if isinstance(data, Watermark):
+            return self._watermark(data)
+        self._validate_batch(self._arrow_input(data))
+        batch = _table_batch(data, self._path)
+        self._position += 1
+        return Data(batch, Cursor(self._position.to_bytes(16, "big"), {}))
+
+    def _arrow_input(self, data: TableData) -> pa.Table | pa.RecordBatch:
+        try:
+            return _arrow_table(data, self._path)
+        except TypeError:
+            self._reject(": expected Arrow Table, RecordBatch or table Batch")
+
+    def _watermark(self, value: Watermark) -> Watermark:
+        if not isinstance(self._config.policy, SourceProvidedWatermarks):
+            self._reject(": Watermark requires SourceProvidedWatermarks")
+        return value
+
+    def _validate_batch(self, table: pa.Table | pa.RecordBatch) -> None:
+        if not table.schema.equals(self._config.schema):
+            self._reject(".schema: expected declared Arrow schema")
+        if table.num_rows > self._config.budget.max_rows:
+            self._reject(": batch exceeds edge_budget.max_rows")
+        if table.nbytes > self._config.budget.max_bytes:
+            self._reject(": batch exceeds edge_budget.max_bytes")
+        if self._config.ordered_column is not None:
+            self._validate_time(table[self._config.ordered_column])
+
+    def _validate_time(self, column: pa.ChunkedArray | pa.Array) -> None:
+        if column.null_count:
+            self._reject(".event_time: expected non-null event time")
+        # Compare in the declared unit without datetime/range conversion. Native
+        # generation retains its checked conversion and minimum-minus-delay error.
+        values = column.cast(pa.int64())
+        if not len(values):
+            return
+        if self._is_ordered(values):
+            self._last_time = values[len(values) - 1].as_py()
+            return
+        self._reject_out_of_order(values)
+
+    def _is_ordered(self, values: pa.ChunkedArray | pa.Array) -> bool:
+        if self._last_time is not None and values[0].as_py() < self._last_time:
+            return False
+        decreases = pc.less(values.slice(1), values.slice(0, len(values) - 1))
+        return not pc.any(decreases).as_py()
+
+    def _reject_out_of_order(self, values: pa.ChunkedArray | pa.Array) -> NoReturn:
+        previous = self._last_time
+        for current in values.to_pylist():
+            if previous is not None and current < previous:
+                self._reject(
+                    ".event_time: expected nondecreasing event time; select an "
+                    "explicit watermark policy for out-of-order input"
+                )
+            previous = current
+        raise RuntimeError(f"{self._path}.event_time: ordering violation not found")
+
+    def _reject(self, message: str) -> NoReturn:
+        self.failure = self._path + message
+        raise ValueError(self.failure)
+
+    async def close(self) -> None:
+        if self._closed or self._iterator is None:
+            return
+        self._closed = True
+        close = getattr(self._iterator, "aclose", None)
+        if close is not None:
+            await close()
+
+
+def _source_binding(
+    data: StreamInput,
+    name: str,
+    value: TableExpr,
+    budget: EdgeBudget,
+    policy: WatermarkPolicy | None,
+) -> tuple[SourceBinding, _IterableSource | None]:
+    if isinstance(data, SourceBinding):
+        return data, None
+    if not isinstance(data, AsyncIterable):
+        raise TypeError(
+            f"stream.inputs.{name}: expected async iterable or SourceBinding"
+        )
+    config = _source_config(name, value, budget, policy)
+    source = _IterableSource(data, config)
+    return SourceBinding(source, watermark_policy=config.policy), source