calc-flow-python 4.0.0__cp313-abi3-win_amd64.whl

calc_flow/symbolic/program.py

@@ -0,0 +1,377 @@
+"""Immutable feature sets and programs over symbolic declarations.
+
+``FeatureSet`` and ``Program`` are the program-level declaration values of the
+frozen public surface. A program owns its declared inputs and outputs, the
+``calc_flow.symbolic.declaration.v1`` program fingerprint, and the declaration
+processing entry points ``analyze``/``explain``. Compilation to execution plans
+is a later lowering stage and is deliberately absent here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from calc_flow.symbolic.domains import type_name
+from calc_flow.symbolic.expr import ArrayExpr, ColumnExpr, Parameter, TableExpr
+from calc_flow.symbolic.nodes import (
+    _MAGIC,
+    CStr,
+    Node,
+    _text,
+    _u64,
+)
+from calc_flow.symbolic.types import CompileMode
+
+if TYPE_CHECKING:
+    from calc_flow.pipeline import BatchExecutionPlan, Runtime, StreamExecutionPlan
+    from calc_flow.symbolic.analyzer import AnalysisResult
+    from calc_flow.symbolic.types import LatePolicy
+
+_PROGRAM_TAG = 0x21
+
+
+@dataclass(frozen=True, slots=True, eq=False, init=False)
+class FeatureSet:
+    """An ordered immutable set of uniquely named column expressions."""
+
+    _features: tuple[tuple[str, ColumnExpr], ...]
+
+    def __init__(
+        self,
+        features: Sequence[tuple[str, ColumnExpr]] = (),
+        /,
+    ) -> None:
+        copied: list[tuple[str, ColumnExpr]] = []
+        seen: set[str] = set()
+        for index, item in enumerate(features):
+            if not isinstance(item, tuple) or len(item) != 2:
+                raise TypeError(
+                    f"features[{index}]: must be a (name, ColumnExpr) pair; got"
+                    f" {type_name(item)}"
+                )
+            name, value = item
+            if type(name) is not str:
+                raise TypeError(
+                    f"features[{index}].name: must be a string; got {type_name(name)}"
+                )
+            if not isinstance(value, ColumnExpr):
+                raise TypeError(
+                    f"features[{index}].value: must be a ColumnExpr; got"
+                    f" {type_name(value)}"
+                )
+            if name in seen:
+                raise ValueError(
+                    f"features[{index}].name: duplicate_name: duplicate feature"
+                    f" name {name!r}"
+                )
+            seen.add(name)
+            copied.append((name, value))
+        object.__setattr__(self, "_features", tuple(copied))
+
+    @property
+    def features(self) -> tuple[tuple[str, ColumnExpr], ...]:
+        """The declared features in declaration order."""
+
+        return self._features
+
+    def with_feature(self, name: str, value: ColumnExpr, /) -> FeatureSet:
+        """Return a new feature set with one feature appended."""
+
+        return FeatureSet((*self._features, (name, value)))
+
+
+def _node_name(node: Node, /) -> str:
+    value = node.attr("name")
+    if isinstance(value, CStr):
+        return value.value
+    raise TypeError("declaration node is missing its name attribute")
+
+
+def _collect_program_nodes(
+    inputs: tuple[TableExpr | Parameter[object], ...],
+    outputs: tuple[tuple[str, TableExpr | ArrayExpr], ...],
+    /,
+) -> dict[str, tuple[bytes, Node]]:
+    """Return every unique node reachable from a declared input or output."""
+
+    nodes: dict[str, tuple[bytes, Node]] = {}
+
+    def visit(node: Node) -> None:
+        existing = nodes.get(node.digest)
+        if existing is not None:
+            if existing[0] != node.node_bytes:
+                raise ValueError(
+                    "Program.fingerprint: unresolved_type: digest collision for"
+                    f" node {node.op.name}@{node.op.version}"
+                )
+            return
+        nodes[node.digest] = (node.node_bytes, node)
+        for child in node.args:
+            visit(child)
+
+    for value in inputs:
+        visit(value._node)
+    for _, value in outputs:
+        visit(value._node)
+    return nodes
+
+
+def _program_fingerprint(
+    name: str,
+    inputs: tuple[TableExpr | Parameter[object], ...],
+    outputs: tuple[tuple[str, TableExpr | ArrayExpr], ...],
+    /,
+) -> str:
+    """Compute the frozen v1 program fingerprint over the declaration graph."""
+
+    nodes = _collect_program_nodes(inputs, outputs)
+    edges = _program_edges(nodes)
+    node_records = sorted(
+        ((bytes.fromhex(digest), record) for digest, record in nodes.items()),
+        key=lambda item: item[0],
+    )
+    body = (
+        bytes((_PROGRAM_TAG,))
+        + _text(name)
+        + _u64(len(inputs))
+        + b"".join(_input_records(inputs))
+        + _u64(len(outputs))
+        + b"".join(
+            _text(output_name) + bytes.fromhex(value._node.digest)
+            for output_name, value in outputs
+        )
+        + _u64(len(node_records))
+        + b"".join(
+            digest + _u64(len(record[0])) + record[0] for digest, record in node_records
+        )
+        + _u64(len(edges))
+        + b"".join(parent + _u64(index) + child for parent, index, child in edges)
+    )
+    return hashlib.sha256(_MAGIC + b"\x02" + _u64(len(body)) + body).hexdigest()
+
+
+def _program_edges(
+    nodes: dict[str, tuple[bytes, Node]],
+    /,
+) -> list[tuple[bytes, int, bytes]]:
+    edges: list[tuple[bytes, int, bytes]] = []
+    for _, node in nodes.values():
+        for index, child in enumerate(node.args):
+            edges.append(
+                (
+                    bytes.fromhex(node.digest),
+                    index,
+                    bytes.fromhex(child.digest),
+                )
+            )
+    edges.sort()
+    return edges
+
+
+def _input_records(
+    inputs: tuple[TableExpr | Parameter[object], ...],
+    /,
+) -> list[bytes]:
+    return [
+        _text(_node_name(value._node)) + bytes.fromhex(value._node.digest)
+        for value in inputs
+    ]
+
+
+def _validated_inputs(
+    inputs: Sequence[TableExpr | Parameter[object]], /
+) -> tuple[TableExpr | Parameter[object], ...]:
+    declared: dict[str, str] = {}
+    copied: list[TableExpr | Parameter[object]] = []
+    for index, value in enumerate(inputs):
+        if not isinstance(value, (TableExpr, Parameter)):
+            raise TypeError(
+                f"Program.inputs[{index}]: expected TableExpr |"
+                f" Parameter[object]; got {type_name(value)}"
+            )
+        if value._node.op.name not in ("table_input", "parameter"):
+            raise ValueError(
+                f"Program.inputs[{index}]: invalid_literal: program inputs"
+                " must be declared table_input or parameter values; got"
+                f" {value._node.op.name}"
+            )
+        input_name = _node_name(value._node)
+        root = "static_inputs" if isinstance(value, Parameter) else "inputs"
+        if input_name in declared:
+            raise ValueError(
+                f"{root}.{input_name}: duplicate_name: duplicate input name"
+                f" {input_name!r}"
+            )
+        declared[input_name] = root
+        copied.append(value)
+    return tuple(copied)
+
+
+def _validated_output(index: int, item: object, /) -> tuple[str, TableExpr | ArrayExpr]:
+    if not isinstance(item, tuple) or len(item) != 2:
+        raise TypeError(
+            f"Program.outputs[{index}]: must be a (name, TableExpr |"
+            f" ArrayExpr) pair; got {type_name(item)}"
+        )
+    output_name, value = item
+    if type(output_name) is not str:
+        raise TypeError(
+            f"Program.outputs[{index}].name: must be a string; got"
+            f" {type_name(output_name)}"
+        )
+    if not output_name:
+        raise ValueError(
+            f"Program.outputs[{index}].name: invalid_literal: must be a"
+            " non-empty string"
+        )
+    if not isinstance(value, (TableExpr, ArrayExpr)):
+        raise TypeError(
+            f"Program.outputs[{index}].value: expected TableExpr |"
+            f" ArrayExpr; got {type_name(value)}"
+        )
+    return output_name, value
+
+
+def _validated_outputs(
+    outputs: Sequence[tuple[str, TableExpr | ArrayExpr]], /
+) -> tuple[tuple[str, TableExpr | ArrayExpr], ...]:
+    copied: list[tuple[str, TableExpr | ArrayExpr]] = []
+    names: set[str] = set()
+    for index, item in enumerate(outputs):
+        output_name, value = _validated_output(index, item)
+        if output_name in names:
+            raise ValueError(
+                f"outputs.{output_name}: duplicate_name: duplicate output"
+                f" name {output_name!r}"
+            )
+        names.add(output_name)
+        copied.append((output_name, value))
+    return tuple(copied)
+
+
+@dataclass(frozen=True, slots=True, eq=False, init=False)
+class Program:
+    """An immutable program of declared inputs, outputs, and expressions."""
+
+    _name: str
+    _inputs: tuple[TableExpr | Parameter[object], ...]
+    _outputs: tuple[tuple[str, TableExpr | ArrayExpr], ...]
+    _fingerprint: str
+
+    def __init__(
+        self,
+        name: str,
+        /,
+        *,
+        inputs: Sequence[TableExpr | Parameter[object]] = (),
+        outputs: Sequence[tuple[str, TableExpr | ArrayExpr]] = (),
+    ) -> None:
+        if type(name) is not str:
+            raise TypeError(f"Program.name must be a string; got {type_name(name)}")
+        if not name:
+            raise ValueError(
+                "Program.name: invalid_literal: must be a non-empty string"
+            )
+        copied_inputs = _validated_inputs(inputs)
+        copied_outputs = _validated_outputs(outputs)
+        object.__setattr__(self, "_name", name)
+        object.__setattr__(self, "_inputs", copied_inputs)
+        object.__setattr__(self, "_outputs", copied_outputs)
+        object.__setattr__(
+            self,
+            "_fingerprint",
+            _program_fingerprint(name, copied_inputs, copied_outputs),
+        )
+
+    @property
+    def name(self) -> str:
+        """The declared program name."""
+
+        return self._name
+
+    @property
+    def inputs(self) -> tuple[TableExpr | Parameter[object], ...]:
+        """The declared inputs in declaration order."""
+
+        return self._inputs
+
+    @property
+    def outputs(self) -> tuple[tuple[str, TableExpr | ArrayExpr], ...]:
+        """The declared outputs in declaration order."""
+
+        return self._outputs
+
+    @property
+    def fingerprint(self) -> str:
+        """The runtime-independent v1 program fingerprint."""
+
+        return self._fingerprint
+
+    def with_input(self, value: TableExpr | Parameter[object], /) -> Program:
+        """Return a new program with one declared input appended."""
+
+        return Program(
+            self._name,
+            inputs=(*self._inputs, value),
+            outputs=self._outputs,
+        )
+
+    def output(self, name: str, value: TableExpr | ArrayExpr, /) -> Program:
+        """Return a new program with one declared output appended."""
+
+        return Program(
+            self._name,
+            inputs=self._inputs,
+            outputs=(*self._outputs, (name, value)),
+        )
+
+    def analyze(self, runtime: Runtime, /, *, mode: CompileMode) -> AnalysisResult:
+        """Analyze this program against one immutable capability snapshot."""
+
+        from calc_flow.symbolic.analyzer import analyze_program
+
+        return analyze_program(self, runtime, mode)
+
+    def explain(self, runtime: Runtime, /, *, mode: CompileMode) -> str:
+        """Render deterministic analysis facts for this program."""
+
+        from calc_flow.symbolic.analyzer import explain_program
+
+        return explain_program(self, runtime, mode)
+
+    def compile_batch(self, runtime: Runtime, /) -> BatchExecutionPlan:
+        """Lower this program to a strict project-v3 batch execution plan.
+
+        Compilation is declaration processing only: it captures one immutable
+        capability snapshot, lowers one strict project-v3 document, and invokes
+        the Rust graph compiler for final validation. No data, source, sink,
+        or runner is accepted.
+        """
+
+        from calc_flow.symbolic.lower import compile_program_batch
+
+        return compile_program_batch(self, runtime)
+
+    def compile_stream(
+        self,
+        runtime: Runtime,
+        /,
+        *,
+        allowed_lateness_micros: int = 0,
+        late_policy: LatePolicy = "error",
+    ) -> StreamExecutionPlan:
+        """Lower this program to a strict project-v3 continuous plan.
+
+        The lateness arguments are validated and serialized into every lowered
+        rolling or cross-section stage. Row-local-only programs accept the same
+        compile signature but have no stateful late-row surface.
+        """
+
+        from calc_flow.symbolic.lower import compile_program_stream
+
+        return compile_program_stream(
+            self, runtime, allowed_lateness_micros, late_policy
+        )

calc_flow/symbolic/types.py

@@ -0,0 +1,110 @@
+"""Strict declaration types and validators for symbolic expressions."""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import Literal
+
+from calc_flow.capabilities import PORTABLE_ARROW_TYPES
+from calc_flow.symbolic.domains import type_name
+
+type BatchKind = Literal["table", "array"]
+type CompileMode = Literal["batch", "stream"]
+type LatePolicy = Literal["error", "drop"]
+type ScalarLiteral = None | bool | int | float | str
+
+# The initial portable field spelling adds ``timestamp[us, UTC]`` to the
+# current portable Arrow names (API note section 2.2).
+TABLE_FIELD_TYPES = frozenset((*PORTABLE_ARROW_TYPES, "timestamp[us, UTC]"))
+# Array declaration dtypes stay inside the static-input digest v1 coverage:
+# bool, the integer types, float32, and float64.
+ARRAY_DTYPES = frozenset(
+    (
+        "bool",
+        "float32",
+        "float64",
+        "int16",
+        "int32",
+        "int64",
+        "int8",
+        "uint16",
+        "uint32",
+        "uint64",
+        "uint8",
+    )
+)
+
+
+@dataclass(frozen=True, slots=True)
+class Field:
+    """One exact table field declaration with host-type validation."""
+
+    name: str
+    data_type: str
+    nullable: bool = True
+
+    def __post_init__(self) -> None:
+        if type(self.name) is not str:
+            raise TypeError(f"Field.name must be a string; got {type_name(self.name)}")
+        if type(self.data_type) is not str:
+            raise TypeError(
+                f"Field.data_type must be a string; got {type_name(self.data_type)}"
+            )
+        if type(self.nullable) is not bool:
+            raise TypeError(
+                f"Field.nullable must be a boolean; got {type_name(self.nullable)}"
+            )
+
+
+def require_str(value: object, path: str, /) -> str:
+    if type(value) is not str:
+        raise TypeError(f"{path} must be a string; got {type_name(value)}")
+    return value
+
+
+def require_non_empty_str(value: object, path: str, /) -> str:
+    require_str(value, path)
+    if not value:  # type: ignore[arg-type]
+        raise ValueError(f"{path}: invalid_literal: must be a non-empty string")
+    return value  # type: ignore[return-value]
+
+
+def require_int(value: object, path: str, /) -> int:
+    if type(value) is not int:
+        raise TypeError(f"{path} must be an integer; got {type_name(value)}")
+    return value
+
+
+def require_positive_int(value: object, path: str, /) -> int:
+    require_int(value, path)
+    if value <= 0:
+        raise ValueError(f"{path}: invalid_literal: must be a positive integer")
+    return value
+
+
+def require_non_negative_int(value: object, path: str, /) -> int:
+    require_int(value, path)
+    if value < 0:
+        raise ValueError(f"{path}: invalid_literal: must be a non-negative integer")
+    return value
+
+
+def require_finite_number(value: object, path: str, /) -> int | float:
+    if type(value) not in (int, float):
+        raise TypeError(f"{path} must be a finite number; got {type_name(value)}")
+    if isinstance(value, float) and not math.isfinite(value):
+        raise ValueError(f"{path}: invalid_literal: must be finite")
+    return value
+
+
+def check_table_field_type(data_type: str, path: str, /) -> None:
+    if data_type not in TABLE_FIELD_TYPES:
+        raise ValueError(
+            f"{path}: unsupported_type: unknown portable Arrow type {data_type!r}"
+        )
+
+
+def check_array_dtype(dtype: str, path: str, /) -> None:
+    if dtype not in ARRAY_DTYPES:
+        raise ValueError(f"{path}: unsupported_type: unknown array dtype {dtype!r}")

calc_flow/symbolic/windows.py

@@ -0,0 +1,153 @@
+"""Explicit row-count, duration, and event-time grouping declarations."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+from calc_flow.symbolic.domains import namespace_error
+from calc_flow.symbolic.expr import ColumnExpr
+from calc_flow.symbolic.types import require_positive_int
+
+
+@dataclass(frozen=True, slots=True)
+class RowFrame:
+    """A positive row-count rolling frame ``rows [i - size + 1, i]``."""
+
+    size: int
+
+    def __post_init__(self) -> None:
+        require_positive_int(self.size, "RowFrame.size")
+
+
+@dataclass(frozen=True, slots=True)
+class DurationFrame:
+    """A positive exact-microseconds duration frame ``(t - micros, t]``."""
+
+    micros: int
+
+    def __post_init__(self) -> None:
+        require_positive_int(self.micros, "DurationFrame.micros")
+
+
+@dataclass(frozen=True, slots=True, eq=False)
+class EventTimeBucket:
+    """A fixed UTC event-time bucket of exact positive width."""
+
+    event_time: ColumnExpr
+    width_micros: int
+    partition_by: tuple[ColumnExpr, ...] = ()
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.event_time, ColumnExpr):
+            raise namespace_error(
+                "event_time_bucket", "event_time", "ColumnExpr", self.event_time
+            )
+        require_positive_int(
+            self.width_micros, "calc_flow.symbolic.event_time_bucket.width_micros"
+        )
+        object.__setattr__(self, "partition_by", tuple(self.partition_by))
+        for index, column in enumerate(self.partition_by):
+            if not isinstance(column, ColumnExpr):
+                raise namespace_error(
+                    "event_time_bucket",
+                    f"partition_by[{index}]",
+                    "ColumnExpr",
+                    column,
+                )
+
+
+@dataclass(frozen=True, slots=True, eq=False)
+class CrossSectionGroup:
+    """One complete cross-section grouping declaration."""
+
+    event_time: ColumnExpr
+    bucket: EventTimeBucket | None
+    partition_by: tuple[ColumnExpr, ...]
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.event_time, ColumnExpr):
+            raise namespace_error(
+                "cross_section_group",
+                "event_time",
+                "ColumnExpr",
+                self.event_time,
+            )
+        if self.bucket is not None and not isinstance(self.bucket, EventTimeBucket):
+            raise namespace_error(
+                "cross_section_group",
+                "bucket",
+                "EventTimeBucket | None",
+                self.bucket,
+            )
+        object.__setattr__(self, "partition_by", tuple(self.partition_by))
+        for index, column in enumerate(self.partition_by):
+            if not isinstance(column, ColumnExpr):
+                raise namespace_error(
+                    "cross_section_group",
+                    f"partition_by[{index}]",
+                    "ColumnExpr",
+                    column,
+                )
+
+
+def rows(size: int, /) -> RowFrame:
+    """Declare a row-count rolling frame."""
+
+    require_positive_int(size, "calc_flow.symbolic.rows.size")
+    return RowFrame(size)
+
+
+def duration(micros: int, /) -> DurationFrame:
+    """Declare a duration rolling frame in exact microseconds."""
+
+    require_positive_int(micros, "calc_flow.symbolic.duration.micros")
+    return DurationFrame(micros)
+
+
+def exact_time(
+    event_time: ColumnExpr,
+    /,
+    *,
+    partition_by: Sequence[ColumnExpr] = (),
+) -> CrossSectionGroup:
+    """Group by one exact event-time value plus the ordered partition key."""
+
+    if not isinstance(event_time, ColumnExpr):
+        raise namespace_error("exact_time", "event_time", "ColumnExpr", event_time)
+    columns = tuple(partition_by)
+    for index, column in enumerate(columns):
+        if not isinstance(column, ColumnExpr):
+            raise namespace_error(
+                "exact_time", f"partition_by[{index}]", "ColumnExpr", column
+            )
+    return CrossSectionGroup(event_time, None, columns)
+
+
+def event_time_bucket(
+    event_time: ColumnExpr,
+    /,
+    *,
+    width_micros: int,
+    partition_by: Sequence[ColumnExpr] = (),
+) -> CrossSectionGroup:
+    """Group by fixed UTC buckets of exact positive width."""
+
+    if not isinstance(event_time, ColumnExpr):
+        raise namespace_error(
+            "event_time_bucket", "event_time", "ColumnExpr", event_time
+        )
+    require_positive_int(
+        width_micros, "calc_flow.symbolic.event_time_bucket.width_micros"
+    )
+    columns = tuple(partition_by)
+    for index, column in enumerate(columns):
+        if not isinstance(column, ColumnExpr):
+            raise namespace_error(
+                "event_time_bucket",
+                f"partition_by[{index}]",
+                "ColumnExpr",
+                column,
+            )
+    bucket = EventTimeBucket(event_time, width_micros, columns)
+    return CrossSectionGroup(event_time, bucket, columns)

calc_flow/udf.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Any
+
+
+def _validate_scalar_udf_registration(
+    input_types: Sequence[str], function: Any
+) -> tuple[str, ...]:
+    if isinstance(input_types, (str, bytes)) or not isinstance(input_types, Sequence):
+        raise TypeError("input_types must be a sequence of Arrow type names")
+    copied_types = tuple(input_types)
+    if len(copied_types) > 64:
+        raise ValueError("input_types must contain at most 64 entries")
+    if not all(isinstance(value, str) for value in copied_types):
+        raise TypeError("input_types must contain only Arrow type names")
+    if not callable(function):
+        raise TypeError("function must be callable")
+    return copied_types