vgi-python 0.8.0__py3-none-any.whl → 0.8.2__py3-none-any.whl

vgi/_duckdb.py

@@ -32,6 +32,9 @@ def engine_module() -> ModuleType:
 
     The result is cached for the life of the process.
 
+    Returns:
+        The resolved engine module (``haybarn`` if available, else ``duckdb``).
+
     Raises:
         ImportError: if neither engine is installed.
 

vgi/_test_fixtures/aggregate/dynamic.py

@@ -190,7 +190,7 @@ class _DynamicAggregateBase(AggregateFunction[DynamicState]):
     # when DuckDB batches many partitions into shared buffers.
 
     @staticmethod
-    def _slice_to_frame(  # noqa: D417
+    def _slice_to_frame(
         partition: WindowPartition,
         subframes: list[tuple[int, int]],
         data_start: int,
@@ -198,11 +198,17 @@ class _DynamicAggregateBase(AggregateFunction[DynamicState]):
         """Slice all partition columns to the frame rows.
 
         Args:
+            partition: The window partition whose columns are sliced.
+            subframes: List of ``(begin, end)`` index tuples describing the
+                row ranges to include in the frame.
             data_start: Index where data columns begin (header columns are
                 ``[0 .. data_start)``). NULL-drop is applied on data columns
                 only — matches the filtering ``_do_update`` performs in the
                 non-window path.
 
+        Returns:
+            A table containing only the frame rows across all partition columns.
+
         """
         num_cols = partition.inputs.num_columns
         cols = [partition.inputs.column(i) for i in range(num_cols)]

vgi/_test_fixtures/bad_enum.py

@@ -0,0 +1,72 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Fixture worker that deliberately advertises an unrecognized enum value.
+
+This fixture exercises the C++ extension's *wire-enum validation* end-to-end:
+the catalog-metadata parser (``ParseFunctionInfo`` in
+``vgi/src/vgi_catalog_api.cpp``) must reject an enum string it does not
+recognize with a loud ``IOException`` rather than silently falling back to a
+default. A silent fallback would run with behavior inconsistent with what the
+worker declared (e.g. treating a ``SPECIAL`` null-handling function as
+``DEFAULT``).
+
+The trick is entirely Python-side and needs no extension rebuild. The normal
+metadata path can only ever emit valid enum names because the values come from
+typed Python ``Enum`` members. To get a bogus string onto the wire we override
+:meth:`ExampleCatalog._function_to_info` for one scalar function (``double``)
+and swap its ``null_handling`` for :class:`_BogusNullHandling.WEIRD` — a real
+``Enum`` member whose ``.name`` is ``"WEIRD"``. The vgi-rpc serializer converts
+any ``Enum`` field to ``value.name`` (see ``ArrowSerializableDataclass``), so
+``"WEIRD"`` lands in the ``null_handling`` Arrow column and the C++ parser
+trips on it the moment the ``double`` function's metadata is loaded.
+
+Otherwise this is a drop-in replacement for ``vgi-fixture-worker``: every other
+function and the catalog are inherited unchanged from :class:`ExampleWorker`,
+so any function except ``double`` still resolves normally.
+
+Registered as the ``vgi-fixture-bad-enum-worker`` entry point.
+"""
+
+from __future__ import annotations
+
+from dataclasses import replace
+from enum import Enum
+
+from vgi._test_fixtures.worker import ExampleCatalog, ExampleWorker
+from vgi.catalog.catalog_interface import FunctionInfo
+
+# The scalar function whose null_handling we corrupt. Tests reference this name
+# to force the broken metadata onto the parse path.
+BAD_ENUM_FUNCTION = "double"
+
+
+class _BogusNullHandling(Enum):
+    """An enum member whose ``.name`` is a value the C++ parser cannot map."""
+
+    WEIRD = "WEIRD"
+
+
+class BadEnumCatalog(ExampleCatalog):
+    """ExampleCatalog that advertises a bogus null_handling for one function."""
+
+    def _function_to_info(self, func_cls: type, schema_name: str) -> FunctionInfo:
+        info = super()._function_to_info(func_cls, schema_name)
+        if info.name == BAD_ENUM_FUNCTION and info.null_handling is not None:
+            # FunctionInfo is frozen; replace() returns a corrupted copy.
+            return replace(info, null_handling=_BogusNullHandling.WEIRD)  # type: ignore[arg-type]
+        return info
+
+
+class BadEnumWorker(ExampleWorker):
+    """ExampleWorker that serves the example catalog with one bad enum value."""
+
+    catalog_interface = BadEnumCatalog
+
+
+def main() -> None:
+    """Run the bad-enum fixture worker process."""
+    BadEnumWorker.main()
+
+
+if __name__ == "__main__":
+    main()

vgi/_test_fixtures/narrow_bind/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Narrow-bind reproducer fixture.
+
+Exposes a catalog whose virtual table advertises *more* columns in its
+listing (``catalog_schema_contents_tables`` / ``catalog_table_get``) than
+its scan function returns from ``on_bind``. A client that trusts the bind
+``output_schema`` without checking it against the planned catalog columns
+indexes past the end of the worker's narrower batch in
+``ArrowTableFunction::ArrowToDuckDB`` and SIGSEGVs. The fix makes the
+client fail closed at bind with a clear ``BinderException``.
+
+Driven by ``test/sql/integration/narrow_bind_mismatch.test`` in
+``~/Development/vgi``.
+"""

vgi/_test_fixtures/narrow_bind/worker.py

@@ -0,0 +1,237 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Narrow-bind reproducer worker.
+
+Two virtual tables, each backed by a table function:
+
+  * ``mismatch`` — advertises columns ``{id, val}`` in its catalog listing
+    but its scan function ``narrow_scan`` binds to ``{id}`` only. This is
+    the inconsistency that used to segfault the client at scan time
+    (``ArrowTableFunction::ArrowToDuckDB`` walking off the end of the
+    worker's 1-column batch). The client must now refuse it at bind with a
+    clear ``BinderException``.
+
+  * ``consistent`` — advertises ``{id, val}`` and its scan function
+    ``wide_scan`` binds to ``{id, val}``. Positive control: this must keep
+    working unchanged.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Annotated, Any
+
+import pyarrow as pa
+from vgi_rpc import ArrowSerializableDataclass
+from vgi_rpc.rpc import OutputCollector
+
+from vgi import Worker
+from vgi.arguments import Arg
+from vgi.catalog import Catalog, Schema
+from vgi.catalog.catalog_interface import (
+    AttachOpaqueData,
+    ReadOnlyCatalogInterface,
+    ScanFunctionResult,
+    SchemaInfo,
+    SchemaObjectType,
+    SerializedSchema,
+    TableInfo,
+    TransactionOpaqueData,
+)
+from vgi.function import Function
+from vgi.invocation import BindResponse
+from vgi.table_function import (
+    BindParams,
+    ProcessParams,
+    TableFunctionGenerator,
+    init_single_worker,
+)
+
+CATALOG_NAME = "narrow_bind"
+
+# What the catalog advertises for both tables: two columns.
+_TABLE_SCHEMA: pa.Schema = pa.schema([pa.field("id", pa.int64()), pa.field("val", pa.int64())])
+# What the narrow scan function actually binds to: one column.
+_NARROW_BIND_SCHEMA: pa.Schema = pa.schema([pa.field("id", pa.int64())])
+
+
+@dataclass(kw_only=True)
+class _State(ArrowSerializableDataclass):
+    done: bool = False
+
+
+@dataclass(frozen=True)
+class _Args:
+    count: Annotated[int, Arg(0, doc="rows", ge=0)]
+
+
+@init_single_worker
+class NarrowScan(TableFunctionGenerator[_Args, _State]):
+    """Binds to a NARROWER schema than the catalog advertises (the bug)."""
+
+    class Meta:
+        name = "narrow_scan"
+        description = "bind reports a narrower schema than the table advertises"
+
+    @classmethod
+    def on_bind(cls, params: BindParams[_Args]) -> BindResponse:
+        return BindResponse(output_schema=_NARROW_BIND_SCHEMA)
+
+    @classmethod
+    def initial_state(cls, params: ProcessParams[_Args]) -> _State:
+        return _State()
+
+    @classmethod
+    def process(cls, params: ProcessParams[_Args], state: _State, out: OutputCollector) -> None:
+        if state.done:
+            out.finish()
+            return
+        state.done = True
+        out.emit(pa.RecordBatch.from_pydict({"id": [0, 1, 2]}, schema=params.output_schema))
+
+
+@init_single_worker
+class WideScan(TableFunctionGenerator[_Args, _State]):
+    """Binds to the full advertised schema (positive control — must work)."""
+
+    class Meta:
+        name = "wide_scan"
+        description = "bind matches the table's advertised schema"
+
+    @classmethod
+    def on_bind(cls, params: BindParams[_Args]) -> BindResponse:
+        return BindResponse(output_schema=_TABLE_SCHEMA)
+
+    @classmethod
+    def initial_state(cls, params: ProcessParams[_Args]) -> _State:
+        return _State()
+
+    @classmethod
+    def process(cls, params: ProcessParams[_Args], state: _State, out: OutputCollector) -> None:
+        if state.done:
+            out.finish()
+            return
+        state.done = True
+        out.emit(pa.RecordBatch.from_pydict({"id": [0, 1, 2], "val": [10, 20, 30]}, schema=params.output_schema))
+
+
+_FUNCTIONS: list[type[Function]] = [NarrowScan, WideScan]
+
+_CATALOG = Catalog(
+    name=CATALOG_NAME,
+    default_schema="main",
+    schemas=[
+        Schema(
+            name="main",
+            comment="narrow-bind reproducer catalog",
+            functions=list(_FUNCTIONS),
+            tables=[],
+        ),
+    ],
+)
+
+
+def _serialize_schema(s: pa.Schema) -> bytes:
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, s):
+        pass
+    return sink.getvalue().to_pybytes()
+
+
+# table name -> scan function name. Both advertise _TABLE_SCHEMA (2 cols).
+_TABLE_FUNCTIONS = {
+    "mismatch": "narrow_scan",
+    "consistent": "wide_scan",
+}
+
+
+class NarrowBindCatalog(ReadOnlyCatalogInterface):
+    catalog = _CATALOG
+    catalog_name = CATALOG_NAME
+
+    def _info(self, table_name: str) -> TableInfo:
+        return TableInfo(
+            comment=f"narrow-bind reproducer table -> {_TABLE_FUNCTIONS[table_name]}",
+            tags={},
+            name=table_name,
+            schema_name="main",
+            columns=SerializedSchema(_serialize_schema(_TABLE_SCHEMA)),
+            not_null_constraints=[],
+            unique_constraints=[],
+            check_constraints=[],
+        )
+
+    def schemas(
+        self, *, attach_opaque_data: AttachOpaqueData, transaction_opaque_data: TransactionOpaqueData | None
+    ) -> list[SchemaInfo]:
+        infos = super().schemas(attach_opaque_data=attach_opaque_data, transaction_opaque_data=transaction_opaque_data)
+        for i, info in enumerate(infos):
+            if info.name == "main":
+                infos[i] = SchemaInfo(
+                    attach_opaque_data=info.attach_opaque_data,
+                    name=info.name,
+                    comment=info.comment,
+                    tags=info.tags,
+                    estimated_object_count={
+                        **(info.estimated_object_count or {}),
+                        "table": len(_TABLE_FUNCTIONS),
+                    },
+                )
+        return infos
+
+    def schema_contents(
+        self,
+        *,
+        attach_opaque_data: AttachOpaqueData,
+        transaction_opaque_data: TransactionOpaqueData | None,
+        name: str,
+        type: Any,
+    ) -> Any:
+        if name.lower() == "main" and type == SchemaObjectType.TABLE:
+            return [self._info(table_name) for table_name in _TABLE_FUNCTIONS]
+        return super().schema_contents(
+            attach_opaque_data=attach_opaque_data, transaction_opaque_data=transaction_opaque_data, name=name, type=type
+        )
+
+    def table_get(
+        self,
+        *,
+        attach_opaque_data: AttachOpaqueData,
+        transaction_opaque_data: TransactionOpaqueData | None,
+        schema_name: str,
+        name: str,
+        at_unit: str | None = None,
+        at_value: str | None = None,
+    ) -> TableInfo | None:
+        if schema_name.lower() != "main":
+            return None
+        if name in _TABLE_FUNCTIONS:
+            return self._info(name)
+        return None
+
+    def table_scan_function_get(
+        self,
+        *,
+        attach_opaque_data: AttachOpaqueData,
+        transaction_opaque_data: TransactionOpaqueData | None,
+        schema_name: str,
+        name: str,
+        at_unit: str | None,
+        at_value: str | None,
+    ) -> ScanFunctionResult:
+        fn = _TABLE_FUNCTIONS.get(name)
+        if fn is None:
+            raise ValueError(f"unknown narrow-bind reproducer table: {name}")
+        return ScanFunctionResult(
+            function_name=fn,
+            positional_arguments=[pa.scalar(3, type=pa.int64())],
+            named_arguments={},
+            required_extensions=[],
+        )
+
+
+class NarrowBindWorker(Worker):
+    catalog_interface = NarrowBindCatalog
+    catalog_name = CATALOG_NAME
+    catalog = _CATALOG
+    functions = list(_FUNCTIONS)

vgi/_test_fixtures/scalar/__init__.py

@@ -52,12 +52,15 @@ from vgi._test_fixtures.scalar.null_handling import (
 from vgi._test_fixtures.scalar.random_demo import (
     BernoulliFunction,
     HashSeedFunction,
+    QuerySeedFunction,
     RandomBytesFunction,
     RandomIntFunction,
 )
 from vgi._test_fixtures.scalar.settings_secrets import (
     MultiplyBySettingFunction,
     ReturnSecretValueFunction,
+    ScaleBySettingFunction,
+    SecretFieldFunction,
     WhoAmIFunction,
 )
 from vgi._test_fixtures.scalar.type_info import (
@@ -100,9 +103,12 @@ __all__ = [
     "PairTypeIntIntFunction",
     "PairTypeIntStrFunction",
     "PairTypeStrStrFunction",
+    "QuerySeedFunction",
     "RandomBytesFunction",
     "RandomIntFunction",
     "ReturnSecretValueFunction",
+    "ScaleBySettingFunction",
+    "SecretFieldFunction",
     "SmartFormatPrefixFunction",
     "SmartFormatWidthFunction",
     "SumValuesFunction",

vgi/_test_fixtures/scalar/random_demo.py

@@ -136,6 +136,50 @@ class HashSeedFunction(ScalarFunction):
         return pa.array([seed + i for i in range(_length)], type=pa.int64())
 
 
+class QuerySeedFunction(ScalarFunction):
+    """Adds a per-query-stable seed to each input value.
+
+    Demonstrates ``FunctionStability.CONSISTENT_WITHIN_QUERY`` — the only
+    fixture that emits this stability variant. Semantically the value is fixed
+    for the duration of a single query but may differ across queries (like
+    ``now()``). DuckDB has no behavioral consumer that this fixture asserts; it
+    exists so the wire path for the third stability value stays exercised and
+    so other-language workers must specify it.
+
+    Example:
+        SQL:    SELECT query_seed(value) FROM data
+
+    """
+
+    class Meta:
+        """Function metadata."""
+
+        name = "query_seed"
+        description = "Add a per-query-stable seed to each value (demonstrates CONSISTENT_WITHIN_QUERY stability)"
+        stability = FunctionStability.CONSISTENT_WITHIN_QUERY
+        examples = [
+            FunctionExample(
+                sql="SELECT query_seed(value) FROM data",
+                description="Offset each value by a seed that is constant within a query",
+            ),
+        ]
+
+    @classmethod
+    def compute(
+        cls,
+        value: Annotated[pa.Int64Array, Param(doc="Value to offset")],
+    ) -> Annotated[pa.Int64Array, Returns()]:
+        """Add a fixed per-query offset to each value.
+
+        The offset is deterministic here (a constant) so SQL tests have a
+        stable expected output; the stability flag is what is under test, not
+        the numeric result.
+        """
+        import pyarrow.compute as pc
+
+        return pc.add(value, 1000)
+
+
 class RandomBytesFunction(ScalarFunction):
     """Generates deterministic pseudo-random binary blobs from a seed."""
 

vgi/_test_fixtures/scalar/settings_secrets.py

@@ -42,6 +42,79 @@ class MultiplyBySettingFunction(ScalarFunction):
         return pc.multiply(multiplier, value)
 
 
+class ScaleBySettingFunction(ScalarFunction):
+    """Scale the input value by the float (DOUBLE) setting ``scale_factor``.
+
+    Companion to :class:`MultiplyBySettingFunction`, but reads a floating-point
+    setting rather than an integer one.
+
+    Example:
+        SQL:    SELECT scale_by_setting(4.0)
+
+    """
+
+    class Meta:
+        """Function metadata."""
+
+        name = "scale_by_setting"
+        description = "Scale the input value by the float setting `scale_factor`"
+        examples = [
+            FunctionExample(
+                sql="SELECT scale_by_setting(4.0)",
+                description="Scale the input value by the float setting's value",
+            ),
+        ]
+
+    @classmethod
+    def compute(
+        cls,
+        value: Annotated[pa.DoubleArray, Param(doc="Value to scale")],
+        scale_factor: Annotated[pa.Scalar[Any] | None, Setting()],
+    ) -> Annotated[pa.DoubleArray, Returns()]:
+        """Generate the result for each row."""
+        factor = 1.0 if scale_factor is None or scale_factor.as_py() is None else scale_factor.as_py()
+        return pc.multiply(pa.scalar(factor, type=pa.float64()), value)
+
+
+class SecretFieldFunction(ScalarFunction):
+    """Look up individual secret fields by name.
+
+    ``port`` is read by named lookup on the ``vgi_example`` secret and
+    ``secret_string`` by field name; the result mirrors the wire behaviour of
+    the worker-side named/positional secret field accessors.
+
+    Example:
+        SQL:    SELECT secret_field()
+
+    """
+
+    class Meta:
+        """Function metadata."""
+
+        name = "secret_field"
+        description = "Look up secret fields by name"
+        examples = [
+            FunctionExample(
+                sql="SELECT secret_field()",
+                description="Look up secret fields by name",
+            ),
+        ]
+
+    @classmethod
+    def compute(
+        cls,
+        vgi_example: Annotated[dict[str, pa.Scalar[Any]], Secret("vgi_example")],
+        _length: Annotated[int, OutputLength()],
+    ) -> Annotated[pa.StringArray, Returns()]:
+        """Generate the result for each row."""
+        port = vgi_example.get("port")
+        name = vgi_example.get("secret_string")
+        port_s = "" if port is None else str(port.as_py())
+        name_s = "" if name is None else str(name.as_py())
+        result = f"port={port_s};name={name_s}"
+        return pa.array([result for _ in range(_length)], type=pa.string())
+
+
 class ReturnSecretValueFunction(ScalarFunction):
     """Return the value of a secret.
 

vgi/_test_fixtures/table/__init__.py

@@ -45,6 +45,7 @@ from vgi._test_fixtures.table.filters import (
     FilterEchoFunction,
     FilterEchoPartitionedFunction,
     FilterEchoTableScanFunction,
+    FilteredColumnsEchoFunction,
     SpatialFilterExampleFunction,
     ValuePruneFunction,
 )
@@ -81,6 +82,7 @@ from vgi._test_fixtures.table.pairs import (
 from vgi._test_fixtures.table.partition_columns import (
     CountryPartitionedSalesFunction,
     DisjointRangePartitionedFunction,
+    OverlappingRangePartitionedFunction,
     PartitionedWithExplicitOverrideFunction,
     RegionYearPartitionedFunction,
 )
@@ -123,6 +125,7 @@ from vgi._test_fixtures.table.settings import (
     StructSettingsFunction,
 )
 from vgi._test_fixtures.table.transaction_storage import TxCachedValueFunction
+from vgi._test_fixtures.table.typed_probe import TypedProbeFunction
 from vgi._test_fixtures.table.versioned import (
     _CURRENT_VERSION,
     _VERSIONED_CONSTRAINTS_CURRENT,
@@ -137,6 +140,7 @@ from vgi._test_fixtures.table.versioned import (
 )
 
 __all__ = [
+    "TypedProbeFunction",
     "_CURRENT_VERSION",
     "_VERSIONED_CONSTRAINTS_CURRENT",
     "_VERSIONED_CONSTRAINTS_DATA",
@@ -161,6 +165,7 @@ __all__ = [
     "FilterEchoFunction",
     "FilterEchoPartitionedFunction",
     "FilterEchoTableScanFunction",
+    "FilteredColumnsEchoFunction",
     "GeneratorExceptionFunction",
     "ValuePruneFunction",
     "LateMaterializationFunction",
@@ -178,6 +183,7 @@ __all__ = [
     "NestedSequenceFunction",
     "NonMonotoneBatchIndexFunction",
     "OrderEchoFunction",
+    "OverlappingRangePartitionedFunction",
     "PartitionedBatchIndexFunction",
     "PartitionedBatchIndexMarkedFunction",
     "PartitionedFixedOrderFunction",