vgi-python 0.33.0__tar.gz → 0.34.0__tar.gz

{vgi_python-0.33.0 → vgi_python-0.34.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.33.0
+Version: 0.34.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.33.0 → vgi_python-0.34.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.33.0"
+version = "0.34.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.33.0 → vgi_python-0.34.0}/tests/table_in_out/test_blended_exchange.py

@@ -280,3 +280,73 @@ class TestHasFinalizeSkip:
         assert len(with_finalize) == 1
         assert len(without_finalize) == 1
         assert with_finalize[0].column("i").to_pylist() == without_finalize[0].column("i").to_pylist()
+
+
+class TestBlendedAny:
+    """`blended_any` / `blended_any_varargs`: positional input columns declared ANY.
+
+    An ANY arg names no concrete Arrow type, so the output type can only come from
+    the input schema the client supplies at bind. These drive that over the wire:
+    the same registration must bind to whatever type each call resolves to, and
+    echo the column back untouched (nested types and nulls included).
+    """
+
+    @pytest.mark.parametrize(
+        "array",
+        [
+            pa.array(["a", None, "c"], type=pa.string()),
+            pa.array([1, 2, None], type=pa.int32()),
+            pa.array(
+                [{"message": "lost pkg", "tier": "gold"}, None, {"message": None, "tier": "free"}],
+                type=pa.struct([("message", pa.string()), ("tier", pa.string())]),
+            ),
+            pa.array([[1, 2], [], None], type=pa.list_(pa.int64())),
+        ],
+        ids=["string", "int32", "struct", "list"],
+    )
+    def test_output_is_typed_from_the_input_and_echoed(self, fixture_worker: str, array: pa.Array[Any]) -> None:
+        """One registration, bound per call to the resolved input type."""
+        batch = pa.record_batch({"value": array})
+        bound: list[pa.Schema] = []
+
+        with Client(fixture_worker) as client:
+            results = list(
+                client.table_in_out_function(
+                    function_name="blended_any",
+                    schema_path=["main"],
+                    input=iter([batch]),
+                    bind_result_callback=lambda r: bound.append(r.output_schema),
+                    has_finalize=False,
+                )
+            )
+
+        assert [s.names for s in bound] == [["value"]]
+        assert bound[0].field("value").type == array.type
+        (out,) = results
+        assert out.column("value").type == array.type
+        # A null struct and a struct with a null field must stay distinct.
+        assert out.column("value").to_pylist() == array.to_pylist()
+
+    def test_varargs_columns_each_keep_their_own_type(self, fixture_worker: str) -> None:
+        """The vararg element type is ANY, so every runtime column types itself."""
+        batch = pa.record_batch(
+            {
+                "col0": pa.array(["a", "b"], type=pa.string()),
+                "col1": pa.array([[1], [2, 3]], type=pa.list_(pa.int32())),
+                "col2": pa.array([{"k": 1.5}, None], type=pa.struct([("k", pa.float64())])),
+            }
+        )
+
+        with Client(fixture_worker) as client:
+            (out,) = list(
+                client.table_in_out_function(
+                    function_name="blended_any_varargs",
+                    schema_path=["main"],
+                    input=iter([batch]),
+                    has_finalize=False,
+                )
+            )
+
+        assert out.schema.names == ["col0", "col1", "col2"]
+        assert out.schema.types == batch.schema.types
+        assert out.to_pylist() == batch.to_pylist()

{vgi_python-0.33.0 → vgi_python-0.34.0}/tests/table_in_out/test_blended_metadata.py

@@ -148,3 +148,43 @@ class TestBlendedFootguns:
 
         with pytest.raises(TypeError, match="at least one positional Arg"):
             resolve_metadata(_NamedOnly)
+
+
+class TestBlendedAnyResolution:
+    """A blended function may declare its positional input column(s) ANY.
+
+    The ANY marker lives on the argument *spec* — the ``vgi_type=any`` field
+    metadata the client reads to learn it must take the column's type from the
+    call rather than from the declaration.
+    """
+
+    def test_any_positional_is_a_blended_input_column(self) -> None:
+        """The ANY arg resolves as a positional input column, not a bind constant."""
+        from vgi._test_fixtures.table_in_out import BlendedAnyFunction
+        from vgi.argument_spec import extract_argument_specs
+
+        m = resolve_metadata(BlendedAnyFunction)
+        assert m.input_from_args is True
+        assert m.has_finalize is False
+        (spec,) = extract_argument_specs(BlendedAnyFunction)
+        assert (spec.name, spec.position, spec.is_any_type, spec.is_varargs) == ("value", 0, True, False)
+
+    def test_any_varargs_is_a_blended_input_column(self) -> None:
+        """The varargs form keeps both markers: ANY element type and varargs."""
+        from vgi._test_fixtures.table_in_out import BlendedAnyVarargsFunction
+        from vgi.argument_spec import extract_argument_specs
+
+        assert resolve_metadata(BlendedAnyVarargsFunction).input_from_args is True
+        (spec,) = extract_argument_specs(BlendedAnyVarargsFunction)
+        assert (spec.name, spec.position, spec.is_any_type, spec.is_varargs) == ("values", 0, True, True)
+
+    def test_any_marker_survives_the_wire_schema(self) -> None:
+        """The client learns the arg is ANY from the serialized arguments schema."""
+        from vgi._test_fixtures.table_in_out import BlendedAnyFunction, BlendedAnyVarargsFunction
+        from vgi.argument_spec import argument_specs_to_schema, extract_argument_specs, schema_to_argument_specs
+
+        for function in (BlendedAnyFunction, BlendedAnyVarargsFunction):
+            specs = extract_argument_specs(function)
+            wire = schema_to_argument_specs(argument_specs_to_schema(specs))
+            assert [s.is_any_type for s in wire] == [True]
+            assert [s.is_varargs for s in wire] == [s.is_varargs for s in specs]

{vgi_python-0.33.0 → vgi_python-0.34.0}/tests/test_argument_spec.py

@@ -4,6 +4,7 @@
 
 from __future__ import annotations
 
+import dataclasses
 from typing import Annotated, Any
 
 import pyarrow as pa
@@ -856,7 +857,7 @@ class TestArgumentConstraints:
         assert defaults.column("label")[0].as_py() is None
 
     def test_typed_default_requires_concrete_parameter_type(self) -> None:
-        """An ANY parameter cannot carry an untyped protocol default."""
+        """An ANY parameter cannot carry an untyped protocol default VALUE."""
         spec = ArgumentSpec(
             name="value",
             position=0,
@@ -865,9 +866,65 @@ class TestArgumentConstraints:
             default_value=1,
         )
 
-        with pytest.raises(ValueError, match="typed defaults require an explicit parameter type"):
+        with pytest.raises(ValueError, match="a typed default requires an explicit parameter type"):
             parameter_default_values_from_specs([spec], require_trailing=True)
 
+    def test_an_any_parameter_may_default_to_none(self) -> None:
+        """`None` IS representable on an ANY parameter — a null array holds nulls.
+
+        It is column presence, not the value, that marks a parameter as having a
+        default, so this costs the protocol nothing. It buys a function the
+        ability to declare a *required* ANY argument with ``default=None`` so
+        that omitting it reaches ``on_bind()``, where the function can name the
+        argument and show its shape, instead of raising a bare ``KeyError``
+        during argument parsing before any user code runs.
+        """
+        spec = ArgumentSpec(
+            name="questions",
+            position="questions",
+            arrow_type=pa.null(),
+            is_any_type=True,
+            default_value=None,
+        )
+
+        defaults = parameter_default_values_from_specs([spec], require_trailing=True)
+
+        assert defaults is not None
+        assert defaults.schema.names == ["questions"]
+        assert defaults.column("questions").type == pa.null()
+        assert defaults.column("questions")[0].as_py() is None
+
+    def test_an_optional_any_argument_is_still_an_any_argument(self) -> None:
+        """``AnyArrow | None`` must stay ANY-typed, not collapse to a NULL column.
+
+        A marker type is recognised by identity, so an Optional wrapper used to
+        hide it: the parameter silently became NULL-typed and DuckDB could then
+        cast nothing into it (`Unimplemented type for cast (STRUCT(...) -> NULL)`).
+        Declaring a REQUIRED ANY argument nullable, with ``default=None``, is how
+        a function gets to reject the missing case in its own ``on_bind()`` with
+        a message naming the argument.
+        """
+
+        @dataclasses.dataclass(slots=True, frozen=True, kw_only=True)
+        class _Args:
+            """Args with a required-but-nullable ANY parameter."""
+
+            payload: Annotated[AnyArrow | None, Arg("payload", default=None)] = None
+
+        class _Fn(TableInOutFunction[_Args]):
+            """A function declaring one optional ANY argument."""
+
+            FunctionArguments = _Args
+
+            class Meta:
+                """Metadata."""
+
+                name = "optional_any"
+
+        (spec,) = [sp for sp in extract_argument_specs(_Fn) if sp.name == "payload"]
+        assert spec.is_any_type, "an optional ANY argument lost its ANY marker"
+        assert pa.types.is_null(spec.arrow_type)
+
     def test_typed_defaults_reject_case_insensitive_duplicate_names(self) -> None:
         """Default-batch field names must identify one signature slot."""
         specs = [

{vgi_python-0.33.0 → vgi_python-0.34.0}/uv.lock

@@ -2580,7 +2580,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.33.0"
+version = "0.34.0"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

{vgi_python-0.33.0 → vgi_python-0.34.0}/vgi/_test_fixtures/table_in_out.py

@@ -51,7 +51,7 @@ from vgi_rpc import ArrowSerializableDataclass, ArrowType
 from vgi_rpc.log import Level
 from vgi_rpc.rpc import OutputCollector
 
-from vgi.arguments import Arg, Setting, TableInput
+from vgi.arguments import AnyArrow, Arg, Setting, TableInput
 from vgi.cache_control import CacheControl
 from vgi.invocation import BindResponse
 from vgi.metadata import FunctionExample
@@ -94,6 +94,8 @@ __all__ = [
     "GeoEncode3Function",
     "RowSumFunction",
     "BlendedDropFunction",
+    "BlendedAnyFunction",
+    "BlendedAnyVarargsFunction",
     "SubstreamPartialSumFunction",
     "MultiBatchFinishFunction",
     "SumAllColumnsFunction",
@@ -1880,6 +1882,87 @@ class BlendedExplodeFunction(RowTransformFunction[_ExplodeArgs]):
         cast("VgiOutputCollector", out).emit(out_batch, parent_rows=parent_rows)
 
 
+@dataclass(slots=True, frozen=True, kw_only=True)
+class _AnyArgs:
+    """One positional ANY-typed input column."""
+
+    value: Annotated[AnyArrow, Arg(0, doc="Input column of any type (echoed back unchanged)")]
+
+
+class BlendedAnyFunction(RowTransformFunction[_AnyArgs]):
+    """Blended 1->1 echo of a single ANY-typed input column.
+
+    Proves a blended positional arg may be declared ANY: the declared type names
+    no concrete Arrow type, so the client must build the worker's input schema
+    from the type DuckDB actually resolved for the call (a STRUCT built per row,
+    a whole-row struct, a LIST, a plain VARCHAR ...) rather than from the
+    declaration. The output column ``value`` is bound to that resolved input
+    type at bind time and the column is passed through untouched, so a test
+    asserts on DuckDB's own ``typeof(value)`` and on the round-tripped values —
+    nothing here depends on how an SDK happens to render a type name.
+    """
+
+    class Meta:
+        name = "blended_any"
+        description = "Blended 1->1 echo of one ANY-typed input column (output typed from the input)"
+        categories = ["blended", "test"]
+
+    @classmethod
+    def on_bind(cls, params: BindParams[_AnyArgs]) -> BindResponse:
+        input_schema = params.bind_call.input_schema
+        assert input_schema is not None
+        return BindResponse(output_schema=pa.schema([pa.field("value", input_schema.field(0).type)]))
+
+    @classmethod
+    def process(
+        cls,
+        params: ProcessParams[_AnyArgs],
+        state: None,
+        batch: pa.RecordBatch,
+        out: OutputCollector,
+    ) -> None:
+        out.emit(pa.record_batch([batch.column(0)], schema=params.output_schema))
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class _AnyVarargsArgs:
+    """N positional ANY-typed input columns, each free to be a different type."""
+
+    values: Annotated[list[AnyArrow], Arg(0, varargs=True, doc="Input columns of any types (echoed back)")]
+
+
+class BlendedAnyVarargsFunction(RowTransformFunction[_AnyVarargsArgs]):
+    """Blended 1->1 echo of N ANY-typed VARARGS input columns.
+
+    The varargs counterpart of ``blended_any``: every runtime column may resolve
+    to a different concrete type, so the client must take each column's type
+    from the call rather than from the (ANY) vararg element type. Output columns
+    are ``col0..colN-1`` — the names the client generates for varargs blended
+    input — each bound to its own resolved input type.
+    """
+
+    class Meta:
+        name = "blended_any_varargs"
+        description = "Blended 1->1 echo of N ANY-typed varargs input columns (col0..colN-1)"
+        categories = ["blended", "test"]
+
+    @classmethod
+    def on_bind(cls, params: BindParams[_AnyVarargsArgs]) -> BindResponse:
+        input_schema = params.bind_call.input_schema
+        assert input_schema is not None
+        return BindResponse(output_schema=pa.schema([pa.field(f"col{i}", f.type) for i, f in enumerate(input_schema)]))
+
+    @classmethod
+    def process(
+        cls,
+        params: ProcessParams[_AnyVarargsArgs],
+        state: None,
+        batch: pa.RecordBatch,
+        out: OutputCollector,
+    ) -> None:
+        out.emit(pa.record_batch(cls.input_columns(batch), schema=params.output_schema))
+
+
 class CachedExplodeFunction(RowTransformFunction[_ExplodeArgs]):
     """Cacheable blended 1->N fan-out advertising vgi.cache.per_value.
 

{vgi_python-0.33.0 → vgi_python-0.34.0}/vgi/_test_fixtures/worker.py

@@ -267,6 +267,8 @@ from vgi._test_fixtures.table.tt_pushdown import (
 )
 from vgi._test_fixtures.table_in_out import (
     BatchIndexBufferInputFunction,
+    BlendedAnyFunction,
+    BlendedAnyVarargsFunction,
     BlendedDropFunction,
     BlendedExplodeFunction,
     BufferEmitWideFunction,
@@ -432,6 +434,8 @@ _EXAMPLE_CATALOG = Catalog(
                 GeoEncode3Function,
                 RowSumFunction,
                 BlendedDropFunction,
+                BlendedAnyFunction,
+                BlendedAnyVarargsFunction,
                 BlendedExplodeFunction,
                 CachedExplodeFunction,
                 ProjectableBlendedFunction,

{vgi_python-0.33.0 → vgi_python-0.34.0}/vgi/argument_spec.py

@@ -19,7 +19,8 @@ import warnings
 from collections.abc import Sequence
 from dataclasses import dataclass
 from functools import cache
-from typing import Annotated, Any, get_args, get_origin, get_type_hints
+from types import UnionType
+from typing import Annotated, Any, Union, get_args, get_origin, get_type_hints
 
 import pyarrow as pa
 
@@ -395,14 +396,22 @@ def parameter_default_values_from_specs(
     arrays: list[pa.Array[Any]] = []
     names: list[str] = []
     for spec in default_specs:
-        if pa.types.is_null(spec.arrow_type):
-            raise ValueError(
-                f"Parameter {spec.name!r} has a default but no concrete Arrow type; "
-                "typed defaults require an explicit parameter type"
-            )
         value = spec.default_value
         if isinstance(value, pa.Scalar):
             value = value.as_py()
+        # An ANY-typed parameter (AnyArrow) has no concrete Arrow type, so a
+        # VALUE default cannot be represented. `None` still can: a null-typed
+        # array holds nulls, and it is column PRESENCE, not the value, that
+        # marks a parameter as having a default. Allowing it lets a required
+        # ANY parameter be declared `default=None` so that omitting it reaches
+        # the function's own on_bind() validation — which can name the argument
+        # and show its shape — instead of raising a bare KeyError during
+        # argument parsing, before any user code runs.
+        if pa.types.is_null(spec.arrow_type) and value is not None:
+            raise ValueError(
+                f"Parameter {spec.name!r} has a non-null default but no concrete Arrow type; "
+                "a typed default requires an explicit parameter type (None is allowed)"
+            )
         try:
             arrays.append(pa.array([value], type=spec.arrow_type))
         except (pa.ArrowException, TypeError, ValueError) as exc:
@@ -538,6 +547,30 @@ def _format_range(
     return f"{low}, {high}"
 
 
+def _unwrap_optional(annotation: Any) -> Any:
+    """``T | None`` -> ``T``; anything else unchanged.
+
+    A special marker type (``AnyArrow``, ``TableInput``) is recognised by
+    identity, so an Optional wrapper would hide it — an ``AnyArrow | None``
+    parameter would silently become a NULL-typed one, and DuckDB then cannot
+    cast anything into it. Unwrapping first lets a marker be declared nullable,
+    which is how a REQUIRED ANY argument opts into reaching its function's own
+    ``on_bind()`` validation (with ``default=None``) instead of being rejected
+    during parsing. Mirrors what ``_accepts_none`` already looks for.
+
+    Args:
+        annotation: The declared type of an ``Annotated[T, Arg(...)]`` field.
+
+    Returns:
+        The single non-``None`` member of an optional union, else ``annotation``.
+    """
+    origin = get_origin(annotation)
+    if origin is not Union and origin is not UnionType:
+        return annotation
+    members = [a for a in get_args(annotation) if a is not type(None)]
+    return members[0] if len(members) == 1 else annotation
+
+
 def _constraint_kwargs(arg: Arg[Any]) -> dict[str, Any]:
     """Extract discovery-facing constraint metadata from an [`Arg`][] descriptor.
 
@@ -744,6 +777,7 @@ def _extract_argument_specs(
                 if type_args:
                     infer_type = type_args[0]
 
+            infer_type = _unwrap_optional(infer_type)
             is_table_input = infer_type is TableInput
             is_any_type = infer_type is AnyArrow or infer_type is AnyArrowValue
 
@@ -790,8 +824,8 @@ def _extract_argument_specs(
                 # Check for special types (AnyArrow, TableInput)
                 # Priority: Arg subscript type (Arg[AnyArrow]) > class type hint
                 # Also check _returns_any_arrow_value for Annotated[AnyArrowValue, ...]
-                hint = hints.get(attr_name)
-                type_param = getattr(arg_legacy, "_type_param", None)
+                hint = _unwrap_optional(hints.get(attr_name))
+                type_param = _unwrap_optional(getattr(arg_legacy, "_type_param", None))
                 is_table_input = type_param is TableInput or hint is TableInput
                 is_any_type = (
                     type_param is AnyArrow