vgi-python 0.15.0__tar.gz → 0.16.1__tar.gz

{vgi_python-0.15.0 → vgi_python-0.16.1}/.github/workflows/integration.yml

@@ -32,7 +32,7 @@ env:
   VGI_REF: main
   # The Haybarn release providing the prebuilt haybarn-unittest binary. Must be
   # ABI-compatible with the community-published vgi extension.
-  HAYBARN_RELEASE: haybarn-v1.5.4-rc1
+  HAYBARN_RELEASE: haybarn-v1.5.4-rc3
 
 jobs:
   integration:

{vgi_python-0.15.0 → vgi_python-0.16.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vgi-python
-Version: 0.15.0
+Version: 0.16.1
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.15.0 → vgi_python-0.16.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.15.0"
+version = "0.16.1"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.15.0 → vgi_python-0.16.1}/tests/landing/describe.expected.json

@@ -3,7 +3,7 @@
     {
       "attach_options": [],
       "counts": {
-        "functions": 183,
+        "functions": 203,
         "schemas": 2,
         "tables": 59,
         "views": 3
@@ -655,6 +655,50 @@
               "returns": "binary",
               "type": "scalar"
             },
+            {
+              "args": [
+                {
+                  "desc": "Value",
+                  "name": "value",
+                  "type": "int64"
+                },
+                {
+                  "desc": "Constant addend",
+                  "name": "addend",
+                  "type": "int64"
+                }
+              ],
+              "doc": "value + const addend (advertises vgi.cache.ttl)",
+              "name": "cached_add_const",
+              "returns": "int64",
+              "type": "scalar"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Value to double",
+                  "name": "value",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Doubles a BIGINT value (advertises vgi.cache.ttl for per-value memo)",
+              "name": "cached_double_scalar",
+              "returns": "int64",
+              "type": "scalar"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Value",
+                  "name": "value",
+                  "type": "int64"
+                }
+              ],
+              "doc": "value -> 'lbl-<value>' or NULL for negatives (advertises vgi.cache.ttl)",
+              "name": "cached_label",
+              "returns": "string",
+              "type": "scalar"
+            },
             {
               "args": [
                 {
@@ -1247,6 +1291,30 @@
               "returns": "string",
               "type": "scalar"
             },
+            {
+              "args": [
+                {
+                  "desc": "Input column (ignored)",
+                  "name": "x",
+                  "type": "double"
+                }
+              ],
+              "doc": "Blended 1->0 map emitting a single 0-row batch (literal scan-mode)",
+              "name": "blended_drop",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Fan-out count: emit rows 0..n-1 for this input row",
+                  "name": "n",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Blended 1->N fan-out (emit 0..n-1 per input row) with row provenance",
+              "name": "blended_explode",
+              "type": "table"
+            },
             {
               "args": [
                 {
@@ -1456,6 +1524,54 @@
               "name": "cache_parallel",
               "type": "table"
             },
+            {
+              "args": [
+                {
+                  "desc": "Rows per (region, year) partition",
+                  "name": "rows_per_partition",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Per-partition cacheable over (region, year) SINGLE_VALUE partition columns",
+              "name": "cache_partition_multicol",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Rows per country partition",
+                  "name": "rows_per_country",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Per-partition cacheable; work-queue fan-out (parallel capture); one NULL partition",
+              "name": "cache_partition_parallel",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Rows per country partition",
+                  "name": "rows_per_country",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Per-partition cacheable with projection pushdown + explicit partition_values",
+              "name": "cache_partition_proj",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Rows per country partition",
+                  "name": "rows_per_country",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Per-partition cacheable single-value-partitioned result (vgi.cache.partition_scope)",
+              "name": "cache_partition_scope",
+              "type": "table"
+            },
             {
               "args": [
                 {
@@ -1551,6 +1667,42 @@
               "name": "cacheable_numbers",
               "type": "table"
             },
+            {
+              "args": [
+                {
+                  "desc": "Input column",
+                  "name": "x",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Cacheable blended map x -> x*2 (advertises vgi.cache.ttl)",
+              "name": "cached_double",
+              "type": "table"
+            },
+            {
+              "args": [],
+              "doc": "Cacheable classic (TABLE-input) passthrough (advertises vgi.cache.ttl)",
+              "name": "cached_echo",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Input column",
+                  "name": "x",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Blended map x->x*2 with always-revalidate (304 not_modified) contract",
+              "name": "cached_reval_double",
+              "type": "table"
+            },
+            {
+              "args": [],
+              "doc": "Classic passthrough with always-revalidate (304 not_modified) contract",
+              "name": "cached_reval_echo",
+              "type": "table"
+            },
             {
               "args": [],
               "doc": "Scan colors table (ENUM column)",
@@ -1820,6 +1972,78 @@
               "name": "generator_exception",
               "type": "table"
             },
+            {
+              "args": [
+                {
+                  "desc": "Latitude input column",
+                  "name": "latitude",
+                  "type": "double"
+                },
+                {
+                  "desc": "Longitude input column",
+                  "name": "longitude",
+                  "type": "double"
+                },
+                {
+                  "default": "4",
+                  "desc": "Rounding precision",
+                  "name": "precision",
+                  "named": true,
+                  "type": "int64"
+                }
+              ],
+              "doc": "Blended per-row geo encoder (lat, lon -> geohash)",
+              "name": "geo_encode",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Latitude input column",
+                  "name": "latitude",
+                  "type": "double"
+                },
+                {
+                  "desc": "Longitude input column",
+                  "name": "longitude",
+                  "type": "double"
+                },
+                {
+                  "desc": "Altitude input column",
+                  "name": "altitude",
+                  "type": "double"
+                },
+                {
+                  "default": "4",
+                  "desc": "Rounding precision",
+                  "name": "precision",
+                  "named": true,
+                  "type": "int64"
+                }
+              ],
+              "doc": "Blended per-row geo encoder (lat, lon, alt -> geohash)",
+              "name": "geo_encode",
+              "type": "table"
+            },
+            {
+              "args": [
+                {
+                  "desc": "Input column (echoed as output)",
+                  "name": "x",
+                  "type": "int64"
+                },
+                {
+                  "default": "\"range\"",
+                  "desc": "range | length | base64",
+                  "name": "mode",
+                  "named": true,
+                  "type": "string"
+                }
+              ],
+              "doc": "Adversarial blended fixture emitting malformed vgi_rpc.parent_row",
+              "name": "hostile_provenance",
+              "type": "table"
+            },
             {
               "args": [
                 {
@@ -2237,6 +2461,18 @@
               "name": "profiling_demo",
               "type": "table"
             },
+            {
+              "args": [
+                {
+                  "desc": "Input column",
+                  "name": "x",
+                  "type": "int64"
+                }
+              ],
+              "doc": "Blended 1->1 map with projection_pushdown + two output columns",
+              "name": "projectable_blended",
+              "type": "table"
+            },
             {
               "args": [
                 {
@@ -2349,6 +2585,25 @@
               "name": "rff_struct_scan",
               "type": "table"
             },
+            {
+              "args": [
+                {
+                  "desc": "Numeric input columns",
+                  "name": "values",
+                  "type": "double"
+                },
+                {
+                  "default": "false",
+                  "desc": "Sum absolute values",
+                  "name": "absolute",
+                  "named": true,
+                  "type": "bool"
+                }
+              ],
+              "doc": "Blended per-row varargs sum",
+              "name": "row_sum",
+              "type": "table"
+            },
             {
               "args": [
                 {
@@ -2731,6 +2986,20 @@
               "name": "buffer_input",
               "type": "table_in_out"
             },
+            {
+              "args": [
+                {
+                  "default": "false",
+                  "desc": "Whether to log during processing",
+                  "name": "logging",
+                  "named": true,
+                  "type": "bool"
+                }
+              ],
+              "doc": "Cacheable column-wise sum across all input (advertises vgi.cache.ttl)",
+              "name": "cached_sum_all",
+              "type": "table_in_out"
+            },
             {
               "args": [],
               "doc": "Worker raises during combine (test)",
@@ -2943,6 +3212,12 @@
               "name": "slow_cancellable_buffering",
               "type": "table_in_out"
             },
+            {
+              "args": [],
+              "doc": "Per-substream partial sum emitted at finalize (parallel streaming finalize)",
+              "name": "substream_partial_sum",
+              "type": "table_in_out"
+            },
             {
               "args": [
                 {
@@ -2959,7 +3234,7 @@
             },
             {
               "args": [],
-              "doc": "Distributed sum using simple callback API",
+              "doc": "Distributed sum using the buffered (Sink+Combine+Source) model",
               "name": "sum_all_columns_simple_distributed",
               "type": "table_in_out"
             }

vgi_python-0.16.1/tests/table_in_out/generator/test_substream_partial_sum_function.py

@@ -0,0 +1,70 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Tests for SubstreamPartialSumFunction (parallel streaming finalize, A4).
+
+At the single-worker client level there is exactly one substream, so ``finish()``
+sees all input and emits one partial row equal to the whole sum. The per-substream
+fan-out (many partials re-aggregated by an outer ``SELECT sum()``) is exercised by
+the C++ integration test ``table_in_out/parallel_finalize.test`` over both
+transports; here we assert the worker-side transform/finish contract in isolation.
+"""
+
+from __future__ import annotations
+
+import pyarrow as pa
+
+from tests.conftest import filter_non_empty, make_schema
+from vgi.client import Client
+
+
+class TestSubstreamPartialSumFunction:
+    """Worker-side contract for substream_partial_sum."""
+
+    def test_partial_sum_single_batch(self, fixture_worker: str) -> None:
+        """One substream (single client worker) → one partial = the whole sum."""
+        schema = make_schema([pa.field("n", pa.int64())])
+        batch = pa.RecordBatch.from_pydict({"n": [1, 2, 3, 4, 5]}, schema=schema)
+        with Client(fixture_worker) as client:
+            output = list(
+                client.table_in_out_function(
+                    function_name="substream_partial_sum",
+                    input=iter([batch]),
+                )
+            )
+        non_empty = filter_non_empty(output)
+        assert len(non_empty) == 1
+        assert non_empty[0].to_pydict() == {"n": [15]}
+
+    def test_partial_sum_many_batches(self, fixture_worker: str) -> None:
+        """Accumulates across many input batches, emits one partial at finalize."""
+        schema = make_schema([pa.field("n", pa.int64())])
+        batches = [
+            pa.RecordBatch.from_pydict({"n": list(range(i * 100, (i + 1) * 100))}, schema=schema) for i in range(20)
+        ]
+        expected = sum(range(0, 2000))
+        with Client(fixture_worker) as client:
+            output = list(
+                client.table_in_out_function(
+                    function_name="substream_partial_sum",
+                    input=iter(batches),
+                )
+            )
+        non_empty = filter_non_empty(output)
+        assert len(non_empty) == 1
+        assert non_empty[0].to_pydict() == {"n": [expected]}
+
+    def test_partial_sum_empty_input(self, fixture_worker: str) -> None:
+        """No input rows → the single substream's partial is 0."""
+        schema = make_schema([pa.field("n", pa.int64())])
+        empty = pa.RecordBatch.from_pydict({"n": []}, schema=schema)
+        with Client(fixture_worker) as client:
+            output = list(
+                client.table_in_out_function(
+                    function_name="substream_partial_sum",
+                    input=iter([empty]),
+                )
+            )
+        non_empty = filter_non_empty(output)
+        # A single substream with zero rows still finalizes → one 0 partial.
+        assert len(non_empty) == 1
+        assert non_empty[0].to_pydict() == {"n": [0]}

{vgi_python-0.15.0 → vgi_python-0.16.1}/tests/table_in_out/generator/test_sum_all_columns_function.py

@@ -197,13 +197,13 @@ class TestSumAllColumnsFunctionDistributed:
 
 
 class TestSumAllColumnsSimpleDistributed:
-    """Tests for sum_all_columns_simple_distributed (TableInOutFunction)."""
+    """Tests for sum_all_columns_simple_distributed (buffered global reduction)."""
 
     def test_sum_simple_distributed_basic(self, fixture_worker: str, numeric_batches: list[pa.RecordBatch]) -> None:
         """Should sum all numeric columns across all batches."""
         with Client(fixture_worker) as client:
             output_batches = list(
-                client.table_in_out_function(
+                client.table_buffering_function(
                     function_name="sum_all_columns_simple_distributed",
                     input=iter(numeric_batches),
                 )
@@ -238,7 +238,7 @@ class TestSumAllColumnsSimpleDistributed:
 
         with Client(fixture_worker) as client:
             output_batches = list(
-                client.table_in_out_function(
+                client.table_buffering_function(
                     function_name="sum_all_columns_simple_distributed",
                     input=iter(batches),
                 )
@@ -256,7 +256,7 @@ class TestSumAllColumnsSimpleDistributed:
         """Should exclude non-numeric columns from output."""
         with Client(fixture_worker) as client:
             output_batches = list(
-                client.table_in_out_function(
+                client.table_buffering_function(
                     function_name="sum_all_columns_simple_distributed",
                     input=iter(simple_batches),
                 )
@@ -276,7 +276,7 @@ class TestSumAllColumnsSimpleDistributed:
 
         with Client(fixture_worker) as client:
             output_batches = list(
-                client.table_in_out_function(
+                client.table_buffering_function(
                     function_name="sum_all_columns_simple_distributed",
                     input=iter([empty_batch]),
                 )

vgi_python-0.16.1/tests/table_in_out/test_blended_metadata.py

@@ -0,0 +1,150 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Metadata resolution + foot-gun guards for blended RowTransformFunction (Phase B)."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Annotated
+
+import pyarrow as pa
+import pytest
+
+from vgi.arguments import Arg, TableInput
+from vgi.invocation import BindResponse
+from vgi.metadata import (
+    CatalogFunctionType,
+    ResolvedMetadata,
+    arrow_to_metadata,
+    metadata_to_arrow,
+    resolve_metadata,
+)
+from vgi.schema_utils import schema
+from vgi.table_function import BindParams
+from vgi.table_in_out_function import RowTransformFunction
+
+
+@dataclass(slots=True, frozen=True, kw_only=True)
+class _XYArgs:
+    """Two positional input columns + one named option."""
+
+    x: Annotated[float, Arg(0)]
+    y: Annotated[float, Arg(1)]
+    opt: Annotated[int, Arg("opt", default=1)] = 1
+
+
+class _GoodBlended(RowTransformFunction[_XYArgs]):
+    """A valid blended function."""
+
+    class Meta:
+        """Metadata."""
+
+        name = "good_blended"
+
+    @classmethod
+    def on_bind(cls, params: BindParams[_XYArgs]) -> BindResponse:
+        """Fixed output schema."""
+        return BindResponse(output_schema=schema({"z": pa.float64()}))
+
+
+class TestBlendedResolution:
+    """Happy-path resolution of a blended function."""
+
+    def test_resolves_as_blended_table(self) -> None:
+        """input_from_args=True, function_type=TABLE, no finalize."""
+        m = resolve_metadata(_GoodBlended)
+        assert m.function_type is CatalogFunctionType.TABLE
+        assert m.input_from_args is True
+        assert m.has_finalize is False
+        positions = {p.position for p in m.parameters}
+        assert 0 in positions and 1 in positions and "opt" in positions
+
+    def test_arrow_roundtrip_preserves_input_from_args(self) -> None:
+        """input_from_args survives the Arrow + dict round-trips."""
+        m = resolve_metadata(_GoodBlended)
+        assert arrow_to_metadata(metadata_to_arrow(m)).input_from_args is True
+        assert ResolvedMetadata.from_dict(m.to_dict()).input_from_args is True
+
+
+class TestBlendedFootguns:
+    """resolve_metadata rejects the invalid blended shapes."""
+
+    def test_reject_finalize_override(self) -> None:
+        """A blended fn that overrides finish()/finalize() is rejected."""
+
+        class _WithFinalize(RowTransformFunction[_XYArgs]):
+            """Illegal: blended with a finalize."""
+
+            class Meta:
+                """Metadata."""
+
+                name = "blended_finalize"
+
+            @classmethod
+            def finish(cls, params, states):  # noqa: ANN001, ANN206, D102
+                return []
+
+        with pytest.raises(TypeError, match="cannot override finalize"):
+            resolve_metadata(_WithFinalize)
+
+    def test_reject_table_input(self) -> None:
+        """A blended fn that declares a TableInput arg is rejected."""
+
+        @dataclass(slots=True, frozen=True, kw_only=True)
+        class _TblArgs:
+            """Illegal: a TableInput on a blended fn."""
+
+            x: Annotated[float, Arg(0)]
+            data: Annotated[TableInput, Arg(1)]
+
+        class _WithTable(RowTransformFunction[_TblArgs]):
+            """Illegal: blended with TableInput."""
+
+            class Meta:
+                """Metadata."""
+
+                name = "blended_table"
+
+        with pytest.raises(TypeError, match="must not declare a TableInput"):
+            resolve_metadata(_WithTable)
+
+    def test_reject_positional_const(self) -> None:
+        """A blended fn with a positional const arg is rejected."""
+
+        @dataclass(slots=True, frozen=True, kw_only=True)
+        class _ConstArgs:
+            """Illegal: a positional const on a blended fn."""
+
+            k: Annotated[int, Arg(0, const=True)]
+            x: Annotated[float, Arg(1)]
+
+        class _WithConst(RowTransformFunction[_ConstArgs]):
+            """Illegal: blended with a positional const."""
+
+            class Meta:
+                """Metadata."""
+
+                name = "blended_const"
+
+        with pytest.raises(TypeError, match="cannot take a positional const"):
+            resolve_metadata(_WithConst)
+
+    def test_reject_zero_positional(self) -> None:
+        """A blended fn with no positional args is rejected."""
+
+        @dataclass(slots=True, frozen=True, kw_only=True)
+        class _NamedOnlyArgs:
+            """Illegal: no positional input columns."""
+
+            opt: Annotated[int, Arg("opt", default=1)] = 1
+
+        class _NamedOnly(RowTransformFunction[_NamedOnlyArgs]):
+            """Illegal: blended with no positional args."""
+
+            class Meta:
+                """Metadata."""
+
+                name = "blended_named_only"
+
+        with pytest.raises(TypeError, match="at least one positional Arg"):
+            resolve_metadata(_NamedOnly)