vgi-python 0.14.0__tar.gz → 0.15.0__tar.gz

{vgi_python-0.14.0 → vgi_python-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vgi-python
-Version: 0.14.0
+Version: 0.15.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.14.0 → vgi_python-0.15.0}/ci/run-integration.sh

@@ -257,12 +257,43 @@ rm -f "$STAGE/test/_warm.test"
 # simple_writable); the http lane's writes self-skip.
 echo "Running suite ($SUITE_GLOB, transport=$TRANSPORT) ..."
 suite_rc=0
+
+# run_unittest — invoke haybarn-unittest, streaming its output, and additionally
+# fail on a fatal-signal report that the process's own exit code cannot express.
+#
+# Catch2 arms handlers for SIGTERM/SIGINT/SIGSEGV/... for the duration of a test
+# case. Those handlers are inherited by any process the extension fork()s, and
+# run in the child if a signal lands before it execs. The child then prints a
+# full "FAILED: ... due to a fatal error condition: SIGTERM" block plus a run
+# summary — the *parent's* accumulated counters, since it's an address-space
+# copy — and dies. The parent never sees it, records no failure, and exits 0.
+# The only trace is on stdout, so that is what we scan. Seen on the shm lane in
+# https://github.com/Query-farm/vgi-python/actions/runs/29051359074.
+run_unittest() {
+  local log rc=0
+  log="$(mktemp)"
+  # `set +e` rather than `|| true`: the latter runs before PIPESTATUS is read and
+  # overwrites it with true's 0, silently swallowing every real test failure.
+  set +e
+  "$HAYBARN_UNITTEST" "$@" 2>&1 | tee "$log"
+  rc="${PIPESTATUS[0]}"
+  set -e
+  if grep -q 'due to a fatal error condition' "$log"; then
+    echo "::error::a forked child ran the test harness's signal handler (see the" \
+         "'fatal error condition' block above). The parent exited $rc and would" \
+         "otherwise have passed. This is invisible to the exit code by construction."
+    rc=1
+  fi
+  rm -f "$log"
+  return "$rc"
+}
+
 if [ "$TRANSPORT" = "launch" ]; then
-  "$HAYBARN_UNITTEST" "$SUITE_GLOB" || suite_rc=$?
+  run_unittest "$SUITE_GLOB" || suite_rc=$?
 else
-  "$HAYBARN_UNITTEST" "$SUITE_GLOB" "~test/sql/integration/simple_writable/*" || suite_rc=$?
+  run_unittest "$SUITE_GLOB" "~test/sql/integration/simple_writable/*" || suite_rc=$?
   echo "Running simple_writable (isolated process) ..."
-  "$HAYBARN_UNITTEST" "test/sql/integration/simple_writable/*" || suite_rc=$?
+  run_unittest "test/sql/integration/simple_writable/*" || suite_rc=$?
 fi
 
 exit "$suite_rc"

{vgi_python-0.14.0 → vgi_python-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.14.0"
+version = "0.15.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.14.0 → vgi_python-0.15.0}/tests/catalog/test_declarative.py

@@ -14,13 +14,13 @@ Tests cover:
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import ClassVar
+from typing import Annotated, ClassVar
 
 import pyarrow as pa
 import pytest
 from vgi_rpc.rpc import OutputCollector
 
-from vgi import Worker
+from vgi import Arg, Arguments, Worker
 from vgi.catalog import (
     AttachOpaqueData,
     Catalog,
@@ -50,6 +50,7 @@ from vgi.table_function import (
     bind_fixed_schema,
     init_single_worker,
 )
+from vgi.table_in_out_function import TableInOutGenerator
 
 # =============================================================================
 # Test Fixtures: Example Functions for Function-Backed Tables
@@ -61,6 +62,45 @@ class EmptyArgs:
     """No arguments."""
 
 
+@dataclass(frozen=True)
+class CountedRowsArgs:
+    """A required positional argument plus an optional named one."""
+
+    count: Annotated[int, Arg(0, doc="Number of rows to generate", ge=0)]
+    label: Annotated[str, Arg("label", default="rows", doc="Label for the rows")]
+
+
+@init_single_worker
+@bind_fixed_schema
+class CountedRowsFunction(TableFunctionGenerator[CountedRowsArgs]):
+    """Table function with a required positional arg, for arg-propagation tests."""
+
+    class Meta:  # noqa: D106
+        name = "counted_rows"
+        description = "Generate N rows"
+
+    FIXED_SCHEMA: ClassVar[pa.Schema] = schema({"n": pa.int64()})
+
+    @classmethod
+    def process(  # noqa: D102
+        cls, params: ProcessParams[CountedRowsArgs], state: None, out: OutputCollector
+    ) -> None:
+        out.emit(
+            pa.RecordBatch.from_pydict(
+                {"n": list(range(params.args.count))},
+                schema=params.output_schema,
+            )
+        )
+        out.finish()
+
+
+class CountedRowsInsert(TableInOutGenerator[None, None]):
+    """Stub insert function, for the write-path arg-propagation test."""
+
+    class Meta:  # noqa: D106
+        name = "counted_rows_insert"
+
+
 @init_single_worker
 @bind_fixed_schema
 class UsersFunction(TableFunctionGenerator[EmptyArgs]):
@@ -262,6 +302,87 @@ class TestTableWithFunction:
         assert info.update_function is None
         assert info.delete_function is None
 
+    def test_inlined_scan_function_carries_declared_positional_arguments(self) -> None:
+        """``Table.arguments`` must reach the scan-time bind, not just ``bind()``.
+
+        Regression: the inlined ``ScanFunctionResult`` used to hardcode empty
+        arguments, so a backing function with a required positional ``Arg(0)``
+        saw zero positional args at scan and failed with
+        ``IndexError: Argument 0: index out of range``.
+        """
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+        )
+        info = table.to_table_info("main")
+        assert info.scan_function is not None
+        batch, _ = deserialize_record_batch(info.scan_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert sfr.function_name == "counted_rows"
+        assert [s.as_py() for s in sfr.positional_arguments] == [7]
+        assert sfr.named_arguments == {}
+
+    def test_inlined_scan_function_carries_declared_named_arguments(self) -> None:
+        """Named args declared on the Table also reach the scan-time bind."""
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(
+                positional=(pa.scalar(3),),
+                named={"label": pa.scalar("widgets")},
+            ),
+        )
+        info = table.to_table_info("main")
+        assert info.scan_function is not None
+        batch, _ = deserialize_record_batch(info.scan_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert [s.as_py() for s in sfr.positional_arguments] == [3]
+        assert {k: v.as_py() for k, v in sfr.named_arguments.items()} == {"label": "widgets"}
+
+    def test_inlined_write_functions_ignore_table_arguments(self) -> None:
+        """Only the *scan* function is parameterized by ``Table.arguments``.
+
+        Insert/update/delete functions receive their rows via the streaming table
+        input, so their inlined results stay argument-free.
+        """
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+            insert_function=CountedRowsInsert,
+        )
+        info = table.to_table_info("main")
+        assert info.insert_function is not None
+        batch, _ = deserialize_record_batch(info.insert_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert sfr.positional_arguments == []
+        assert sfr.named_arguments == {}
+
+    def test_none_positional_slot_is_rejected(self) -> None:
+        """A ``None`` positional slot would shift later args — reject it loudly.
+
+        Exercised on the ``scan_arguments_from`` helper directly: a Table carrying
+        such a slot already fails earlier, in bind()-based schema derivation. This
+        guards the serializer against ever silently dropping the slot instead.
+        """
+        from vgi.catalog.catalog_interface import scan_arguments_from
+
+        with pytest.raises(ValueError, match="positional slot 0 is None"):
+            scan_arguments_from(Arguments(positional=(None, pa.scalar(7))))
+
+    def test_scan_arguments_from_none_is_empty(self) -> None:
+        """A table declaring no arguments serializes empty args (unchanged behavior)."""
+        from vgi.catalog.catalog_interface import scan_arguments_from
+
+        assert scan_arguments_from(None) == ([], {})
+
     def test_table_propagates_cardinality_to_table_info(self) -> None:
         """``Table.cardinality_estimate`` / ``cardinality_max`` flow into TableInfo.
 
@@ -1121,8 +1242,6 @@ class TestSchemaDescriptor:
         client hits a cold miss. Reporting an explicit ``0`` for empty
         categories lets the client skip the bulk RPC entirely.
         """
-        from typing import Annotated
-
         from vgi.arguments import Param, Returns
         from vgi.scalar_function import ScalarFunction
 
@@ -1174,8 +1293,6 @@ class TestLegacyFunctionsListSchemaInfo:
     """
 
     def _interface(self) -> ReadOnlyCatalogInterface:
-        from typing import Annotated
-
         from vgi.arguments import Param, Returns
         from vgi.scalar_function import ScalarFunction
 
@@ -1694,6 +1811,70 @@ class TestTableScanFunctionGet:
         )
         assert isinstance(result, ScanFunctionResult)
         assert result.function_name == "users"
+        # No declared Table.arguments => empty args, unchanged behavior.
+        assert result.positional_arguments == []
+        assert result.named_arguments == {}
+
+    def test_auto_scan_forwards_declared_arguments(self) -> None:
+        """The RPC auto-impl forwards ``Table.arguments`` to the scan bind.
+
+        Companion to the inlined-path regression: workers that don't inline
+        ``TableInfo.scan_function`` reach the scan through this RPC instead.
+        """
+        counted = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(
+                positional=(pa.scalar(7),),
+                named={"label": pa.scalar("widgets")},
+            ),
+        )
+
+        class TestCatalog(ReadOnlyCatalogInterface):
+            catalog = Catalog(
+                name="test",
+                schemas=[Schema(name="main", tables=[counted])],
+            )
+
+        result = TestCatalog().table_scan_function_get(
+            attach_opaque_data=AttachOpaqueData(b"test"),
+            transaction_opaque_data=None,
+            schema_name="main",
+            name="counted",
+            at_unit=None,
+            at_value=None,
+        )
+        assert result.function_name == "counted_rows"
+        assert [s.as_py() for s in result.positional_arguments] == [7]
+        assert {k: v.as_py() for k, v in result.named_arguments.items()} == {"label": "widgets"}
+
+    def test_scan_branches_default_inherits_declared_arguments(self) -> None:
+        """The default ``table_scan_branches_get`` delegates to the legacy method.
+
+        So the one-branch synthesis must carry the declared arguments too.
+        """
+        counted = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+        )
+
+        class TestCatalog(ReadOnlyCatalogInterface):
+            catalog = Catalog(
+                name="test",
+                schemas=[Schema(name="main", tables=[counted])],
+            )
+
+        result = TestCatalog().table_scan_branches_get(
+            attach_opaque_data=AttachOpaqueData(b"test"),
+            transaction_opaque_data=None,
+            schema_name="main",
+            name="counted",
+            at_unit=None,
+            at_value=None,
+        )
+        assert len(result.branches) == 1
+        assert [s.as_py() for s in result.branches[0].positional_arguments] == [7]
 
     def test_explicit_columns_table_raises(self) -> None:
         """Tables with explicit columns raise NotImplementedError."""

vgi_python-0.15.0/tests/catalog/test_required_filters.py

@@ -0,0 +1,92 @@
+"""Unit tests for the ``Table.required_filters`` field.
+
+The field is purely declarative on the Python side — enforcement runs in the C++
+optimizer extension. It is conjunctive normal form: an AND (outer tuple) of
+OR-groups (inner tuples) of dotted-path column references. These tests cover:
+
+1. Default-empty behaviour (no field set → ``TableInfo`` ships an empty list).
+2. Populated round-trip (singleton groups + a multi-path OR-group preserved).
+3. Validation: each path's leading dotted segment must be a real column.
+4. Empty-string paths and empty OR-groups are rejected loudly.
+"""
+
+from __future__ import annotations
+
+import pyarrow as pa
+import pytest
+
+from vgi.catalog.descriptors import Table
+
+
+class TestRequiredFiltersField:
+    """Behaviour of the declarative required_filters field."""
+
+    def _bbox_columns(self) -> pa.Schema:
+        """Return a small schema with a ``bbox`` STRUCT plus flat id/ticker."""
+        return pa.schema(
+            [
+                ("id", pa.int64()),
+                ("ticker", pa.string()),
+                (
+                    "bbox",
+                    pa.struct(
+                        [
+                            ("xmin", pa.float64()),
+                            ("ymin", pa.float64()),
+                            ("xmax", pa.float64()),
+                            ("ymax", pa.float64()),
+                        ]
+                    ),
+                ),
+            ]
+        )
+
+    def test_default_is_empty_tuple(self) -> None:
+        """Default is an empty tuple; TableInfo wire field is an empty list."""
+        t = Table(name="t", columns=self._bbox_columns())
+        assert t.required_filters == ()
+        info = t.to_table_info("main")
+        assert info.required_filters == []
+
+    def test_singleton_groups_round_trip(self) -> None:
+        """Singleton groups (the AND-only case) survive as a list of lists."""
+        groups = (("bbox.xmin",), ("bbox.xmax",), ("bbox.ymin",), ("bbox.ymax",))
+        t = Table(name="place", columns=self._bbox_columns(), required_filters=groups)
+        assert t.required_filters == groups
+        info = t.to_table_info("main")
+        assert info.required_filters == [list(g) for g in groups]
+
+    def test_or_group_round_trip(self) -> None:
+        """A multi-path OR-group survives alongside a singleton mandatory group."""
+        groups = (("id",), ("id", "ticker"))
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=groups)
+        info = t.to_table_info("main")
+        assert info.required_filters == [["id"], ["id", "ticker"]]
+
+    def test_top_level_path(self) -> None:
+        """A top-level column name (no dots) is a valid path."""
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("id",),))
+        assert t.to_table_info("main").required_filters == [["id"]]
+
+    def test_nested_path_is_not_unpacked_for_validation(self) -> None:
+        """Only the leading dotted segment is validated; deeper segments pass through."""
+        # Subfield validity isn't checked here (descriptor doesn't unpack struct
+        # schemas) — only the leading segment matters. Typos like ``bbox.nope``
+        # go through and DuckDB catches them at scan time.
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("bbox.nope",),))
+        assert t.required_filters == (("bbox.nope",),)
+
+    def test_unknown_leading_segment_raises(self) -> None:
+        """A path whose leading segment is not a real column is rejected."""
+        with pytest.raises(ValueError, match=r"unknown column 'nope'"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("nope.xmin",),))
+
+    def test_empty_string_path_raises(self) -> None:
+        """An empty string path is rejected loudly rather than silently accepted."""
+        with pytest.raises(ValueError, match="must not contain empty strings"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("",),))
+
+    def test_empty_group_raises(self) -> None:
+        """An empty OR-group is rejected loudly."""
+        with pytest.raises(ValueError, match="must not contain empty groups"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=((),))

{vgi_python-0.14.0 → vgi_python-0.15.0}/tests/catalog/test_serialization.py

@@ -1149,10 +1149,10 @@ class TestArrowSchemaCorrectness:
         assert schema.field("cardinality_estimate").type == pa.int64()
         assert schema.field("column_statistics").type == pa.binary()
         assert schema.field("bind_result").type == pa.binary()
-        assert schema.field("required_field_filter_paths").type == pa.list_(pa.string())
-        # required_field_filter_paths is the LAST field (positional schema
-        # compatibility: old C++ extensions ignore-by-position trailing fields).
-        assert schema.field(len(schema) - 1).name == "required_field_filter_paths"
+        assert schema.field("required_filters").type == pa.list_(pa.list_(pa.string()))
+        # required_filters is the LAST field (positional schema compatibility:
+        # old C++ extensions ignore-by-position trailing fields).
+        assert schema.field(len(schema) - 1).name == "required_filters"
 
     def test_view_info_schema(self) -> None:
         """Verify ViewInfo Arrow schema."""

{vgi_python-0.14.0 → vgi_python-0.15.0}/tests/landing/describe.expected.json

@@ -5,7 +5,7 @@
       "counts": {
         "functions": 183,
         "schemas": 2,
-        "tables": 58,
+        "tables": 59,
         "views": 3
       },
       "data_version_spec": null,
@@ -225,9 +225,14 @@
             },
             {
               "cols": 2,
-              "comment": "rff_none — control table with no required_field_filter_paths (opt-out fast path).",
+              "comment": "rff_none — control table with no required_filters (opt-out fast path).",
               "name": "rff_none"
             },
+            {
+              "cols": 2,
+              "comment": "rff_or — requires a filter on 'a' OR 'b' (a single OR-group).",
+              "name": "rff_or"
+            },
             {
               "cols": 2,
               "comment": "rff_parquet — native read_parquet delegation with bbox.* required filters.",
@@ -2316,13 +2321,13 @@
             },
             {
               "args": [],
-              "doc": "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_field_filter_paths tests",
+              "doc": "rff_nested — nested STRUCT(wrapper.mid.leaf) for required_filters tests",
               "name": "rff_nested_scan",
               "type": "table"
             },
             {
               "args": [],
-              "doc": "rff_none — control table with no required_field_filter_paths",
+              "doc": "rff_none — control table with no required_filters",
               "name": "rff_none_scan",
               "type": "table"
             },
@@ -2334,13 +2339,13 @@
             },
             {
               "args": [],
-              "doc": "rff_simple — flat columns (a, b) for required_field_filter_paths tests",
+              "doc": "rff_simple — flat columns (a, b) for required_filters tests",
               "name": "rff_simple_scan",
               "type": "table"
             },
             {
               "args": [],
-              "doc": "rff_struct — STRUCT(s.a, s.b) + other for required_field_filter_paths tests",
+              "doc": "rff_struct — STRUCT(s.a, s.b) + other for required_filters tests",
               "name": "rff_struct_scan",
               "type": "table"
             },

{vgi_python-0.14.0 → vgi_python-0.15.0}/vgi/_test_fixtures/table/required_filters.py

@@ -3,7 +3,7 @@
 """Scan functions backing the ``rff_*`` required-filter sqllogictest Tables.
 
 Used by the ``vgi_required_filters_*.test`` matrix. These fixtures exercise
-the ``Table.required_field_filter_paths`` field +
+the ``Table.required_filters`` field +
 the C++ optimizer extension that enforces it. The five tables form a small
 matrix:
 
@@ -99,10 +99,10 @@ RFF_NONE_COLUMNS = pa.schema(
 )
 
 # rff_rowid — a row-id column (virtual, hidden from SELECT *) alongside a bbox
-# struct with required_field_filter_paths. A `WHERE rowid = N` predicate pushes
+# struct with required_filters. A `WHERE rowid = N` predicate pushes
 # a table_filter keyed by the COLUMN_IDENTIFIER_ROW_ID sentinel (>> column
 # count), which the optimizer's required-filter check must skip rather than
-# index out of bounds. See required_field_filter_paths_native.test.
+# index out of bounds. See required_filters_native.test.
 RFF_ROWID_COLUMNS = pa.schema(
     [
         pa.field("row_id", pa.int64(), metadata={b"is_row_id": b""}),
@@ -124,7 +124,7 @@ RFF_ROWID_COLUMNS = pa.schema(
 
 RffSimpleScanFunction = _static_scan_function(
     func_name="rff_simple_scan",
-    func_description="rff_simple — flat columns (a, b) for required_field_filter_paths tests",
+    func_description="rff_simple — flat columns (a, b) for required_filters tests",
     output_schema=RFF_SIMPLE_COLUMNS,
     data={
         "a": [1, 2, 3],
@@ -134,7 +134,7 @@ RffSimpleScanFunction = _static_scan_function(
 
 RffStructScanFunction = _static_scan_function(
     func_name="rff_struct_scan",
-    func_description="rff_struct — STRUCT(s.a, s.b) + other for required_field_filter_paths tests",
+    func_description="rff_struct — STRUCT(s.a, s.b) + other for required_filters tests",
     output_schema=RFF_STRUCT_COLUMNS,
     data={
         "s": [
@@ -148,7 +148,7 @@ RffStructScanFunction = _static_scan_function(
 
 RffNestedScanFunction = _static_scan_function(
     func_name="rff_nested_scan",
-    func_description="rff_nested — nested STRUCT(wrapper.mid.leaf) for required_field_filter_paths tests",
+    func_description="rff_nested — nested STRUCT(wrapper.mid.leaf) for required_filters tests",
     output_schema=RFF_NESTED_COLUMNS,
     data={
         "wrapper": [
@@ -174,7 +174,7 @@ RffMultiScanFunction = _static_scan_function(
 
 RffNoneScanFunction = _static_scan_function(
     func_name="rff_none_scan",
-    func_description="rff_none — control table with no required_field_filter_paths",
+    func_description="rff_none — control table with no required_filters",
     output_schema=RFF_NONE_COLUMNS,
     data={
         "a": [1, 2, 3],