vgi-python 0.13.0__tar.gz → 0.15.0__tar.gz

{vgi_python-0.13.0 → vgi_python-0.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vgi-python
-Version: 0.13.0
+Version: 0.15.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python
@@ -162,7 +162,7 @@ Requires-Dist: httpx>=0.24
 Requires-Dist: platformdirs
 Requires-Dist: pyarrow
 Requires-Dist: typer>=0.9
-Requires-Dist: vgi-rpc>=0.23.0
+Requires-Dist: vgi-rpc>=0.24.0
 Provides-Extra: azure
 Requires-Dist: azure-identity>=1.16.0; extra == 'azure'
 Requires-Dist: pymssql>=2.3.0; extra == 'azure'

{vgi_python-0.13.0 → vgi_python-0.15.0}/ci/run-integration.sh

@@ -73,6 +73,11 @@ if [ "$TRANSPORT" = "http" ]; then
   #   * buffer_input/sizes.test — input buffering semantics differ over HTTP
   #     (same known limitation). (scale.test_slow is a .test_slow file and is
   #     never staged below, which only finds *.test.)
+  # cache/revalidate.test now runs on http too: HTTP conditional revalidation is
+  # implemented (C++ /init-request validators + vgi-rpc >=0.24.0 surfacing them to
+  # the producer's first process()). It needs a community vgi extension carrying
+  # that C++ change; if the http lane fails on it, the community extension predates
+  # the change and needs republishing.
   EXTRA_SKIP=(
     -not -name 'projection_pushdown_repro.test'
     -not -name 'dynamic_filter.test'
@@ -252,12 +257,43 @@ rm -f "$STAGE/test/_warm.test"
 # simple_writable); the http lane's writes self-skip.
 echo "Running suite ($SUITE_GLOB, transport=$TRANSPORT) ..."
 suite_rc=0
+
+# run_unittest — invoke haybarn-unittest, streaming its output, and additionally
+# fail on a fatal-signal report that the process's own exit code cannot express.
+#
+# Catch2 arms handlers for SIGTERM/SIGINT/SIGSEGV/... for the duration of a test
+# case. Those handlers are inherited by any process the extension fork()s, and
+# run in the child if a signal lands before it execs. The child then prints a
+# full "FAILED: ... due to a fatal error condition: SIGTERM" block plus a run
+# summary — the *parent's* accumulated counters, since it's an address-space
+# copy — and dies. The parent never sees it, records no failure, and exits 0.
+# The only trace is on stdout, so that is what we scan. Seen on the shm lane in
+# https://github.com/Query-farm/vgi-python/actions/runs/29051359074.
+run_unittest() {
+  local log rc=0
+  log="$(mktemp)"
+  # `set +e` rather than `|| true`: the latter runs before PIPESTATUS is read and
+  # overwrites it with true's 0, silently swallowing every real test failure.
+  set +e
+  "$HAYBARN_UNITTEST" "$@" 2>&1 | tee "$log"
+  rc="${PIPESTATUS[0]}"
+  set -e
+  if grep -q 'due to a fatal error condition' "$log"; then
+    echo "::error::a forked child ran the test harness's signal handler (see the" \
+         "'fatal error condition' block above). The parent exited $rc and would" \
+         "otherwise have passed. This is invisible to the exit code by construction."
+    rc=1
+  fi
+  rm -f "$log"
+  return "$rc"
+}
+
 if [ "$TRANSPORT" = "launch" ]; then
-  "$HAYBARN_UNITTEST" "$SUITE_GLOB" || suite_rc=$?
+  run_unittest "$SUITE_GLOB" || suite_rc=$?
 else
-  "$HAYBARN_UNITTEST" "$SUITE_GLOB" "~test/sql/integration/simple_writable/*" || suite_rc=$?
+  run_unittest "$SUITE_GLOB" "~test/sql/integration/simple_writable/*" || suite_rc=$?
   echo "Running simple_writable (isolated process) ..."
-  "$HAYBARN_UNITTEST" "test/sql/integration/simple_writable/*" || suite_rc=$?
+  run_unittest "test/sql/integration/simple_writable/*" || suite_rc=$?
 fi
 
 exit "$suite_rc"

{vgi_python-0.13.0 → vgi_python-0.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.13.0"
+version = "0.15.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [
@@ -40,7 +40,7 @@ dependencies = [
     "pyarrow",
     "typer>=0.9",
     "platformdirs",
-    "vgi-rpc>=0.23.0",
+    "vgi-rpc>=0.24.0",
     "httpx>=0.24",
 ]
 

{vgi_python-0.13.0 → vgi_python-0.15.0}/tests/catalog/test_declarative.py

@@ -14,13 +14,13 @@ Tests cover:
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import ClassVar
+from typing import Annotated, ClassVar
 
 import pyarrow as pa
 import pytest
 from vgi_rpc.rpc import OutputCollector
 
-from vgi import Worker
+from vgi import Arg, Arguments, Worker
 from vgi.catalog import (
     AttachOpaqueData,
     Catalog,
@@ -50,6 +50,7 @@ from vgi.table_function import (
     bind_fixed_schema,
     init_single_worker,
 )
+from vgi.table_in_out_function import TableInOutGenerator
 
 # =============================================================================
 # Test Fixtures: Example Functions for Function-Backed Tables
@@ -61,6 +62,45 @@ class EmptyArgs:
     """No arguments."""
 
 
+@dataclass(frozen=True)
+class CountedRowsArgs:
+    """A required positional argument plus an optional named one."""
+
+    count: Annotated[int, Arg(0, doc="Number of rows to generate", ge=0)]
+    label: Annotated[str, Arg("label", default="rows", doc="Label for the rows")]
+
+
+@init_single_worker
+@bind_fixed_schema
+class CountedRowsFunction(TableFunctionGenerator[CountedRowsArgs]):
+    """Table function with a required positional arg, for arg-propagation tests."""
+
+    class Meta:  # noqa: D106
+        name = "counted_rows"
+        description = "Generate N rows"
+
+    FIXED_SCHEMA: ClassVar[pa.Schema] = schema({"n": pa.int64()})
+
+    @classmethod
+    def process(  # noqa: D102
+        cls, params: ProcessParams[CountedRowsArgs], state: None, out: OutputCollector
+    ) -> None:
+        out.emit(
+            pa.RecordBatch.from_pydict(
+                {"n": list(range(params.args.count))},
+                schema=params.output_schema,
+            )
+        )
+        out.finish()
+
+
+class CountedRowsInsert(TableInOutGenerator[None, None]):
+    """Stub insert function, for the write-path arg-propagation test."""
+
+    class Meta:  # noqa: D106
+        name = "counted_rows_insert"
+
+
 @init_single_worker
 @bind_fixed_schema
 class UsersFunction(TableFunctionGenerator[EmptyArgs]):
@@ -262,6 +302,87 @@ class TestTableWithFunction:
         assert info.update_function is None
         assert info.delete_function is None
 
+    def test_inlined_scan_function_carries_declared_positional_arguments(self) -> None:
+        """``Table.arguments`` must reach the scan-time bind, not just ``bind()``.
+
+        Regression: the inlined ``ScanFunctionResult`` used to hardcode empty
+        arguments, so a backing function with a required positional ``Arg(0)``
+        saw zero positional args at scan and failed with
+        ``IndexError: Argument 0: index out of range``.
+        """
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+        )
+        info = table.to_table_info("main")
+        assert info.scan_function is not None
+        batch, _ = deserialize_record_batch(info.scan_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert sfr.function_name == "counted_rows"
+        assert [s.as_py() for s in sfr.positional_arguments] == [7]
+        assert sfr.named_arguments == {}
+
+    def test_inlined_scan_function_carries_declared_named_arguments(self) -> None:
+        """Named args declared on the Table also reach the scan-time bind."""
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(
+                positional=(pa.scalar(3),),
+                named={"label": pa.scalar("widgets")},
+            ),
+        )
+        info = table.to_table_info("main")
+        assert info.scan_function is not None
+        batch, _ = deserialize_record_batch(info.scan_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert [s.as_py() for s in sfr.positional_arguments] == [3]
+        assert {k: v.as_py() for k, v in sfr.named_arguments.items()} == {"label": "widgets"}
+
+    def test_inlined_write_functions_ignore_table_arguments(self) -> None:
+        """Only the *scan* function is parameterized by ``Table.arguments``.
+
+        Insert/update/delete functions receive their rows via the streaming table
+        input, so their inlined results stay argument-free.
+        """
+        from vgi_rpc.utils import deserialize_record_batch
+
+        table = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+            insert_function=CountedRowsInsert,
+        )
+        info = table.to_table_info("main")
+        assert info.insert_function is not None
+        batch, _ = deserialize_record_batch(info.insert_function)
+        sfr = ScanFunctionResult.deserialize(batch)
+        assert sfr.positional_arguments == []
+        assert sfr.named_arguments == {}
+
+    def test_none_positional_slot_is_rejected(self) -> None:
+        """A ``None`` positional slot would shift later args — reject it loudly.
+
+        Exercised on the ``scan_arguments_from`` helper directly: a Table carrying
+        such a slot already fails earlier, in bind()-based schema derivation. This
+        guards the serializer against ever silently dropping the slot instead.
+        """
+        from vgi.catalog.catalog_interface import scan_arguments_from
+
+        with pytest.raises(ValueError, match="positional slot 0 is None"):
+            scan_arguments_from(Arguments(positional=(None, pa.scalar(7))))
+
+    def test_scan_arguments_from_none_is_empty(self) -> None:
+        """A table declaring no arguments serializes empty args (unchanged behavior)."""
+        from vgi.catalog.catalog_interface import scan_arguments_from
+
+        assert scan_arguments_from(None) == ([], {})
+
     def test_table_propagates_cardinality_to_table_info(self) -> None:
         """``Table.cardinality_estimate`` / ``cardinality_max`` flow into TableInfo.
 
@@ -1121,8 +1242,6 @@ class TestSchemaDescriptor:
         client hits a cold miss. Reporting an explicit ``0`` for empty
         categories lets the client skip the bulk RPC entirely.
         """
-        from typing import Annotated
-
         from vgi.arguments import Param, Returns
         from vgi.scalar_function import ScalarFunction
 
@@ -1174,8 +1293,6 @@ class TestLegacyFunctionsListSchemaInfo:
     """
 
     def _interface(self) -> ReadOnlyCatalogInterface:
-        from typing import Annotated
-
         from vgi.arguments import Param, Returns
         from vgi.scalar_function import ScalarFunction
 
@@ -1694,6 +1811,70 @@ class TestTableScanFunctionGet:
         )
         assert isinstance(result, ScanFunctionResult)
         assert result.function_name == "users"
+        # No declared Table.arguments => empty args, unchanged behavior.
+        assert result.positional_arguments == []
+        assert result.named_arguments == {}
+
+    def test_auto_scan_forwards_declared_arguments(self) -> None:
+        """The RPC auto-impl forwards ``Table.arguments`` to the scan bind.
+
+        Companion to the inlined-path regression: workers that don't inline
+        ``TableInfo.scan_function`` reach the scan through this RPC instead.
+        """
+        counted = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(
+                positional=(pa.scalar(7),),
+                named={"label": pa.scalar("widgets")},
+            ),
+        )
+
+        class TestCatalog(ReadOnlyCatalogInterface):
+            catalog = Catalog(
+                name="test",
+                schemas=[Schema(name="main", tables=[counted])],
+            )
+
+        result = TestCatalog().table_scan_function_get(
+            attach_opaque_data=AttachOpaqueData(b"test"),
+            transaction_opaque_data=None,
+            schema_name="main",
+            name="counted",
+            at_unit=None,
+            at_value=None,
+        )
+        assert result.function_name == "counted_rows"
+        assert [s.as_py() for s in result.positional_arguments] == [7]
+        assert {k: v.as_py() for k, v in result.named_arguments.items()} == {"label": "widgets"}
+
+    def test_scan_branches_default_inherits_declared_arguments(self) -> None:
+        """The default ``table_scan_branches_get`` delegates to the legacy method.
+
+        So the one-branch synthesis must carry the declared arguments too.
+        """
+        counted = Table(
+            name="counted",
+            function=CountedRowsFunction,
+            arguments=Arguments(positional=(pa.scalar(7),)),
+        )
+
+        class TestCatalog(ReadOnlyCatalogInterface):
+            catalog = Catalog(
+                name="test",
+                schemas=[Schema(name="main", tables=[counted])],
+            )
+
+        result = TestCatalog().table_scan_branches_get(
+            attach_opaque_data=AttachOpaqueData(b"test"),
+            transaction_opaque_data=None,
+            schema_name="main",
+            name="counted",
+            at_unit=None,
+            at_value=None,
+        )
+        assert len(result.branches) == 1
+        assert [s.as_py() for s in result.branches[0].positional_arguments] == [7]
 
     def test_explicit_columns_table_raises(self) -> None:
         """Tables with explicit columns raise NotImplementedError."""

vgi_python-0.15.0/tests/catalog/test_required_filters.py

@@ -0,0 +1,92 @@
+"""Unit tests for the ``Table.required_filters`` field.
+
+The field is purely declarative on the Python side — enforcement runs in the C++
+optimizer extension. It is conjunctive normal form: an AND (outer tuple) of
+OR-groups (inner tuples) of dotted-path column references. These tests cover:
+
+1. Default-empty behaviour (no field set → ``TableInfo`` ships an empty list).
+2. Populated round-trip (singleton groups + a multi-path OR-group preserved).
+3. Validation: each path's leading dotted segment must be a real column.
+4. Empty-string paths and empty OR-groups are rejected loudly.
+"""
+
+from __future__ import annotations
+
+import pyarrow as pa
+import pytest
+
+from vgi.catalog.descriptors import Table
+
+
+class TestRequiredFiltersField:
+    """Behaviour of the declarative required_filters field."""
+
+    def _bbox_columns(self) -> pa.Schema:
+        """Return a small schema with a ``bbox`` STRUCT plus flat id/ticker."""
+        return pa.schema(
+            [
+                ("id", pa.int64()),
+                ("ticker", pa.string()),
+                (
+                    "bbox",
+                    pa.struct(
+                        [
+                            ("xmin", pa.float64()),
+                            ("ymin", pa.float64()),
+                            ("xmax", pa.float64()),
+                            ("ymax", pa.float64()),
+                        ]
+                    ),
+                ),
+            ]
+        )
+
+    def test_default_is_empty_tuple(self) -> None:
+        """Default is an empty tuple; TableInfo wire field is an empty list."""
+        t = Table(name="t", columns=self._bbox_columns())
+        assert t.required_filters == ()
+        info = t.to_table_info("main")
+        assert info.required_filters == []
+
+    def test_singleton_groups_round_trip(self) -> None:
+        """Singleton groups (the AND-only case) survive as a list of lists."""
+        groups = (("bbox.xmin",), ("bbox.xmax",), ("bbox.ymin",), ("bbox.ymax",))
+        t = Table(name="place", columns=self._bbox_columns(), required_filters=groups)
+        assert t.required_filters == groups
+        info = t.to_table_info("main")
+        assert info.required_filters == [list(g) for g in groups]
+
+    def test_or_group_round_trip(self) -> None:
+        """A multi-path OR-group survives alongside a singleton mandatory group."""
+        groups = (("id",), ("id", "ticker"))
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=groups)
+        info = t.to_table_info("main")
+        assert info.required_filters == [["id"], ["id", "ticker"]]
+
+    def test_top_level_path(self) -> None:
+        """A top-level column name (no dots) is a valid path."""
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("id",),))
+        assert t.to_table_info("main").required_filters == [["id"]]
+
+    def test_nested_path_is_not_unpacked_for_validation(self) -> None:
+        """Only the leading dotted segment is validated; deeper segments pass through."""
+        # Subfield validity isn't checked here (descriptor doesn't unpack struct
+        # schemas) — only the leading segment matters. Typos like ``bbox.nope``
+        # go through and DuckDB catches them at scan time.
+        t = Table(name="t", columns=self._bbox_columns(), required_filters=(("bbox.nope",),))
+        assert t.required_filters == (("bbox.nope",),)
+
+    def test_unknown_leading_segment_raises(self) -> None:
+        """A path whose leading segment is not a real column is rejected."""
+        with pytest.raises(ValueError, match=r"unknown column 'nope'"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("nope.xmin",),))
+
+    def test_empty_string_path_raises(self) -> None:
+        """An empty string path is rejected loudly rather than silently accepted."""
+        with pytest.raises(ValueError, match="must not contain empty strings"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=(("",),))
+
+    def test_empty_group_raises(self) -> None:
+        """An empty OR-group is rejected loudly."""
+        with pytest.raises(ValueError, match="must not contain empty groups"):
+            Table(name="t", columns=self._bbox_columns(), required_filters=((),))

{vgi_python-0.13.0 → vgi_python-0.15.0}/tests/catalog/test_serialization.py

@@ -1149,10 +1149,10 @@ class TestArrowSchemaCorrectness:
         assert schema.field("cardinality_estimate").type == pa.int64()
         assert schema.field("column_statistics").type == pa.binary()
         assert schema.field("bind_result").type == pa.binary()
-        assert schema.field("required_field_filter_paths").type == pa.list_(pa.string())
-        # required_field_filter_paths is the LAST field (positional schema
-        # compatibility: old C++ extensions ignore-by-position trailing fields).
-        assert schema.field(len(schema) - 1).name == "required_field_filter_paths"
+        assert schema.field("required_filters").type == pa.list_(pa.list_(pa.string()))
+        # required_filters is the LAST field (positional schema compatibility:
+        # old C++ extensions ignore-by-position trailing fields).
+        assert schema.field(len(schema) - 1).name == "required_filters"
 
     def test_view_info_schema(self) -> None:
         """Verify ViewInfo Arrow schema."""

{vgi_python-0.13.0 → vgi_python-0.15.0}/tests/conformance/test_directory_parity.py

@@ -33,6 +33,15 @@ _EXEMPTIONS: dict[str, str] = {
         "lifecycle/schema/table surfaces are mirrored by test_attach.py, "
         "test_table.py, test_view.py, and test_macro.py."
     ),
+    "cache": (
+        "The table-function result cache lives entirely in the C++ extension "
+        "(key/eligibility, in-memory + content-addressed disk tiers, serve/"
+        "capture, conditional revalidation). The Python worker's only role is "
+        "advertising vgi.cache.* metadata + answering 304 not_modified, which is "
+        "unit-tested by tests/test_cache_control.py (rendering, the emit merge "
+        "path, ProcessParams.if_none_match threading, and the revalidatable "
+        "fixture's 304 behavior)."
+    ),
     "filter_pushdown": (
         "Per-type filter-pushdown coverage is mirrored by tests/test_filter_pushdown.py "
         "and tests/test_filter_pushdown_extension.py, which drive the same predicate "