vgi-python 0.42.0__tar.gz → 0.42.1__tar.gz

{vgi_python-0.42.0 → vgi_python-0.42.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.42.0
+Version: 0.42.1
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.42.0 → vgi_python-0.42.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.42.0"
+version = "0.42.1"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.42.0 → vgi_python-0.42.1}/tests/test_bind_request_at_clause.py

@@ -13,10 +13,14 @@ Python protocol contract (round-trip + additive wire compatibility + accessors).
 
 from __future__ import annotations
 
+from types import SimpleNamespace
+
 import pyarrow as pa
 import pyarrow.ipc as ipc
+import pytest
 
 from vgi.arguments import Arguments
+from vgi.client.client import Client
 from vgi.invocation import FunctionType
 from vgi.protocol import BindRequest, InitRequest
 from vgi.table_function import BindParams, ProcessParams, ResolvedSecrets
@@ -108,3 +112,31 @@ def test_process_params_accessor_none_when_no_init_call() -> None:
     )
     assert params.at_unit is None
     assert params.at_value is None
+
+
+def test_client_bind_forwards_at_clause_and_secrets(monkeypatch: pytest.MonkeyPatch) -> None:
+    """``Client.bind`` carries at_unit/at_value and secrets onto its BindRequest, as table_function does."""
+    captured: list[BindRequest] = []
+    client = Client("unused-worker")
+    client._primary = SimpleNamespace(proxy=None)  # type: ignore[assignment]
+
+    def fake_do_bind(_proxy: object, request: BindRequest, _callback: object) -> object:
+        captured.append(request)
+        return SimpleNamespace()
+
+    monkeypatch.setattr(client, "_do_bind", fake_do_bind)
+    client.bind(
+        function_name="sequence",
+        schema_path=["main"],
+        at_unit="VERSION",
+        at_value="3",
+        secrets={"token": "s3cr3t"},
+    )
+    (request,) = captured
+    assert (request.at_unit, request.at_value) == ("VERSION", "3")
+    assert request.secrets is not None
+
+    captured.clear()
+    client.bind(function_name="sequence", schema_path=["main"])
+    (request,) = captured
+    assert (request.at_unit, request.at_value, request.secrets) == (None, None, None)

{vgi_python-0.42.0 → vgi_python-0.42.1}/tests/test_filter_v2.py

@@ -22,6 +22,7 @@ from vgi import (
     FilterVersionError,
     RuntimeFilterAlgorithmCapability,
     deserialize_filters,
+    filter_v2,
 )
 
 _METADATA: dict[bytes, bytes] = {
@@ -767,3 +768,36 @@ def test_standard_v1_evaluator_requires_oracle_engine(
         with pytest.raises(filter_v2.FilterV2Error, match=f"found {engine} {re.escape(version)}"):
             filter_v2._get_evaluation_connection(context)
         assert opened == []
+
+
+class _NoNbytes:
+    """Stands in for a view-typed batch under pyarrow < 19, where ``nbytes`` raises."""
+
+    def __init__(self, total: int | None) -> None:
+        self._total = total
+
+    @property
+    def nbytes(self) -> int:
+        raise pa.ArrowTypeError("not implemented for string_view")
+
+    def get_total_buffer_size(self) -> int:
+        if self._total is None:
+            raise pa.ArrowNotImplementedError("not implemented")
+        return self._total
+
+
+def test_payload_size_falls_back_when_nbytes_is_unavailable() -> None:
+    """A view-typed payload is measured by total buffer size instead of raising ArrowTypeError."""
+    assert filter_v2._payload_nbytes(_NoNbytes(123)) == 123  # type: ignore[arg-type]
+
+
+def test_payload_size_unmeasurable_is_a_filter_error() -> None:
+    """When no measurement works, the caller sees FilterV2Error, not a raw Arrow error."""
+    with pytest.raises(filter_v2.FilterV2Error, match="cannot measure filter payload size"):
+        filter_v2._payload_nbytes(_NoNbytes(None))  # type: ignore[arg-type]
+
+
+def test_payload_size_of_a_view_typed_batch() -> None:
+    """A real string_view payload is measured without error."""
+    batch = pa.record_batch([pa.array(["a", "bc"], type=pa.string_view())], names=["needles"])
+    assert filter_v2._payload_nbytes(batch) > 0

vgi_python-0.42.1/tests/test_table_function_early_close.py

@@ -0,0 +1,81 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Closing a ``table_function`` generator before its stream ends must not hang.
+
+``Client.table_function`` reads each worker's producer stream on a background
+thread. Before the fix, closing the generator early left those threads reading,
+and the ``Client.stop()`` that followed drained the same stream from the main
+thread: two readers on one pipe, and whichever missed the end of stream blocked
+forever (roughly one run in five). Each case here repeats the early close under
+a watchdog, then checks the pooled worker still serves a complete scan.
+"""
+
+from __future__ import annotations
+
+import threading
+from collections.abc import Callable
+
+import pyarrow as pa
+import pytest
+
+from tests.conftest import SUBPROCESS_FIXTURE_WORKER
+from vgi.arguments import Arguments
+from vgi.client.client import Client, _default_pool
+
+_ITERATIONS = 25
+_WATCHDOG_SECONDS = 120.0
+
+
+def _run_with_watchdog(body: Callable[[], None]) -> None:
+    """Run ``body`` on a daemon thread; fail (rather than hang the suite) if it doesn't finish."""
+    errors: list[BaseException] = []
+
+    def target() -> None:
+        try:
+            body()
+        except BaseException as e:  # noqa: BLE001 - re-raised on the test thread below
+            errors.append(e)
+
+    thread = threading.Thread(target=target, daemon=True)
+    thread.start()
+    thread.join(_WATCHDOG_SECONDS)
+    assert not thread.is_alive(), "closing a table_function scan early hung"
+    if errors:
+        raise errors[0]
+
+
+def _close_after_first_batch(function_name: str, arguments: Arguments) -> None:
+    for _ in range(_ITERATIONS):
+        with Client(SUBPROCESS_FIXTURE_WORKER, pool=_default_pool) as client:
+            scan = client.table_function(function_name=function_name, schema_path=["main"], arguments=arguments)
+            first = next(scan)
+            assert first.num_rows > 0
+            scan.close()
+
+
+def _full_scan_still_works() -> None:
+    with Client(SUBPROCESS_FIXTURE_WORKER, pool=_default_pool) as client:
+        batches = list(
+            client.table_function(
+                function_name="sequence",
+                schema_path=["main"],
+                arguments=Arguments(positional=(pa.scalar(4),)),
+            )
+        )
+    assert pa.Table.from_batches(batches).column("n").to_pylist() == [0, 1, 2, 3]
+
+
+@pytest.mark.parametrize(
+    ("function_name", "arguments"),
+    [
+        # One worker, many small batches.
+        ("sequence", Arguments(positional=(pa.scalar(100_000),), named={"batch_size": pa.scalar(10)})),
+        # Several workers (one per partition), each with batches still to come.
+        ("partitioned_sequence", Arguments(positional=(pa.scalar(30_000),))),
+    ],
+    ids=["single_worker", "multi_worker"],
+)
+def test_early_close_does_not_hang(function_name: str, arguments: Arguments) -> None:
+    """A scan closed after its first batch stops cleanly, and the worker pool stays usable."""
+    _run_with_watchdog(lambda: _close_after_first_batch(function_name, arguments))
+    _run_with_watchdog(_full_scan_still_works)

{vgi_python-0.42.0 → vgi_python-0.42.1}/uv.lock

@@ -2670,7 +2670,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.42.0"
+version = "0.42.1"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

{vgi_python-0.42.0 → vgi_python-0.42.1}/vgi/client/client.py

@@ -2757,7 +2757,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         arguments: Arguments | None = None,
         function_type: FunctionType = FunctionType.TABLE,
         settings: dict[str, Any] | None = None,
+        secrets: dict[str, Any] | None = None,
         transaction_opaque_data: bytes | None = None,
+        at_unit: str | None = None,
+        at_value: str | None = None,
     ) -> BindResponse:
         """Resolve a function's bind response without running init()/process().
 
@@ -2789,8 +2792,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             settings: Optional dictionary of settings/pragmas — some
                 functions' output schema depends on setting values (see
                 `Meta.required_settings`).
+            secrets: Optional dictionary of secret name to value pairs, as for
+                `table_function` — a function whose bind resolves secrets
+                (e.g. a secret-backed source) needs them here too.
             transaction_opaque_data: Optional unique identifier for the
                 DuckDB transaction.
+            at_unit: Optional time travel unit (e.g. 'timestamp', 'version'):
+                bind the function as of a past point, as `table_function`'s
+                `at_unit` does, so a versioned table's historical schema can be
+                discovered. `None` binds live.
+            at_value: Optional time travel value, paired with `at_unit`.
 
         Returns:
             [`BindResponse`][] with `output_schema` and any opaque bind data —
@@ -2814,8 +2825,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                 arguments=arguments,
                 function_type=function_type,
                 settings=settings,
-                secrets=None,
+                secrets=secrets,
                 transaction_opaque_data=transaction_opaque_data,
+                at_unit=at_unit,
+                at_value=at_value,
             )
             return self._do_bind(self._primary.proxy, bind_request, None)
         except ClientError as e:
@@ -3273,15 +3286,23 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         # AnnotatedBatch (not just .batch) so custom_metadata survives the
         # cross-thread handoff for batch_metadata_callback below.
         output_queue: Queue[AnnotatedBatch | BaseException | None] = Queue()
+        # Set when the consumer stops before every worker finished (the
+        # generator was closed early, or a worker failed). Each reader thread is
+        # its stream's only reader, so it is also the one that ends the stream:
+        # a producer stream advances one tick per batch, so a reader notices the
+        # flag within one batch and cancels from its own thread. Ending it from
+        # the consumer's thread instead would put two readers on one pipe.
+        stop = threading.Event()
 
         def read_worker_output(worker: WorkerConnection) -> None:
             """Thread function that reads all output from a single worker."""
             try:
-                if worker.stream is None:
+                stream = worker.stream
+                if stream is None:
                     output_queue.put(None)
                     return
 
-                for output in worker.stream:
+                for output in stream:
                     _logger.debug(
                         "received_output_from_worker worker_index=%s num_rows=%s",
                         worker.worker_index,
@@ -3289,6 +3310,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                     )
                     if output.batch.num_rows > 0:
                         output_queue.put(output)
+                    if stop.is_set():
+                        _logger.debug("table_function_worker_cancelled worker_index=%s", worker.worker_index)
+                        stream.cancel()
+                        break
 
                 output_queue.put(None)  # Signal completion
             except StopIteration:
@@ -3308,41 +3333,59 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             thread.start()
             threads.append(thread)
 
-        # Collect outputs from all workers until all are done
-        workers_finished = 0
-        while workers_finished < num_workers:
-            result = output_queue.get()
-
-            # Check for exceptions from worker threads
-            if isinstance(result, BaseException):
-                if isinstance(result, RpcError):
-                    raise ClientError.from_rpc_error(result) from result
-                raise ClientError(f"Worker thread failed: {result}") from result
-
-            # None signals a worker finished
-            if result is None:
-                workers_finished += 1
-                _logger.debug(
-                    "worker_finished workers_finished=%s total_workers=%s",
-                    workers_finished,
-                    num_workers,
-                )
-                continue
-
-            if batch_metadata_callback is not None:
-                batch_metadata_callback(result.custom_metadata)
-            yield result.batch
-
-        self._join_threads(threads)
-        _logger.debug("all_table_function_workers_complete")
+        # Collect outputs from all workers until all are done. The finally
+        # block also runs when the consumer closes the generator early
+        # (GeneratorExit at the yield) or a worker fails: the reader threads are
+        # told to stop and joined before any stream is touched here.
+        completed = False
+        try:
+            workers_finished = 0
+            while workers_finished < num_workers:
+                result = output_queue.get()
+
+                # Check for exceptions from worker threads
+                if isinstance(result, BaseException):
+                    if isinstance(result, RpcError):
+                        raise ClientError.from_rpc_error(result) from result
+                    raise ClientError(f"Worker thread failed: {result}") from result
+
+                # None signals a worker finished
+                if result is None:
+                    workers_finished += 1
+                    _logger.debug(
+                        "worker_finished workers_finished=%s total_workers=%s",
+                        workers_finished,
+                        num_workers,
+                    )
+                    continue
 
-        # Close streams and secondary workers
-        for worker in all_workers:
-            if worker.stream is not None:
-                worker.stream.close()
-                worker.stream = None
-        self._close_secondary_workers()
-        _logger.debug("parallel_table_function_complete")
+                if batch_metadata_callback is not None:
+                    batch_metadata_callback(result.custom_metadata)
+                yield result.batch
+            completed = True
+        finally:
+            if not completed:
+                stop.set()
+            self._join_threads(threads)
+            _logger.debug("all_table_function_workers_complete completed=%s", completed)
+
+            # Close streams and secondary workers. A stream a reader cancelled is
+            # already closed, so close() is a no-op for it.
+            for worker, thread in zip(all_workers, threads, strict=True):
+                if thread.is_alive():
+                    # Still reading after the join timeout: the stream's state is
+                    # unknown and its reader still owns it. Kill the worker so
+                    # neither that reader nor a later stop() blocks on the pipe.
+                    _logger.warning("table_function_worker_killed worker_index=%s", worker.worker_index)
+                    if worker.proc is not None:
+                        worker.proc.kill()
+                    worker.stream = None
+                    continue
+                if worker.stream is not None:
+                    worker.stream.close()
+                    worker.stream = None
+            self._close_secondary_workers()
+            _logger.debug("parallel_table_function_complete")
 
     # ==================================================================
     # Custom COPY formats

{vgi_python-0.42.0 → vgi_python-0.42.1}/vgi/filter_v2.py

@@ -117,6 +117,34 @@ class FilterV2Error(ValueError):
     """A Filter Encoding v2 document is malformed or cannot be evaluated."""
 
 
+def _payload_nbytes(batch: pa.RecordBatch) -> int:
+    """Size of a filter payload batch, for the payload limit.
+
+    ``RecordBatch.nbytes`` is not implemented for view types (``string_view``,
+    ``binary_view``, ``list_view``) before pyarrow 19 and raises a raw
+    ``ArrowTypeError``. Fall back to the total buffer size there, which walks
+    every buffer generically and can only overcount (shared buffers), never
+    undercount, so the limit still holds.
+
+    Args:
+        batch: The filter payload batch.
+
+    Returns:
+        Its size in bytes, for comparison with ``MAX_PAYLOAD_BYTES``.
+
+    Raises:
+        FilterV2Error: If neither measurement is available for the batch.
+    """
+    try:
+        return int(batch.nbytes)
+    except (pa.ArrowTypeError, pa.ArrowNotImplementedError):
+        pass
+    try:
+        return int(batch.get_total_buffer_size())
+    except (pa.ArrowTypeError, pa.ArrowNotImplementedError) as exc:
+        raise FilterV2Error(f"cannot measure filter payload size: {exc}") from exc
+
+
 class PredicateMode(StrEnum):
     """Whether a predicate must be applied exactly or is only a pruning hint."""
 
@@ -592,7 +620,7 @@ class _Parser:
                 raise FilterV2Error(f"noncanonical payload field name {name!r}")
             if name.startswith("type_") and self.batch.column(i)[0].is_valid:
                 raise FilterV2Error(f"{name} must contain a NULL value")
-        if self.batch.nbytes > MAX_PAYLOAD_BYTES:
+        if _payload_nbytes(self.batch) > MAX_PAYLOAD_BYTES:
             raise FilterV2Error("filter payload exceeds 16 MiB")
         return parse_evaluation_context(self.batch.schema)