vgi-python 0.42.0__py3-none-any.whl → 0.42.1__py3-none-any.whl

vgi/client/client.py

@@ -2757,7 +2757,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         arguments: Arguments | None = None,
         function_type: FunctionType = FunctionType.TABLE,
         settings: dict[str, Any] | None = None,
+        secrets: dict[str, Any] | None = None,
         transaction_opaque_data: bytes | None = None,
+        at_unit: str | None = None,
+        at_value: str | None = None,
     ) -> BindResponse:
         """Resolve a function's bind response without running init()/process().
 
@@ -2789,8 +2792,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             settings: Optional dictionary of settings/pragmas — some
                 functions' output schema depends on setting values (see
                 `Meta.required_settings`).
+            secrets: Optional dictionary of secret name to value pairs, as for
+                `table_function` — a function whose bind resolves secrets
+                (e.g. a secret-backed source) needs them here too.
             transaction_opaque_data: Optional unique identifier for the
                 DuckDB transaction.
+            at_unit: Optional time travel unit (e.g. 'timestamp', 'version'):
+                bind the function as of a past point, as `table_function`'s
+                `at_unit` does, so a versioned table's historical schema can be
+                discovered. `None` binds live.
+            at_value: Optional time travel value, paired with `at_unit`.
 
         Returns:
             [`BindResponse`][] with `output_schema` and any opaque bind data —
@@ -2814,8 +2825,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                 arguments=arguments,
                 function_type=function_type,
                 settings=settings,
-                secrets=None,
+                secrets=secrets,
                 transaction_opaque_data=transaction_opaque_data,
+                at_unit=at_unit,
+                at_value=at_value,
             )
             return self._do_bind(self._primary.proxy, bind_request, None)
         except ClientError as e:
@@ -3273,15 +3286,23 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         # AnnotatedBatch (not just .batch) so custom_metadata survives the
         # cross-thread handoff for batch_metadata_callback below.
         output_queue: Queue[AnnotatedBatch | BaseException | None] = Queue()
+        # Set when the consumer stops before every worker finished (the
+        # generator was closed early, or a worker failed). Each reader thread is
+        # its stream's only reader, so it is also the one that ends the stream:
+        # a producer stream advances one tick per batch, so a reader notices the
+        # flag within one batch and cancels from its own thread. Ending it from
+        # the consumer's thread instead would put two readers on one pipe.
+        stop = threading.Event()
 
         def read_worker_output(worker: WorkerConnection) -> None:
             """Thread function that reads all output from a single worker."""
             try:
-                if worker.stream is None:
+                stream = worker.stream
+                if stream is None:
                     output_queue.put(None)
                     return
 
-                for output in worker.stream:
+                for output in stream:
                     _logger.debug(
                         "received_output_from_worker worker_index=%s num_rows=%s",
                         worker.worker_index,
@@ -3289,6 +3310,10 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                     )
                     if output.batch.num_rows > 0:
                         output_queue.put(output)
+                    if stop.is_set():
+                        _logger.debug("table_function_worker_cancelled worker_index=%s", worker.worker_index)
+                        stream.cancel()
+                        break
 
                 output_queue.put(None)  # Signal completion
             except StopIteration:
@@ -3308,41 +3333,59 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             thread.start()
             threads.append(thread)
 
-        # Collect outputs from all workers until all are done
-        workers_finished = 0
-        while workers_finished < num_workers:
-            result = output_queue.get()
-
-            # Check for exceptions from worker threads
-            if isinstance(result, BaseException):
-                if isinstance(result, RpcError):
-                    raise ClientError.from_rpc_error(result) from result
-                raise ClientError(f"Worker thread failed: {result}") from result
-
-            # None signals a worker finished
-            if result is None:
-                workers_finished += 1
-                _logger.debug(
-                    "worker_finished workers_finished=%s total_workers=%s",
-                    workers_finished,
-                    num_workers,
-                )
-                continue
-
-            if batch_metadata_callback is not None:
-                batch_metadata_callback(result.custom_metadata)
-            yield result.batch
-
-        self._join_threads(threads)
-        _logger.debug("all_table_function_workers_complete")
+        # Collect outputs from all workers until all are done. The finally
+        # block also runs when the consumer closes the generator early
+        # (GeneratorExit at the yield) or a worker fails: the reader threads are
+        # told to stop and joined before any stream is touched here.
+        completed = False
+        try:
+            workers_finished = 0
+            while workers_finished < num_workers:
+                result = output_queue.get()
+
+                # Check for exceptions from worker threads
+                if isinstance(result, BaseException):
+                    if isinstance(result, RpcError):
+                        raise ClientError.from_rpc_error(result) from result
+                    raise ClientError(f"Worker thread failed: {result}") from result
+
+                # None signals a worker finished
+                if result is None:
+                    workers_finished += 1
+                    _logger.debug(
+                        "worker_finished workers_finished=%s total_workers=%s",
+                        workers_finished,
+                        num_workers,
+                    )
+                    continue
 
-        # Close streams and secondary workers
-        for worker in all_workers:
-            if worker.stream is not None:
-                worker.stream.close()
-                worker.stream = None
-        self._close_secondary_workers()
-        _logger.debug("parallel_table_function_complete")
+                if batch_metadata_callback is not None:
+                    batch_metadata_callback(result.custom_metadata)
+                yield result.batch
+            completed = True
+        finally:
+            if not completed:
+                stop.set()
+            self._join_threads(threads)
+            _logger.debug("all_table_function_workers_complete completed=%s", completed)
+
+            # Close streams and secondary workers. A stream a reader cancelled is
+            # already closed, so close() is a no-op for it.
+            for worker, thread in zip(all_workers, threads, strict=True):
+                if thread.is_alive():
+                    # Still reading after the join timeout: the stream's state is
+                    # unknown and its reader still owns it. Kill the worker so
+                    # neither that reader nor a later stop() blocks on the pipe.
+                    _logger.warning("table_function_worker_killed worker_index=%s", worker.worker_index)
+                    if worker.proc is not None:
+                        worker.proc.kill()
+                    worker.stream = None
+                    continue
+                if worker.stream is not None:
+                    worker.stream.close()
+                    worker.stream = None
+            self._close_secondary_workers()
+            _logger.debug("parallel_table_function_complete")
 
     # ==================================================================
     # Custom COPY formats

vgi/filter_v2.py

@@ -117,6 +117,34 @@ class FilterV2Error(ValueError):
     """A Filter Encoding v2 document is malformed or cannot be evaluated."""
 
 
+def _payload_nbytes(batch: pa.RecordBatch) -> int:
+    """Size of a filter payload batch, for the payload limit.
+
+    ``RecordBatch.nbytes`` is not implemented for view types (``string_view``,
+    ``binary_view``, ``list_view``) before pyarrow 19 and raises a raw
+    ``ArrowTypeError``. Fall back to the total buffer size there, which walks
+    every buffer generically and can only overcount (shared buffers), never
+    undercount, so the limit still holds.
+
+    Args:
+        batch: The filter payload batch.
+
+    Returns:
+        Its size in bytes, for comparison with ``MAX_PAYLOAD_BYTES``.
+
+    Raises:
+        FilterV2Error: If neither measurement is available for the batch.
+    """
+    try:
+        return int(batch.nbytes)
+    except (pa.ArrowTypeError, pa.ArrowNotImplementedError):
+        pass
+    try:
+        return int(batch.get_total_buffer_size())
+    except (pa.ArrowTypeError, pa.ArrowNotImplementedError) as exc:
+        raise FilterV2Error(f"cannot measure filter payload size: {exc}") from exc
+
+
 class PredicateMode(StrEnum):
     """Whether a predicate must be applied exactly or is only a pruning hint."""
 
@@ -592,7 +620,7 @@ class _Parser:
                 raise FilterV2Error(f"noncanonical payload field name {name!r}")
             if name.startswith("type_") and self.batch.column(i)[0].is_valid:
                 raise FilterV2Error(f"{name} must contain a NULL value")
-        if self.batch.nbytes > MAX_PAYLOAD_BYTES:
+        if _payload_nbytes(self.batch) > MAX_PAYLOAD_BYTES:
             raise FilterV2Error("filter payload exceeds 16 MiB")
         return parse_evaluation_context(self.batch.schema)
 

{vgi_python-0.42.0.dist-info → vgi_python-0.42.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.42.0
+Version: 0.42.1
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.42.0.dist-info → vgi_python-0.42.1.dist-info}/RECORD

@@ -10,7 +10,7 @@ vgi/cache_control.py,sha256=IyBp9gZTCc4Y-4V7Rs05_o5lSEjbR2jt6cddXPU0dKw,7812
 vgi/copy_from_function.py,sha256=iSkR48b3W9IVQ-HY2I8-5W1QjTtWF7k1GkogHNE7ZMM,7114
 vgi/copy_to_function.py,sha256=8l0v_DyEF1EnFwinStrZDQ-yNFfGJsv3Mr5o2jG30f0,9279
 vgi/exceptions.py,sha256=oX_sZc9xGWi7Xf8cJQf89fX19i3ocDEj_V_76GINgBQ,7294
-vgi/filter_v2.py,sha256=RnuZAPTlAOhUYiY-D3lYlaMvr8OdaR3MhdjVufI-0Dw,57860
+vgi/filter_v2.py,sha256=ezQ1rDweMJJkow-5mQPG5BYVnN93426ObYPD2KzUNSs,58889
 vgi/filter_v2_builder.py,sha256=8OmyzDI6GDdH4pYm2sJFMidzKusi0I8IGNipolPpjX0,5535
 vgi/function.py,sha256=KIfvKk1nzBvRDPXViHNX89tv6gvIMiDj4e8fl6ANqQg,11626
 vgi/function_storage.py,sha256=FjVwxc1BIjWo-82j8YE5qcU8w66sGjwyGsTmCUhn-Kw,67576
@@ -143,7 +143,7 @@ vgi/client/cli_table.py,sha256=H1xrKZF5yYmSe55YSrLWXGIXPmuGuzQcncEBvqL43zs,26498
 vgi/client/cli_transaction.py,sha256=TDW0ZrBy8XM_eYSkYk5ewEsnozfzDJOlopd66Fm7OOE,3142
 vgi/client/cli_utils.py,sha256=ykh7XK-yzcW6WGED2_DhPLIVPSyVG3KUdllV5ztBicw,17458
 vgi/client/cli_view.py,sha256=0qhxoS91-zFMjq8V_Fb0PNW-YDxERu8-zPpxSFEE-G8,8490
-vgi/client/client.py,sha256=GSkcVXQzI0VCmeUf7WY4kR0ZqFtYmeb6EkenarBvX5c,167399
+vgi/client/client.py,sha256=OtrpTHVeM4nbXzJkk77ab_WC5hONuMLoYAD4MlLD5qM,170099
 vgi/client/errors.py,sha256=cSN65cyQMk3crwZCz1IdPErK3mtlEkTNpu-nuQUDOQE,1633
 vgi/http/__init__.py,sha256=hlOwOVcoZqmEKVGk7ganOdr5ryRSpEhLBF9sRD7BkYc,608
 vgi/http/demo_storage.py,sha256=830s-H61thozC3eEuqdnwCpYFfvoJRq9vjyuXa5OURo,8769
@@ -156,8 +156,8 @@ vgi/transactor/_duckdb_compat.py,sha256=sXVZ9JLKAQyGR1BjWczSwdQEavtr-TcZPoVZZnTr
 vgi/transactor/client.py,sha256=7DTeMksogsw6ANjQjGOPpKYrV76rg4_kGjktMJf54jg,4486
 vgi/transactor/protocol.py,sha256=v59IvrKnuvwOXvn_HEcBAbCzDVcx0akgKN0R1mChXg0,5034
 vgi/transactor/server.py,sha256=nzsZQxJZdgappdYX8okrFIrjFtUbdM_Hhskl1fZ2nDY,32553
-vgi_python-0.42.0.dist-info/METADATA,sha256=1c3-T4txCgcbPeUyxS-fOuMogZtSRPvbjNDOeau4pSg,25953
-vgi_python-0.42.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-vgi_python-0.42.0.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
-vgi_python-0.42.0.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
-vgi_python-0.42.0.dist-info/RECORD,,
+vgi_python-0.42.1.dist-info/METADATA,sha256=8JvivAw5wSBb73Xpal2NdQIafuA4enfCQnEQAqs5RL4,25953
+vgi_python-0.42.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+vgi_python-0.42.1.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
+vgi_python-0.42.1.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
+vgi_python-0.42.1.dist-info/RECORD,,