vgi-python 0.35.0__py3-none-any.whl → 0.36.0__py3-none-any.whl

vgi/_test_fixtures/table/filters.py

@@ -1003,7 +1003,10 @@ class FilterEchoPartitionedFunction(TableFunctionGenerator[_FilterEchoPartitione
             end_idx = min(start_idx + chunk, params.args.count)
             work_items.append(struct.pack(">QQ", start_idx, end_idx))
         params.storage.queue_push(work_items)
-        return GlobalInitResponse()
+        # No more readers than work items. Left at the default (unbounded) the
+        # client opens one stream per DuckDB thread -- 48 on a 48-core host for at
+        # most MAX_PARTITIONS items, every extra one an init plus an empty drain.
+        return GlobalInitResponse(max_workers=max(1, len(work_items)))
 
     @classmethod
     def initial_state(cls, params: ProcessParams[_FilterEchoPartitionedArgs]) -> _FilterEchoPartitionedState:

vgi/_test_fixtures/table/sequence.py

@@ -516,7 +516,10 @@ class PartitionedSequenceFunction(
 
         # Always enqueue (even if empty) to register the invocation
         params.storage.queue_push(work_items)
-        return GlobalInitResponse()
+        # No more readers than work items. Left at the default (unbounded) the
+        # client opens one stream per DuckDB thread -- 48 on a 48-core host for at
+        # most MAX_PARTITIONS items, every extra one an init plus an empty drain.
+        return GlobalInitResponse(max_workers=max(1, len(work_items)))
 
     @classmethod
     def initial_state(cls, params: ProcessParams[PartitionedSequenceFunctionArguments]) -> PartitionedSequenceState:

vgi/_test_fixtures/table/splits.py

@@ -35,6 +35,7 @@ from vgi_rpc.rpc import OutputCollector
 
 from vgi.arguments import Arg
 from vgi.cache_control import CacheControl
+from vgi.filter_v2 import replaying_accepted_state
 from vgi.metadata import FunctionExample, PartitionKind
 from vgi.protocol import PlanResponse, ScanSplit, VgiOutputCollector
 from vgi.schema_utils import partition_field, schema
@@ -1055,11 +1056,15 @@ class SplitDynamicFilterFunction(TableFunctionGenerator[SplitSequenceArgs, Split
             init = params.init_call
             merged = None
             if init is not None and init.pushdown_filters is not None:
-                merged = cls.pushdown_filters(
-                    init.pushdown_filters,
-                    join_keys=init.join_keys,
-                    output_schema=init.output_schema,
-                )
+                # The framework validated these at init; re-binding every
+                # predicate through the evaluator on every tick is what made
+                # this fixture cost ~10 ms a batch.
+                with replaying_accepted_state():
+                    merged = cls.pushdown_filters(
+                        init.pushdown_filters,
+                        join_keys=init.join_keys,
+                        output_schema=init.output_schema,
+                    )
             if merged is None:
                 merged = params.current_pushdown_filters
             filter_str = _render_filters_canonical(merged)

vgi/auth.py

@@ -18,8 +18,9 @@ JWT auth (requires ``vgi[oauth]``):
 
 from __future__ import annotations
 
-import contextlib
+import importlib
 from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
 
 from vgi_rpc.rpc import AuthContext, CallContext
 
@@ -43,34 +44,89 @@ __all__ = [
     "TokenResolver",
 ]
 
-# HTTP auth helpers — available when vgi[http] is installed.
-with contextlib.suppress(ImportError):
-    from vgi_rpc.http import (  # noqa: F401
-        AuthUnavailableError,
-        OAuthResourceMetadata,
-        bearer_authenticate,
-        bearer_authenticate_static,
-        chain_authenticate,
-        parse_client_id,
-        parse_client_secret,
-        parse_device_code_client_id,
-        parse_device_code_client_secret,
+# HTTP auth helpers (``vgi[http]``) and JWT auth (``vgi[oauth]``), resolved on
+# first access rather than at import. ``vgi/__init__`` imports this module, and
+# ``vgi_rpc.http`` brings in the HTTP server and client stack (falcon, httpx2,
+# cryptography, joserfc): importing it here put about 180 ms on every
+# ``import vgi``, and so on the start of every subprocess worker, which never
+# uses it. ``from vgi.auth import bearer_authenticate`` works as before; when
+# the extra is missing it raises ImportError, as it did.
+_LAZY_EXPORTS = {
+    "AuthUnavailableError": "vgi_rpc.http",
+    "OAuthResourceMetadata": "vgi_rpc.http",
+    "bearer_authenticate": "vgi_rpc.http",
+    "bearer_authenticate_static": "vgi_rpc.http",
+    "chain_authenticate": "vgi_rpc.http",
+    "parse_client_id": "vgi_rpc.http",
+    "parse_client_secret": "vgi_rpc.http",
+    "parse_device_code_client_id": "vgi_rpc.http",
+    "parse_device_code_client_secret": "vgi_rpc.http",
+    "jwt_authenticate": "vgi_rpc.http._oauth_jwt",
+}
+
+if TYPE_CHECKING:
+    # ``X as X``: explicit re-exports, so type checkers see these as part of
+    # ``vgi.auth`` exactly as they were before they became lazy.
+    from vgi_rpc.http import (
+        AuthUnavailableError as AuthUnavailableError,
+    )
+    from vgi_rpc.http import (
+        OAuthResourceMetadata as OAuthResourceMetadata,
+    )
+    from vgi_rpc.http import (
+        bearer_authenticate as bearer_authenticate,
+    )
+    from vgi_rpc.http import (
+        bearer_authenticate_static as bearer_authenticate_static,
+    )
+    from vgi_rpc.http import (
+        chain_authenticate as chain_authenticate,
+    )
+    from vgi_rpc.http import (
+        parse_client_id as parse_client_id,
+    )
+    from vgi_rpc.http import (
+        parse_client_secret as parse_client_secret,
+    )
+    from vgi_rpc.http import (
+        parse_device_code_client_id as parse_device_code_client_id,
     )
+    from vgi_rpc.http import (
+        parse_device_code_client_secret as parse_device_code_client_secret,
+    )
+    from vgi_rpc.http._oauth_jwt import jwt_authenticate as jwt_authenticate
+
+
+def __getattr__(name: str) -> Any:
+    """Import an HTTP or JWT auth helper on first access (PEP 562).
+
+    Args:
+        name: The attribute being looked up.
+
+    Returns:
+        The helper, which is then cached in this module's namespace.
+
+    Raises:
+        AttributeError: If ``name`` is not a helper this module re-exports, or
+            its extra is not installed.
+    """
+    module_name = _LAZY_EXPORTS.get(name)
+    if module_name is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    try:
+        value = getattr(importlib.import_module(module_name), name)
+    except ImportError as exc:
+        raise AttributeError(
+            f"module {__name__!r} has no attribute {name!r}: it needs {module_name}, which failed to import ({exc})"
+        ) from exc
+    globals()[name] = value
+    return value
+
+
+def __dir__() -> list[str]:
+    """List this module's names, the lazily imported helpers included.
 
-    __all__ += [
-        "AuthUnavailableError",
-        "OAuthResourceMetadata",
-        "bearer_authenticate",
-        "bearer_authenticate_static",
-        "chain_authenticate",
-        "parse_client_id",
-        "parse_client_secret",
-        "parse_device_code_client_id",
-        "parse_device_code_client_secret",
-    ]
-
-# JWT auth — available when vgi[oauth] is installed (requires authlib).
-with contextlib.suppress(ImportError):
-    from vgi_rpc.http._oauth_jwt import jwt_authenticate  # noqa: F401
-
-    __all__ += ["jwt_authenticate"]
+    Returns:
+        The module's attribute names.
+    """
+    return sorted({*globals(), *_LAZY_EXPORTS})

vgi/catalog/catalog_interface.py

@@ -2698,6 +2698,8 @@ class ReadOnlyCatalogInterface(CatalogInterface):
     _function_registry: "dict[tuple[SchemaKey, str], list[type]] | None" = None
     _macro_registry: "dict[tuple[SchemaKey, str], Macro] | None" = None
     _index_registry: "dict[tuple[SchemaKey, str], Index] | None" = None
+    # Function listings by (schema, listing type), filled in by ``_function_infos``.
+    _function_info_cache: "dict[tuple[SchemaKey, SchemaObjectType], tuple[FunctionInfo, ...]] | None" = None
     # Lazy registry build is one-time but the fixture HTTP server is
     # multi-threaded and shares one catalog instance, so concurrent
     # first-requests can race the build. Serialize it under a lock and flip
@@ -2738,6 +2740,7 @@ class ReadOnlyCatalogInterface(CatalogInterface):
         self._function_registry = {}
         self._macro_registry = {}
         self._index_registry = {}
+        self._function_info_cache = {}
 
         def _register_table(schema_key: SchemaKey, table: "Table") -> None:
             key = (schema_key, table.name.lower())
@@ -3320,29 +3323,46 @@ class ReadOnlyCatalogInterface(CatalogInterface):
                 if sn == path_key and macro.macro_type == target_macro_type:
                     results.append(macro.to_macro_info(schema_path))
         else:
-            # SCALAR_FUNCTION or TABLE_FUNCTION
-            for (sn, _), func_classes in self._function_registry.items():
-                if sn != path_key:
-                    continue
-                for func_cls in func_classes:
-                    func_info = self._function_to_info(func_cls, schema_path)
-                    # Filter by function type
-                    if type_enum == SchemaObjectType.SCALAR_FUNCTION and func_info.function_type != FunctionType.SCALAR:
-                        continue
-                    if type_enum == SchemaObjectType.TABLE_FUNCTION and func_info.function_type not in (
-                        FunctionType.TABLE,
-                        FunctionType.TABLE_BUFFERING,
-                    ):
-                        continue
-                    if (
-                        type_enum == SchemaObjectType.AGGREGATE_FUNCTION
-                        and func_info.function_type != FunctionType.AGGREGATE
-                    ):
-                        continue
-                    results.append(func_info)
+            # SCALAR_FUNCTION, TABLE_FUNCTION or AGGREGATE_FUNCTION
+            results.extend(self._function_infos(path_key, schema_path, type_enum))
 
         return results
 
+    def _function_infos(
+        self, path_key: SchemaKey, schema_path: SchemaPath, type_enum: SchemaObjectType
+    ) -> tuple[FunctionInfo, ...]:
+        """Return the schema's functions of one listing type, built on the first request.
+
+        A listing depends only on the registered classes' static metadata, so
+        every request for it returns the same `FunctionInfo` instances. Rebuilding
+        it on each request re-derived every function in the schema to keep the
+        requested type's. It also made the worker encode each one again, at
+        about 0.8 ms per function, because ``FunctionsResponse.from_infos``
+        stores an instance's encoding on the instance.
+        The instances are shared between requests, so they must not be mutated.
+        """
+        assert self._function_registry is not None
+        assert self._function_info_cache is not None
+        key = (path_key, type_enum)
+        cached = self._function_info_cache.get(key)
+        if cached is not None:
+            return cached
+        wanted = {
+            SchemaObjectType.SCALAR_FUNCTION: (FunctionType.SCALAR,),
+            SchemaObjectType.TABLE_FUNCTION: (FunctionType.TABLE, FunctionType.TABLE_BUFFERING),
+            SchemaObjectType.AGGREGATE_FUNCTION: (FunctionType.AGGREGATE,),
+        }.get(type_enum, ())
+        infos: list[FunctionInfo] = []
+        for (sn, _), func_classes in self._function_registry.items():
+            if sn != path_key:
+                continue
+            for func_cls in func_classes:
+                func_info = self._function_to_info(func_cls, schema_path)
+                if func_info.function_type in wanted:
+                    infos.append(func_info)
+        # Concurrent first requests may both build it; keep whichever landed first.
+        return self._function_info_cache.setdefault(key, tuple(infos))
+
     def copy_from_formats(
         self,
         *,

vgi/client/client.py

@@ -33,10 +33,14 @@ the HTTP flow and skip the subprocess branch.
 
 Parallel processing
 -------------------
-When a bind returns ``max_workers > 1`` the client spawns additional
-worker connections and distributes input batches round-robin. Output
-order is non-deterministic in parallel mode. This is optimization; a
-minimal port can ignore it and always use one connection.
+When an init returns ``max_workers > 1`` the client may use that many
+worker connections, capped by the CPU count and ``worker_limit``. For
+scalar and table-in-out calls it deals input batches round-robin and opens
+a secondary connection only when the first batch is dealt to it, so ``n``
+input batches use at most ``n`` connections. A table function has no input,
+so it opens every allowed connection at init. Output order is
+non-deterministic in parallel mode. This is optimization; a minimal port
+can ignore it and always use one connection.
 
 Key classes
 -----------
@@ -258,6 +262,37 @@ class WorkerConnection:
     _launch_ctx: AbstractContextManager[Any] | None = field(default=None, repr=False)
 
 
+@dataclass(frozen=True, slots=True)
+class _FanOut:
+    """How far a stream may fan out, and the init every secondary connection sends.
+
+    Built from the primary's bind and init. A secondary connection's init
+    echoes the primary's request plus its ``execution_id`` and init opaque
+    data, so all connections are parts of one execution.
+
+    Attributes:
+        max_workers: The total connections allowed, the primary included, after
+            ``_determine_max_workers`` applies the client's limits.
+        bind_request: The primary's bind request, carried inside each init.
+        bind_response: The primary's bind response.
+        init_response: The primary's init header. Supplies ``execution_id`` and
+            ``opaque_data``.
+        projection_ids: Projection, repeated on every secondary init.
+        pushdown_filters_batch: Pushed-down filters, repeated on every secondary init.
+        phase: Table-in-out phase, or ``None`` for other function kinds.
+        join_keys: Join-key batches, repeated on every secondary init.
+    """
+
+    max_workers: int
+    bind_request: BindRequest
+    bind_response: BindResponse
+    init_response: GlobalInitResponse
+    projection_ids: list[int] | None
+    pushdown_filters_batch: pa.RecordBatch | None
+    phase: TableInOutFunctionInitPhase | None
+    join_keys: list[pa.RecordBatch] | None
+
+
 class Client(CatalogClientMixin, AggregateClientMixin):
     """Canonical VGI client — HTTP is the path other-language ports mirror.
 
@@ -1731,7 +1766,8 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         join_keys: list[pa.RecordBatch] | None = None,
         at_unit: str | None = None,
         at_value: str | None = None,
-    ) -> tuple[BindRequest, BindResponse, GlobalInitResponse]:
+        fan_out_on_demand: bool = False,
+    ) -> tuple[BindRequest, BindResponse, GlobalInitResponse, _FanOut]:
         """Run the canonical bind → init → fan-out-workers sequence.
 
         All three function entry points (``scalar_function``,
@@ -1744,7 +1780,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         4. Read the `[`GlobalInitResponse`][]` header (carries ``max_workers``
            + ``execution_id`` for secondary workers).
         5. Spawn any additional workers and drive their ``init`` with the
-           primary's execution identity.
+           primary's execution identity — unless ``fan_out_on_demand``, in
+           which case the returned `_FanOut` is handed to
+           ``_distribute_and_collect``, which opens a secondary connection only
+           when an input batch is dealt to it. Input-driven calls (scalar and
+           table-in-out) fan out that way: the server's ``max_workers`` is an
+           upper bound (by default unbounded, so the client's CPU count), and
+           opening that many connections up front for a call with one input
+           batch costs a worker process each on the subprocess transport, all
+           but one of them idle. ``table_function`` has no input to deal, so
+           its connections are all opened here.
 
         Centralizing this keeps HTTP/subprocess differences and protocol
         changes (e.g. future scoped-secret re-bind, init hints) in one
@@ -1809,69 +1854,71 @@ class Client(CatalogClientMixin, AggregateClientMixin):
 
         init_response = stream.typed_header(GlobalInitResponse)
         max_workers = 1 if split_tokens is not None else self._determine_max_workers(init_response.max_workers)
-
-        self._spawn_additional_workers(
-            max_workers,
-            bind_request,
-            bind_response,
-            init_response,
+        fan_out = _FanOut(
+            max_workers=max_workers,
+            bind_request=bind_request,
+            bind_response=bind_response,
+            init_response=init_response,
             projection_ids=projection_ids,
             pushdown_filters_batch=pushdown_filters_batch,
             phase=phase,
             join_keys=join_keys,
         )
+        if not fan_out_on_demand:
+            self._spawn_additional_workers(fan_out)
 
-        return bind_request, bind_response, init_response
+        return bind_request, bind_response, init_response, fan_out
 
-    def _spawn_additional_workers(
-        self,
-        max_workers: int,
-        bind_request: BindRequest,
-        bind_response: BindResponse,
-        global_init_response: GlobalInitResponse,
-        *,
-        projection_ids: list[int] | None = None,
-        pushdown_filters_batch: pa.RecordBatch | None = None,
-        phase: TableInOutFunctionInitPhase | None = None,
-        join_keys: list[pa.RecordBatch] | None = None,
-    ) -> None:
-        """Spawn and initialize additional worker subprocesses in parallel.
+    def _init_secondary_worker(self, worker: WorkerConnection, fan_out: _FanOut) -> None:
+        """Send a secondary worker its ``init``, making it part of the primary's execution.
+
+        Args:
+            worker: The newly opened secondary connection.
+            fan_out: The primary's bind and init, which the secondary init repeats.
+        """
+        worker.substream_id = _substream_id_for(fan_out.phase)
+        worker.stream = self._do_init(
+            worker.proxy,
+            fan_out.bind_request,
+            fan_out.bind_response,
+            projection_ids=fan_out.projection_ids,
+            pushdown_filters_batch=fan_out.pushdown_filters_batch,
+            phase=fan_out.phase,
+            join_keys=fan_out.join_keys,
+            execution_id=fan_out.init_response.execution_id,
+            init_opaque_data=fan_out.init_response.opaque_data,
+            substream_id=worker.substream_id,
+        )
 
-        First spawns all worker subprocesses sequentially (fast operation), then
-        initializes all workers in parallel using threads. Each additional worker
-        receives a secondary init with the execution_id from the primary worker.
+    def _spawn_additional_workers(self, fan_out: _FanOut) -> None:
+        """Open and initialize every secondary connection ``fan_out`` allows, in parallel.
 
-        The spawned workers are appended to self._additional_workers list.
+        Opens the connections one after another, which is fast: a subprocess is
+        only started here, not waited for. Then initializes them in parallel
+        threads, overlapping the workers' startup. Each secondary init carries
+        the primary's ``execution_id``. The connections are appended to
+        ``self._additional_workers``.
 
-        If max_workers is 1 or less, this method returns immediately without
-        spawning any workers.
+        ``table_function`` fans out this way because it has no input to deal.
+        Input-driven calls use `_distribute_and_collect`, which opens a
+        connection only when a batch is dealt to it. When ``max_workers`` is 1
+        or less, this method returns without opening anything.
 
         Args:
-            max_workers: Total number of workers desired (including the primary
-                worker). For example, if max_workers=4, this method spawns
-                3 additional workers (indices 1, 2, 3).
-            bind_request: The original bind request to embed in init.
-            bind_response: The bind response with output schema.
-            global_init_response: The primary worker's init response containing
-                execution_id and opaque_data for secondary init.
-            projection_ids: Optional column indices for projection.
-            pushdown_filters_batch: Optional deserialized filter predicates.
-            phase: Table-in-out function phase (INPUT or FINALIZE).
-            join_keys: Optional serialized join-key batches pushed down for
-                join filtering, echoed to every secondary worker exactly like
-                ``pushdown_filters_batch``.
+            fan_out: The connection limit and the primary's bind and init, which
+                every secondary init repeats.
 
         Raises:
             [`ClientError`][]: If any worker fails to initialize. The exception wraps
                 the first initialization error encountered.
 
         """
-        if max_workers <= 1:
+        if fan_out.max_workers <= 1:
             return
 
         # Spawn all worker subprocesses first (fast)
         new_workers: list[WorkerConnection] = []
-        for worker_index in range(1, max_workers):
+        for worker_index in range(1, fan_out.max_workers):
             worker = self._spawn_worker(worker_index)
             new_workers.append(worker)
             self._additional_workers.append(worker)
@@ -1881,20 +1928,7 @@ class Client(CatalogClientMixin, AggregateClientMixin):
 
         def do_init(worker: WorkerConnection) -> None:
             try:
-                worker.substream_id = _substream_id_for(phase)
-                stream = self._do_init(
-                    worker.proxy,
-                    bind_request,
-                    bind_response,
-                    projection_ids=projection_ids,
-                    pushdown_filters_batch=pushdown_filters_batch,
-                    phase=phase,
-                    join_keys=join_keys,
-                    execution_id=global_init_response.execution_id,
-                    init_opaque_data=global_init_response.opaque_data,
-                    substream_id=worker.substream_id,
-                )
-                worker.stream = stream
+                self._init_secondary_worker(worker, fan_out)
             except Exception as e:
                 init_errors.append(e)
 
@@ -2111,13 +2145,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         output_queue: Queue[tuple[int, list[pa.RecordBatch], list[list[int]] | None] | BaseException],
         *,
         decode_parent_rows: bool = False,
+        init: _FanOut | None = None,
     ) -> None:
         """Thread function that processes batches for a single worker.
 
         Runs in a dedicated thread, pulling (batch_index, batch) tuples from
         the input queue, processing them via _process_batch_on_worker, and
         pushing (batch_index, output_batches, parent_rows_batches) tuples to
-        the output queue.
+        the output queue. With ``init``, first sends the worker its secondary
+        init, so a connection opened on demand starts up without holding up
+        the thread dealing the batches.
 
         When None is received from input_queue, signals thread completion by
         pushing (-1, [], None) to output_queue and exits.
@@ -2132,9 +2169,13 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             output_queue: Thread-safe queue for results.
             decode_parent_rows: Forwarded to `_process_batch_on_worker` — see
                 its docstring.
+            init: The fan-out plan to initialize a just-opened secondary worker
+                from, or ``None`` for a worker already initialized.
 
         """
         try:
+            if init is not None:
+                self._init_secondary_worker(worker, init)
             while True:
                 item = input_queue.get()
                 if item is None:
@@ -2154,7 +2195,7 @@ class Client(CatalogClientMixin, AggregateClientMixin):
     def _distribute_and_collect(
         self,
         *,
-        all_workers: list[WorkerConnection],
+        fan_out: _FanOut,
         first_batch: pa.RecordBatch,
         remaining_input: Iterator[pa.RecordBatch],
         decode_parent_rows: bool = False,
@@ -2166,8 +2207,16 @@ class Client(CatalogClientMixin, AggregateClientMixin):
         worker, spawns a dedicated thread that pulls batches from an input queue,
         sends them to the worker, and pushes results to a shared output queue.
 
+        Batch ``i`` goes to connection ``i % fan_out.max_workers``. The primary
+        is connection 0. A secondary connection is opened, and sent its init,
+        when the first batch is dealt to it, so a call with ``n`` input batches
+        uses ``min(n, max_workers)`` connections. It never opens one that would
+        get no input. Opened connections are appended to
+        ``self._additional_workers``, where the caller closes them.
+
         Args:
-            all_workers: List of all workers (primary + additional).
+            fan_out: The primary's init and the connection limit, from
+                `_initialize_stream_common` with ``fan_out_on_demand=True``.
             first_batch: The first input batch, already consumed from the
                 iterator by the calling method.
             remaining_input: Iterator for remaining input batches.
@@ -2190,42 +2239,48 @@ class Client(CatalogClientMixin, AggregateClientMixin):
             [`ClientError`][]: If a worker thread fails with an exception.
 
         """
-        num_workers = len(all_workers)
-
-        _logger.debug("starting_parallel_processing num_workers=%s", num_workers)
-
-        # Create queues for each worker
-        input_queues: list[Queue[tuple[int, pa.RecordBatch] | None]] = [Queue() for _ in range(num_workers)]
+        assert self._primary is not None
+        max_workers = max(fan_out.max_workers, 1)
+        input_queues: list[Queue[tuple[int, pa.RecordBatch] | None]] = []
         output_queue: Queue[tuple[int, list[pa.RecordBatch], list[list[int]] | None] | BaseException] = Queue()
-
-        # Start worker threads
         threads: list[threading.Thread] = []
-        for i, worker in enumerate(all_workers):
+
+        def start_worker_thread(worker: WorkerConnection, init: _FanOut | None) -> None:
+            input_queue: Queue[tuple[int, pa.RecordBatch] | None] = Queue()
+            input_queues.append(input_queue)
             thread = threading.Thread(
                 target=self._worker_thread_loop,
-                args=(worker, input_queues[i], output_queue),
-                kwargs={"decode_parent_rows": decode_parent_rows},
+                args=(worker, input_queue, output_queue),
+                kwargs={"decode_parent_rows": decode_parent_rows, "init": init},
                 daemon=True,
             )
             thread.start()
             threads.append(thread)
 
-        # Distribute batches round-robin across workers
-        batch_index = 0
-        batches_sent = 0
-
-        # Send first batch
-        worker_idx = batch_index % num_workers
-        input_queues[worker_idx].put((batch_index, first_batch))
-        batches_sent += 1
-        batch_index += 1
+        start_worker_thread(self._primary, None)
 
-        # Send remaining batches
-        for input_batch in remaining_input:
-            worker_idx = batch_index % num_workers
-            input_queues[worker_idx].put((batch_index, input_batch))
-            batches_sent += 1
-            batch_index += 1
+        # Deal batches round-robin, opening a connection the first time one is
+        # dealt to it. Opening is quick (a subprocess is started, not waited
+        # for); the secondary init runs on the connection's own thread.
+        batches_sent = 0
+        try:
+            for batch_index, input_batch in enumerate(itertools.chain((first_batch,), remaining_input)):
+                worker_idx = batch_index % max_workers
+                if worker_idx == len(input_queues):
+                    worker = self._spawn_worker(worker_idx)
+                    self._additional_workers.append(worker)
+                    start_worker_thread(worker, fan_out)
+                input_queues[worker_idx].put((batch_index, input_batch))
+                batches_sent += 1
+        except BaseException:
+            # The input or a connection failed: let the started threads finish
+            # what they were dealt and exit rather than wait forever.
+            for q in input_queues:
+                q.put(None)
+            raise
+
+        num_workers = len(input_queues)
+        _logger.debug("starting_parallel_processing num_workers=%s", num_workers)
 
         # Signal end of input to all workers
         for q in input_queues:
@@ -2377,7 +2432,7 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                 input_schema = first_batch.schema
                 pushdown_filters_batch = self._deserialize_pushdown_filters(pushdown_filters)
 
-                bind_request, bind_response, init_response = self._initialize_stream_common(
+                bind_request, bind_response, init_response, fan_out = self._initialize_stream_common(
                     function_name=function_name,
                     schema_path=schema_path,
                     arguments=arguments,
@@ -2391,12 +2446,12 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                     phase=TableInOutFunctionInitPhase.INPUT,
                     bind_result_callback=bind_result_callback,
                     join_keys=join_keys,
+                    fan_out_on_demand=True,
                 )
 
                 # Process input batches across all workers
-                all_workers = [self._primary] + self._additional_workers
                 yield from self._distribute_and_collect(
-                    all_workers=all_workers,
+                    fan_out=fan_out,
                     first_batch=first_batch,
                     remaining_input=input,
                     decode_parent_rows=parent_row_callback is not None,
@@ -2404,7 +2459,7 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                 )
 
                 # Close all input streams
-                for worker in all_workers:
+                for worker in [self._primary, *self._additional_workers]:
                     if worker.stream is not None:
                         worker.stream.close()
                         worker.stream = None
@@ -3492,7 +3547,7 @@ class Client(CatalogClientMixin, AggregateClientMixin):
 
                 input_schema = first_batch.schema
 
-                self._initialize_stream_common(
+                *_, fan_out = self._initialize_stream_common(
                     function_name=function_name,
                     schema_path=schema_path,
                     arguments=arguments,
@@ -3505,18 +3560,18 @@ class Client(CatalogClientMixin, AggregateClientMixin):
                     pushdown_filters_batch=None,
                     phase=None,
                     bind_result_callback=bind_result_callback,
+                    fan_out_on_demand=True,
                 )
 
                 # Process batches across all workers
-                all_workers = [self._primary] + self._additional_workers
                 yield from self._distribute_and_collect(
-                    all_workers=all_workers,
+                    fan_out=fan_out,
                     first_batch=first_batch,
                     remaining_input=input,
                 )
 
                 # Close streams and secondary workers
-                for worker in all_workers:
+                for worker in [self._primary, *self._additional_workers]:
                     if worker.stream is not None:
                         worker.stream.close()
                         worker.stream = None

vgi/filter_v2.py

@@ -4,9 +4,12 @@
 
 from __future__ import annotations
 
+import contextlib
 import json
 import re
 import threading
+from collections.abc import Iterator
+from contextvars import ContextVar
 from dataclasses import dataclass
 from enum import StrEnum
 from typing import Any
@@ -47,6 +50,54 @@ _KNOWN_ARROW_EXTENSIONS = {
 }
 
 
+#: Set while re-deriving filter state that this worker already accepted (see
+#: :func:`replaying_accepted_state`). Default off: every document arriving from a
+#: client is validated in full.
+_REPLAYING_ACCEPTED_STATE: ContextVar[bool] = ContextVar("vgi_filter_v2_replaying_accepted_state", default=False)
+
+
+@contextlib.contextmanager
+def replaying_accepted_state() -> Iterator[None]:
+    """Re-parse filter documents this worker has already validated and accepted.
+
+    Validating a predicate binds it against the output schema through the
+    embedded DuckDB evaluator -- one query, several milliseconds, per predicate.
+    The spec asks for that once per accepted revision. An HTTP worker, though,
+    rebuilds its filter state from the call and cursor tokens on every turn, and
+    re-validating there made every continuation pay a bind per predicate (and,
+    with dynamic filters, one per delta ever applied). Those tokens are sealed by
+    the worker, so the documents in them are exactly the ones it validated when
+    they first arrived: inside this context the parse is still strict about
+    structure, but the redundant bind is skipped.
+
+    Yields:
+        None.
+
+    """
+    token = _REPLAYING_ACCEPTED_STATE.set(True)
+    try:
+        yield
+    finally:
+        _REPLAYING_ACCEPTED_STATE.reset(token)
+
+
+def delta_revisions(batch: pa.RecordBatch) -> tuple[tuple[str, int], ...]:
+    """Return the ``(id, revision)`` of every update a delta batch carries.
+
+    A structural read only, for bookkeeping over deltas that were already
+    applied (and therefore validated); it does not apply or validate anything.
+
+    Args:
+        batch: A delta document batch.
+
+    Returns:
+        One ``(id, revision)`` pair per update, in document order.
+
+    """
+    document = json.loads(batch.column(0)[0].as_py())
+    return tuple((update["id"], update["revision"]) for update in document["updates"])
+
+
 class FilterV2Error(ValueError):
     """A Filter Encoding v2 document is malformed or cannot be evaluated."""
 
@@ -666,7 +717,11 @@ class _Parser:
                 raise FilterV2Error("runtime_filter predicates must be advisory")
         elif not self._is_boolean(expression):
             raise FilterV2Error("predicate root must resolve to BOOLEAN")
-        if self.output_schema is not None and not isinstance(expression, RuntimeFilter):
+        if (
+            self.output_schema is not None
+            and not isinstance(expression, RuntimeFilter)
+            and not _REPLAYING_ACCEPTED_STATE.get()
+        ):
             empty = pa.RecordBatch.from_arrays(
                 [pa.array([], type=field.type) for field in self.output_schema],
                 schema=self.output_schema,
@@ -1110,7 +1165,12 @@ def _get_evaluation_connection(context: EvaluationContext) -> Any:
                 f"found {engine_name} {engine_version or '<unknown>'}"
             )
 
-        connection = connect()
+        # One thread. Each evaluates one batch against one predicate, which
+        # DuckDB cannot parallelize, and there is one of these databases per
+        # (server thread, context): at the default of one scheduler thread per
+        # core, a waitress worker on a 48-core host carried ~290 threads, and
+        # every evaluation paid to dispatch onto that pool.
+        connection = connect(config={"threads": 1})
         _apply_context(connection, context)
         connections[key] = connection
     return connection

vgi/protocol.py

@@ -63,6 +63,7 @@ from vgi.catalog.catalog_interface import (
     TableInfo,
     ViewInfo,
 )
+from vgi.filter_v2 import delta_revisions, replaying_accepted_state
 from vgi.function import StreamStateCodec, _is_state_codec
 from vgi.function_storage import BoundStorage, FrameworkNS, attach_catalog_bytes
 from vgi.invocation import BindResponse, FunctionType, GlobalInitResponse
@@ -842,6 +843,37 @@ class TransactionBeginResponse(ArrowSerializableDataclass):
     transaction_opaque_data: bytes | None = None
 
 
+#: Instance attribute where `_item_ipc_bytes` keeps an item's encoding.
+_ITEM_IPC_MEMO = "_vgi_item_ipc"
+
+
+def _item_ipc_bytes(info: Any) -> bytes:
+    """Return a catalog item's Arrow IPC encoding, encoded once per instance.
+
+    A catalog item is a frozen dataclass, so its encoding cannot change, and
+    encoding one costs about 0.8 ms (a one-row batch of some 40 columns, plus
+    its schema). A catalog that returns the same instances on every listing,
+    as ``CatalogInterface`` does for functions, therefore pays for the encoding
+    once. The encoding is kept in the instance's ``__dict__``, as
+    ``functools.cached_property`` does on a frozen dataclass. It is not a field,
+    so it takes no part in equality, ``repr``, ``dataclasses.replace`` or
+    serialization. An item without a ``__dict__`` is encoded on every call.
+
+    Args:
+        info: The catalog item (an ``ArrowSerializableDataclass``).
+
+    Returns:
+        The item's Arrow IPC stream bytes.
+    """
+    memo = getattr(info, "__dict__", None)
+    if memo is None:
+        return info.serialize_to_bytes()  # type: ignore[no-any-return]
+    encoded = memo.get(_ITEM_IPC_MEMO)
+    if encoded is None:
+        encoded = memo[_ITEM_IPC_MEMO] = info.serialize_to_bytes()
+    return encoded  # type: ignore[no-any-return]
+
+
 def _catalog_items_response(item_type: type) -> type:
     """Generate a catalog items response class for the given `ArrowSerializableDataclass` type.
 
@@ -868,13 +900,13 @@ def _catalog_items_response(item_type: type) -> type:
 
         @staticmethod
         def from_infos(infos: list) -> _Response:  # type: ignore[type-arg]
-            return _Response(items=[info.serialize_to_bytes() for info in infos])
+            return _Response(items=[_item_ipc_bytes(info) for info in infos])
 
         @staticmethod
         def from_optional(info: object | None) -> _Response:
             if info is None:
                 return _Response(items=[])
-            return _Response(items=[info.serialize_to_bytes()])  # type: ignore[attr-defined]
+            return _Response(items=[_item_ipc_bytes(info)])
 
         def to_infos(self) -> list:  # type: ignore[type-arg]
             return [item_type.deserialize_from_bytes(b) for b in self.items]  # type: ignore[attr-defined]
@@ -1827,17 +1859,84 @@ class _TrackingOutputCollector:
         return getattr(self._inner, name)
 
 
-def _apply_filter_delta_bytes(current: Any, filter_bytes: bytes) -> Any:
-    """Decode one IPC delta and apply it to existing immutable filter state."""
-    if current is None:
-        raise ValueError("dynamic filter delta received without an initial snapshot")
+def _decode_filter_delta(filter_bytes: bytes) -> pa.RecordBatch:
+    """Decode one IPC dynamic-filter delta into its single document batch."""
     if len(filter_bytes) > (17 << 20):
         raise ValueError("dynamic filter IPC payload exceeds the encoded-size limit")
     table = pa.ipc.open_stream(filter_bytes).read_all()
     batches = table.to_batches()
     if len(batches) != 1:
         raise ValueError("dynamic filter metadata must contain exactly one RecordBatch")
-    return current.apply_delta(batches[0])
+    return batches[0]
+
+
+def _apply_filter_delta_bytes(current: Any, filter_bytes: bytes) -> Any:
+    """Decode one IPC delta and apply it to existing immutable filter state."""
+    if current is None:
+        raise ValueError("dynamic filter delta received without an initial snapshot")
+    return current.apply_delta(_decode_filter_delta(filter_bytes))
+
+
+def _record_filter_delta(current: Any, history: list[bytes], filter_bytes: bytes) -> tuple[Any, list[bytes], list[str]]:
+    """Apply one tick's dynamic-filter delta; return the new filters and what a cursor must carry.
+
+    An HTTP stream cannot keep parsed filters between turns, so its cursor
+    carries the deltas it has applied and :func:`_replay_filter_history` rebuilds
+    the state from them on the next turn. Carrying *every* delta made each turn
+    replay all of its predecessors -- quadratic in the tick count, and a Top-N
+    scan gets a delta on nearly every tick (``dynamic_filter.test`` took ~200 s).
+
+    So the history is compacted to the deltas that installed some predicate's
+    *current* revision, tombstones included: for each ``(id, revision)`` of the
+    live state, the first delta carrying it. Replaying just those reproduces the
+    same predicates, values and revisions -- earlier updates to an ID are
+    overwritten by its current revision, later ones were stale and stay stale --
+    and the history stays bounded by the number of predicate IDs instead of
+    growing per tick. Replay cannot always reproduce predicate *order* (an ID
+    removed and later re-added moves to the end), so the order is returned
+    alongside to be restored explicitly.
+
+    Args:
+        current: The stream's current ``PushdownFilters``.
+        history: The compacted deltas applied so far.
+        filter_bytes: The IPC bytes of the delta this tick carried.
+
+    Returns:
+        ``(filters, history, order)``: the updated filters, the compacted
+        history including this delta if it is still needed, and the live
+        predicate IDs in order.
+
+    """
+    updated = _apply_filter_delta_bytes(current, filter_bytes)
+    wanted = set(updated._revision_map().items())
+    kept: list[bytes] = []
+    for delta in [*history, filter_bytes]:
+        carried = set(delta_revisions(_decode_filter_delta(delta))) & wanted
+        if carried:
+            kept.append(delta)
+            wanted -= carried
+    return updated, kept, [p.id for p in updated.predicates]
+
+
+def _replay_filter_history(filters: Any, history: list[bytes], order: list[str]) -> Any:
+    """Rebuild a turn's filters from the init snapshot plus the carried deltas.
+
+    Runs inside :func:`vgi.filter_v2.replaying_accepted_state` (see the call
+    sites): every document here was validated when it first arrived, and both
+    tokens are sealed by the worker.
+
+    Args:
+        filters: The ``PushdownFilters`` parsed from the init snapshot.
+        history: The compacted deltas from :func:`_record_filter_delta`.
+        order: The live predicate order recorded alongside them.
+
+    Returns:
+        The stream's current ``PushdownFilters``.
+
+    """
+    for filter_bytes in history:
+        filters = _apply_filter_delta_bytes(filters, filter_bytes)
+    return filters._with_predicate_order(order) if history else filters
 
 
 @dataclass
@@ -1868,7 +1967,10 @@ class TableProducerState(_VgiCallStateHolder, ProducerState):
     _user_state: Annotated[Any, Transient()] = field(default=None, repr=False)
     _pushdown_filters: Annotated[Any, Transient()] = field(default=None, repr=False)  # PushdownFilters | None
     _auto_apply: Annotated[bool, Transient()] = field(default=False, repr=False)
+    # The dynamic-filter deltas this stream must replay on an HTTP turn, compacted by
+    # _record_filter_delta, and the live predicate order they rebuild.
     _filter_delta_history: list[bytes] = field(default_factory=list, repr=False)
+    _filter_predicate_order: list[str] = field(default_factory=list, repr=False)
     _vgi_tracer: Annotated[VgiTracer, Transient()] = field(default_factory=get_noop_tracer, repr=False)
     # Conditional-revalidation validators read off the first tick's custom_metadata
     # and surfaced to the generator via ProcessParams (M6). None on a normal call.
@@ -1935,13 +2037,17 @@ class TableProducerState(_VgiCallStateHolder, ProducerState):
         # manual consumers see the same required/current state after HTTP rehydrate.
         self._auto_apply = func_cls._should_auto_apply_filters()
         if self._call.init_call.pushdown_filters is not None:
-            self._pushdown_filters = func_cls.pushdown_filters(
-                self._call.init_call.pushdown_filters,
-                join_keys=self._call.init_call.join_keys,
-                output_schema=self._call.init_call.output_schema,
-            )
-            for filter_bytes in self._filter_delta_history:
-                self._pushdown_filters = _apply_filter_delta_bytes(self._pushdown_filters, filter_bytes)
+            # Everything re-parsed here was validated when it first arrived.
+            with replaying_accepted_state():
+                self._pushdown_filters = _replay_filter_history(
+                    func_cls.pushdown_filters(
+                        self._call.init_call.pushdown_filters,
+                        join_keys=self._call.init_call.join_keys,
+                        output_schema=self._call.init_call.output_schema,
+                    ),
+                    self._filter_delta_history,
+                    self._filter_predicate_order,
+                )
             self._params = dataclasses.replace(self._params, current_pushdown_filters=self._pushdown_filters)
         # Restore _user_state from serialized bytes if available
         if self._user_state_bytes is not None:
@@ -1978,8 +2084,9 @@ class TableProducerState(_VgiCallStateHolder, ProducerState):
         if len(encoded_filters) > (24 << 20):
             raise ValueError("base64 dynamic filter metadata exceeds the encoded-size limit")
         filter_bytes = base64.b64decode(encoded_filters, validate=True)
-        self._pushdown_filters = _apply_filter_delta_bytes(self._pushdown_filters, filter_bytes)
-        self._filter_delta_history.append(filter_bytes)
+        self._pushdown_filters, self._filter_delta_history, self._filter_predicate_order = _record_filter_delta(
+            self._pushdown_filters, self._filter_delta_history, filter_bytes
+        )
 
     def produce(self, out: OutputCollector, ctx: CallContext) -> None:
         """Produce the next output batch from the table function."""
@@ -2052,7 +2159,10 @@ class TableInOutExchangeState(_VgiCallStateHolder, ExchangeState):
     _user_state: Annotated[Any, Transient()] = field(default=None, repr=False)
     _pushdown_filters: Annotated[Any, Transient()] = field(default=None, repr=False)  # PushdownFilters | None
     _auto_apply: Annotated[bool, Transient()] = field(default=False, repr=False)
+    # The dynamic-filter deltas this stream must replay on an HTTP turn, compacted by
+    # _record_filter_delta, and the live predicate order they rebuild.
     _filter_delta_history: list[bytes] = field(default_factory=list, repr=False)
+    _filter_predicate_order: list[str] = field(default_factory=list, repr=False)
     _vgi_tracer: Annotated[VgiTracer, Transient()] = field(default_factory=get_noop_tracer, repr=False)
 
     def __post_init__(self) -> None:
@@ -2113,13 +2223,17 @@ class TableInOutExchangeState(_VgiCallStateHolder, ExchangeState):
         )
         self._auto_apply = func_cls._should_auto_apply_filters()
         if self._call.init_call.pushdown_filters is not None:
-            self._pushdown_filters = func_cls.pushdown_filters(
-                self._call.init_call.pushdown_filters,
-                join_keys=self._call.init_call.join_keys,
-                output_schema=self._call.init_call.output_schema,
-            )
-            for filter_bytes in self._filter_delta_history:
-                self._pushdown_filters = _apply_filter_delta_bytes(self._pushdown_filters, filter_bytes)
+            # Everything re-parsed here was validated when it first arrived.
+            with replaying_accepted_state():
+                self._pushdown_filters = _replay_filter_history(
+                    func_cls.pushdown_filters(
+                        self._call.init_call.pushdown_filters,
+                        join_keys=self._call.init_call.join_keys,
+                        output_schema=self._call.init_call.output_schema,
+                    ),
+                    self._filter_delta_history,
+                    self._filter_predicate_order,
+                )
             self._params = dataclasses.replace(self._params, current_pushdown_filters=self._pushdown_filters)
         # Restore _user_state from serialized bytes if available
         if self._user_state_bytes is not None:
@@ -2141,8 +2255,9 @@ class TableInOutExchangeState(_VgiCallStateHolder, ExchangeState):
                 if len(encoded) > (24 << 20):
                     raise ValueError("base64 dynamic filter metadata exceeds the encoded-size limit")
                 filter_bytes = base64.b64decode(encoded, validate=True)
-                self._pushdown_filters = _apply_filter_delta_bytes(self._pushdown_filters, filter_bytes)
-                self._filter_delta_history.append(filter_bytes)
+                self._pushdown_filters, self._filter_delta_history, self._filter_predicate_order = _record_filter_delta(
+                    self._pushdown_filters, self._filter_delta_history, filter_bytes
+                )
         params = dataclasses.replace(
             self._params,
             auth_context=ctx.auth,

vgi/scalar_function.py

@@ -755,7 +755,8 @@ class ScalarFunctionGenerator(vgi.function.Function):
         """Initialize the function during the init API call.
 
         Override to perform one-time setup that should happen after bind
-        but before processing batches. The default returns max_processes=1.
+        but before processing batches. The default leaves ``max_workers``
+        unbounded, so the caller decides how many connections to use.
 
         Args:
             bind_call: The original BindCall with arguments and schema.
@@ -769,7 +770,7 @@ class ScalarFunctionGenerator(vgi.function.Function):
             storage: [`BoundStorage`][] for storing data across calls.
 
         Returns:
-            [`GlobalInitResponse`][] with max_processes and optional opaque data.
+            [`GlobalInitResponse`][] with ``max_workers`` and optional opaque data.
 
         """
         return GlobalInitResponse()

vgi/table_filter_pushdown.py

@@ -1131,6 +1131,42 @@ class PushdownFilters:
             raise FilterDeserializationError(f"Failed to parse filter delta: {exc}") from exc
         return _pushdown_from_v2_state(state)
 
+    def _revision_map(self) -> dict[str, int]:
+        """Map every predicate ID seen in this scan, tombstones included, to its last applied revision.
+
+        Returns:
+            ``{predicate_id: revision}``; empty for filters not decoded from v2.
+
+        """
+        return dict(self._v2_state.revisions) if self._v2_state is not None else {}
+
+    def _with_predicate_order(self, order: list[str]) -> PushdownFilters:
+        """Return these filters with their v2 predicates arranged in ``order``.
+
+        Only the order changes: ``order`` must name exactly the live predicate IDs.
+        Used to restore the order a state had before its delta history was
+        compacted (an ID removed and later re-added moves to the end, which a
+        shorter history does not reproduce by itself).
+
+        Args:
+            order: The live predicate IDs in the order to restore.
+
+        Returns:
+            Filters with the same predicates, arranged in ``order``.
+
+        Raises:
+            FilterDeserializationError: If ``order`` does not name exactly the
+                live predicates.
+
+        """
+        state = self._v2_state
+        if state is None or [p.id for p in state.predicates] == order:
+            return self
+        by_id = {p.id: p for p in state.predicates}
+        if len(order) != len(by_id) or set(order) != set(by_id):
+            raise FilterDeserializationError("recorded predicate order does not match the replayed filter state")
+        return _pushdown_from_v2_state(dataclasses.replace(state, predicates=tuple(by_id[i] for i in order)))
+
     def get_join_keys_batch(self) -> pa.RecordBatch | None:
         """Return a merged join keys batch for temp table registration.
 

{vgi_python-0.35.0.dist-info → vgi_python-0.36.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.35.0
+Version: 0.36.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.35.0.dist-info → vgi_python-0.36.0.dist-info}/RECORD

@@ -5,12 +5,12 @@ vgi/aggregate_function.py,sha256=v03wBSy2Rrl9F2pMYvu8GnGrTCQrAVm01HXIFNdjnm4,268
 vgi/argument_spec.py,sha256=SADH97VlFu3asI8CWZ_I7XTuSxwqFciFgdupdAf-tkY,34231
 vgi/arguments.py,sha256=ncnC1CS7KN-2rDjSw9vdf1uHqV5ozebet_eZtcq_ujw,72300
 vgi/attach_header.py,sha256=sxpzCOVa5fl_-kQSCZgeUxS5I9Kuv8VCUjAOF5s1pr0,3782
-vgi/auth.py,sha256=TlD5bhdyXnFuAoCnoHMU9ffjewVEnpwO5auIvp9GsCA,2486
+vgi/auth.py,sha256=m97hDDR8bXIQRBvqhHQZesBrNkAnKgSH6jhuXtGubjI,4688
 vgi/cache_control.py,sha256=IyBp9gZTCc4Y-4V7Rs05_o5lSEjbR2jt6cddXPU0dKw,7812
 vgi/copy_from_function.py,sha256=oroQe5qYzZTJ-ppzQTQ-OruEF1ZlwTBNLBoXYfN-P2o,7133
 vgi/copy_to_function.py,sha256=8l0v_DyEF1EnFwinStrZDQ-yNFfGJsv3Mr5o2jG30f0,9279
 vgi/exceptions.py,sha256=oX_sZc9xGWi7Xf8cJQf89fX19i3ocDEj_V_76GINgBQ,7294
-vgi/filter_v2.py,sha256=3cW6ukHOb0kx13-mN2UEvfib4_Z2r94zVT2E5zQZwiM,53769
+vgi/filter_v2.py,sha256=pSdmZF4HgSr9ZuM_mU3NMwUuAw0X4sZ7KonTMeGnjUw,56243
 vgi/filter_v2_builder.py,sha256=8OmyzDI6GDdH4pYm2sJFMidzKusi0I8IGNipolPpjX0,5535
 vgi/function.py,sha256=KIfvKk1nzBvRDPXViHNX89tv6gvIMiDj4e8fl6ANqQg,11626
 vgi/function_storage.py,sha256=FjVwxc1BIjWo-82j8YE5qcU8w66sGjwyGsTmCUhn-Kw,67576
@@ -22,10 +22,10 @@ vgi/meta_worker.py,sha256=WggIKvDAlahSrBPaVM7ZqVWk3Xc3JNE70EgeoXdy-3w,31725
 vgi/metadata.py,sha256=M8KZObX1wwCr2lPjgDL0-79gPbeM_NSOIqYCIAAck5M,74719
 vgi/otel.py,sha256=JlUDnk2c0UXSo_j1PJdS67Y--t55sXPg054ADuvHANc,15452
 vgi/profiling.py,sha256=rK7vij3oHhXC3JHI2BsMySsXh7_hrXi0aXpCW8iFSIc,3671
-vgi/protocol.py,sha256=m9W00W-c9jv2kqISp_wb8E8Rtity_60_12nzE3_d9Xg,165016
+vgi/protocol.py,sha256=iKuo-k6oH9nss_33LfYcekBBNhsM-yeHY0T5fwgg7iw,170333
 vgi/protocol_version.txt,sha256=wo_MpTY3vIjhJK8XJd8Ty5jGne3v1i-zzb4c22t2BiQ,6
 vgi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-vgi/scalar_function.py,sha256=zj6U1e5mXm4hk_ZB_CAjYmeO1LZBJcEMzNDzhwpXPmQ,53415
+vgi/scalar_function.py,sha256=YBy5s4eEBYTMjACGPE-8-82BObyJoM9qt9cnm24b_RQ,53485
 vgi/schema_path.py,sha256=GVlkZlCUj7GbVhgiocSMC6Ff5iTJhKqiYE62hMIEKiI,2403
 vgi/schema_utils.py,sha256=zQCliUgIpO2u77Mhtgg3OFvOEDNgJopu6kj2fw8YP1I,7945
 vgi/secret_protocol.py,sha256=iCDl7nF7Nb4cJV4vpfbldunTDqDJWzCzS7uCmOACf7U,7151
@@ -33,7 +33,7 @@ vgi/secret_service.py,sha256=r7qCzfnOMirf9wSCUe-mgFe3z6v9_cDRzixuSAZG_bs,8973
 vgi/serve.py,sha256=5UmvRAGcHj5RKCOUQMXG9RFVYAbLZsHwwk1qVI9fipY,63844
 vgi/split_token.py,sha256=tutcksj-hgWrLY7UW3sSqkyoaD-cMqNTOeff40MGl_g,10850
 vgi/table_buffering_function.py,sha256=O5TcIdqk8-YViCQCkb8bOevO-zhe3IoyzLgwXDO3Cjc,19531
-vgi/table_filter_pushdown.py,sha256=2wJK09OkZZm-WYFsB-um4lbIMZV6gU1fcx8EsHglnA4,61861
+vgi/table_filter_pushdown.py,sha256=agzbDOz_-64E2uNZdDOQs8woP1CBAE_SU5rv2N4fqWk,63435
 vgi/table_function.py,sha256=8Pti3uFnsRaPCVG-CGYl9r3GHJF146Fp_62oXXwHKKw,73502
 vgi/table_in_out_function.py,sha256=F2QEtsbGEYHDVyQ4Cf9OCW7HlWXeqekAyx607foB-bs,19393
 vgi/worker.py,sha256=MGMQmy6PPKCqo5q_9fBS3z3zf4y6-UKdJAJiHGfnk50,249553
@@ -96,7 +96,7 @@ vgi/_test_fixtures/table/batch_index.py,sha256=P5ds0xgikuEQanSEWVWKMLbdvIzUeJraI
 vgi/_test_fixtures/table/batch_index_broken.py,sha256=kZOGrLL7ZW1rmwPmNEYRmiF_vqIfHsfXioq5vKPWHk0,7314
 vgi/_test_fixtures/table/cache.py,sha256=7oohKpFJ_si7LoKE7QINkz5oIyj1sx8Kiobpwo4Sn7w,72322
 vgi/_test_fixtures/table/catalog_scans.py,sha256=5j1Sx02-HWK7bFurDu4e9HiS3Q9BmBukA3sAErH4GHE,5080
-vgi/_test_fixtures/table/filters.py,sha256=x-k3fOHdA65-45CZUmyGxpmI0uk9eqaNixrs97Wlv70,41918
+vgi/_test_fixtures/table/filters.py,sha256=pOvuBFr0Rj7PXUWILUdfGf735sJda-iphJ_aQyiw_VY,42197
 vgi/_test_fixtures/table/late_materialization.py,sha256=BT1FOpQNeP4IXBS3hUvAmO-5jscl5iiQ11fvxUfN6DM,10355
 vgi/_test_fixtures/table/make_series.py,sha256=K-G_YNq25Kb7I5bp6XK4rCZzwMYNTxgKH3M_O-AQAlA,8812
 vgi/_test_fixtures/table/misc.py,sha256=71WOIFqk5ntnEIqsG-57rZ9DY7ShQqMKHi7yluNAlM4,16250
@@ -108,9 +108,9 @@ vgi/_test_fixtures/table/profiling_example.py,sha256=Rt3fgKxJhr7Q8QQRss2-OV13wh2
 vgi/_test_fixtures/table/required_filters.py,sha256=StJeS2tQYyXjivNs_tkcRBq4rVtIGoGQDWcggib1Rxg,7079
 vgi/_test_fixtures/table/same_name_cached.py,sha256=JMXSXTcqyO7XwWqIizgddlTvLYuUy02Qcq9ZkQnTYU0,4415
 vgi/_test_fixtures/table/same_name_schemas.py,sha256=yRti9HXO3-Y9ahiLIHoaZ31LvWjtPSbjxmPJyoQ_0mQ,4930
-vgi/_test_fixtures/table/sequence.py,sha256=yGn4b3Co06KgkYAUuCj27Fm8GDcHbWFq6zSvNPwHOqw,26073
+vgi/_test_fixtures/table/sequence.py,sha256=0kOckCwRu19RMKgFrUcnnnDR8e0utjX_E2kD3w6NnRU,26352
 vgi/_test_fixtures/table/settings.py,sha256=C4J5OKY3vgNbB09QAiQlzwhI3Tc5wtewHHLFhO5xvo4,16689
-vgi/_test_fixtures/table/splits.py,sha256=XvN-oAmLg3suVXSsCEuvVjDNSeVuvl-nc2gXWY5Q5sE,48010
+vgi/_test_fixtures/table/splits.py,sha256=jN1pt_dL0f9JAJXfMzIIO2-V6qH9vS4GF5q2xejjUl4,48333
 vgi/_test_fixtures/table/transaction_storage.py,sha256=cGSaM25cE6brwbUJ0-mchocX0UfwOb7uscCATKLYeME,6086
 vgi/_test_fixtures/table/tt_pushdown.py,sha256=jMELJ2HkG5-yIuYqKRYaPSouJxs3N-tiWRPthoPZ9ko,7420
 vgi/_test_fixtures/table/typed_probe.py,sha256=JjgLeHuhkXAVnCcezgqR7ayXMp3tTTixegftCwjKOSY,5122
@@ -123,7 +123,7 @@ vgi/_test_fixtures/writable/worker.py,sha256=ti-YFqVdYTMPIvDC8iwvaZ_CYUFTIf9xod2
 vgi/catalog/__init__.py,sha256=edc2wuvloEgY2WnnJNngV3xP0DN_p49SElYQanGzq4A,2629
 vgi/catalog/_descriptor_spec.py,sha256=fyeWOC3eN04UXDAOnxFeOS1VwGBWoBCMOzZfRYKPFH4,9773
 vgi/catalog/attach_option.py,sha256=4cgJZjTVCnFv8TrLYnfjy-XZLAx5ux4UisJRcBeXkfo,5735
-vgi/catalog/catalog_interface.py,sha256=ucv4QvMwqU7xdWtSBOLn2bXo0VhyXOQCyN51zL0ILjU,157262
+vgi/catalog/catalog_interface.py,sha256=-JdVTfkTOC2RYickktUiESF5UCuLVnPnLCUBQrTaN_4,158452
 vgi/catalog/descriptors.py,sha256=JvI0mqDy6DGmtnhZw-U2Z9ZROMjz4KejxGrw7XmNnGw,49362
 vgi/catalog/duckdb_statistics.py,sha256=Bzw4UGO-zdKrontaCcRV2palO8G3XLCWh-AhZozvQI8,16894
 vgi/catalog/secret_type.py,sha256=1DoEMyQun9K5z53817rUuhG1YpHSEAsazEmubkRLVL0,3745
@@ -139,7 +139,7 @@ vgi/client/cli_table.py,sha256=H1xrKZF5yYmSe55YSrLWXGIXPmuGuzQcncEBvqL43zs,26498
 vgi/client/cli_transaction.py,sha256=TDW0ZrBy8XM_eYSkYk5ewEsnozfzDJOlopd66Fm7OOE,3142
 vgi/client/cli_utils.py,sha256=_JoNiVu12NV4Pg2b1wtWFt0L0uIWJ9p9mBIfIu-QIuM,17387
 vgi/client/cli_view.py,sha256=0qhxoS91-zFMjq8V_Fb0PNW-YDxERu8-zPpxSFEE-G8,8490
-vgi/client/client.py,sha256=ocboi5d_h4KT1RXMEWMXAWYKSC5OH9IsJ4ah6z6C1JI,163751
+vgi/client/client.py,sha256=cvBQHzM2Z2lamC-fkBAvZD4HuRxB03aUkEKFVo_E9GU,167312
 vgi/client/errors.py,sha256=cSN65cyQMk3crwZCz1IdPErK3mtlEkTNpu-nuQUDOQE,1633
 vgi/http/__init__.py,sha256=hlOwOVcoZqmEKVGk7ganOdr5ryRSpEhLBF9sRD7BkYc,608
 vgi/http/demo_storage.py,sha256=830s-H61thozC3eEuqdnwCpYFfvoJRq9vjyuXa5OURo,8769
@@ -152,8 +152,8 @@ vgi/transactor/_duckdb_compat.py,sha256=sXVZ9JLKAQyGR1BjWczSwdQEavtr-TcZPoVZZnTr
 vgi/transactor/client.py,sha256=7DTeMksogsw6ANjQjGOPpKYrV76rg4_kGjktMJf54jg,4486
 vgi/transactor/protocol.py,sha256=v59IvrKnuvwOXvn_HEcBAbCzDVcx0akgKN0R1mChXg0,5034
 vgi/transactor/server.py,sha256=nzsZQxJZdgappdYX8okrFIrjFtUbdM_Hhskl1fZ2nDY,32553
-vgi_python-0.35.0.dist-info/METADATA,sha256=lNsV-ESefoEMsTf3l8hEhgiqthtqmbRBHFHRoGDY7VE,25617
-vgi_python-0.35.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
-vgi_python-0.35.0.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
-vgi_python-0.35.0.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
-vgi_python-0.35.0.dist-info/RECORD,,
+vgi_python-0.36.0.dist-info/METADATA,sha256=AURP4Y2G9g1qjUinaOSYm_owju8eSRHibiK_8Z0B7p8,25617
+vgi_python-0.36.0.dist-info/WHEEL,sha256=THafob7ofN-NsuMN7Mg4qZyHaQI7KkD-QlcQatYhXPo,87
+vgi_python-0.36.0.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
+vgi_python-0.36.0.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
+vgi_python-0.36.0.dist-info/RECORD,,