vgi-python 0.35.0__tar.gz → 0.36.1__tar.gz

{vgi_python-0.35.0 → vgi_python-0.36.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.35.0
+Version: 0.36.1
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.35.0 → vgi_python-0.36.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.35.0"
+version = "0.36.1"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [
@@ -223,7 +223,12 @@ warn_unused_ignores = false
 python-version = "3.13"
 
 [tool.pytest.ini_options]
-addopts = "--ruff"
+# worksteal: xdist's default "load" hands each worker a contiguous slice of the
+# collection up front, so tests/catalog/test_integration.py (30 tests that each
+# start a fresh catalog worker process, ~0.7 s apiece) all landed on one xdist
+# worker, which then ran for 20 s after the rest had finished. worksteal lets
+# idle workers take queued tests from busy ones. Ignored without -n.
+addopts = "--ruff --dist worksteal"
 testpaths = ["tests"]
 
 [tool.coverage.run]

{vgi_python-0.35.0 → vgi_python-0.36.1}/tests/client/test_stop_force.py

@@ -26,8 +26,11 @@ import pytest
 from vgi.arguments import Arguments
 from vgi.client.client import Client, _default_pool
 
-# Long enough that a graceful teardown would have to wait on it if the worker were stuck.
-_SLEEP_MS = 5_000
+# How long the worker sleeps before each row. The first row alone costs this
+# much, so it is kept short; the forced stop is held to half of it (see
+# test_force_returns_promptly_mid_stream), which a teardown that waited out the
+# sleep would miss by a wide margin, and a kill (milliseconds) clears.
+_SLEEP_MS = 2_000
 
 
 def _started_client(**kwargs: Any) -> Client:
@@ -65,10 +68,13 @@ def test_force_returns_promptly_mid_stream(tmp_path: Any) -> None:
     )
     next(gen)  # worker is now mid-stream, sleeping between batches
 
+    # The client's reader thread asks for the next row as soon as it has the
+    # first, so the worker is now inside that row's sleep. A stop that waited
+    # it out would take nearly the whole sleep; hold the kill to half of it.
     started = time.monotonic()
     client.stop(force=True)
     elapsed = time.monotonic() - started
-    assert elapsed < _SLEEP_MS / 1000.0, f"forced stop waited {elapsed:.2f}s on a sleeping worker"
+    assert elapsed < _SLEEP_MS / 2000.0, f"forced stop waited {elapsed:.2f}s on a sleeping worker"
 
 
 def test_graceful_stop_is_unchanged() -> None:

{vgi_python-0.35.0 → vgi_python-0.36.1}/tests/conftest.py

@@ -2,19 +2,29 @@
 
 """Shared fixtures for VGI tests."""
 
+import contextlib
 import logging
 import os
 import shlex
 import shutil
 import subprocess
 import sys
+import threading
 import uuid
 from typing import Any
 
-import pyarrow as pa
-import pytest
+# pyarrow imports numpy, and numpy's OpenBLAS starts one thread per core at
+# import and busy-waits them: about 2.7 CPU-seconds per process on a 48-core
+# machine. No test does linear algebra. This module is loaded before xdist
+# starts its workers, so every process of the session inherits the setting:
+# xdist workers, fixture and example workers, HTTP servers, CLI runs. On
+# 48 cores it cut the suite's CPU time from 363 s to 208 s.
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
 
-from vgi import schema
+import pyarrow as pa  # noqa: E402
+import pytest  # noqa: E402
+
+from vgi import schema  # noqa: E402
 
 # =============================================================================
 # Fixture-worker transport
@@ -66,6 +76,38 @@ CATALOG_WORKER = _fixture_command(sys.executable, "-m", "vgi._test_fixtures.cata
 SUBPROCESS_FIXTURE_WORKER = "vgi-fixture-worker"
 
 
+def pytest_configure(config: pytest.Config) -> None:
+    """Start the shared fixture worker while the test processes collect.
+
+    Otherwise the first tests to reach it all wait for its start (about a
+    second of imports) and then for the first function listings, which build
+    and encode every function's metadata before it is cached; each costs
+    the one GIL-bound worker 0.2-0.3 s, and the early tests run them all at
+    once. Collection takes several seconds, so warming the worker in the
+    background here costs nothing. Only the controlling process does it.
+    """
+    if hasattr(config, "workerinput") or TEST_TRANSPORT != "launch":
+        return
+    threading.Thread(target=_warm_fixture_worker, name="warm-fixture-worker", daemon=True).start()
+
+
+def _warm_fixture_worker() -> None:
+    """Launch the fixture worker and fill its function-listing caches."""
+    from vgi.catalog import SchemaObjectType
+    from vgi.client import Client
+
+    # Best effort: a worker that cannot start fails the tests that use it, with
+    # the real error.
+    with contextlib.suppress(Exception), Client(FIXTURE_WORKER) as client:
+        attach = client.catalog_attach(name="example", options={}, data_version_spec=None, implementation_version=None)
+        for kind in (
+            SchemaObjectType.TABLE_FUNCTION,
+            SchemaObjectType.SCALAR_FUNCTION,
+            SchemaObjectType.AGGREGATE_FUNCTION,
+        ):
+            list(client.schema_contents(attach_opaque_data=attach.attach_opaque_data, path=["main"], type=kind))
+
+
 def pytest_unconfigure(config: pytest.Config) -> None:
     """Stop this session's launched workers rather than leaving them to idle out.
 
@@ -132,6 +174,24 @@ def empty_batch_from_schema(schema: pa.Schema) -> pa.RecordBatch:
     return pa.RecordBatch.from_pydict({field.name: [] for field in schema}, schema=schema)
 
 
+def record_opened_connections(client: Any) -> list[int]:
+    """Record the index of each worker connection a started ``client`` opens from now on.
+
+    The primary connection (index 0) is opened when the client starts, so call
+    this afterwards: the list then holds only secondary connections, which a
+    call opens for fan-out. Only records; each connection is opened for real.
+    """
+    opened: list[int] = []
+    spawn = client._spawn_worker
+
+    def recording_spawn(worker_index: int) -> Any:
+        opened.append(worker_index)
+        return spawn(worker_index)
+
+    client._spawn_worker = recording_spawn
+    return opened
+
+
 # =============================================================================
 # Fixtures
 # =============================================================================

{vgi_python-0.35.0 → vgi_python-0.36.1}/tests/scalar/test_client.py

@@ -4,12 +4,13 @@
 
 from __future__ import annotations
 
+import os
 from typing import cast
 
 import pyarrow as pa
 import pytest
 
-from tests.conftest import assert_total_rows
+from tests.conftest import assert_total_rows, record_opened_connections
 from vgi import schema
 from vgi.arguments import Arguments
 from vgi.client import Client
@@ -388,6 +389,7 @@ class TestScalarFunctionParallel:
         batches = [pa.RecordBatch.from_pydict({"x": list(range(i * 100, (i + 1) * 100))}, schema=s) for i in range(10)]
 
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             outputs = list(
                 client.scalar_function(
                     function_name="double",
@@ -397,6 +399,11 @@ class TestScalarFunctionParallel:
                 )
             )
 
+        # Ten batches dealt round-robin reach every connection the client
+        # allows: four, or one per core on a smaller machine (GitHub's macOS
+        # runners have three).
+        assert opened == list(range(1, min(4, os.cpu_count() or 1)))
+
         # Should get all 1000 rows back
         assert_total_rows(outputs, 1000)
 
@@ -675,9 +682,12 @@ class TestRandomIntFunction:
 class TestScalarMultiWorkerEdgeCases:
     """Tests for edge cases with multiple workers for scalar functions.
 
-    These tests expose timeout/hang bugs when:
+    These tests exposed timeout/hang bugs when:
     - Processing parquet with one batch of zero rows
-    - Additional workers spawned but don't receive batches
+    - Additional workers were spawned but received no batches
+
+    The client now opens a secondary connection only when a batch is dealt to
+    it, so no connection sits idle. The tests check that too.
     """
 
     def test_zero_row_batch_single_worker(self, fixture_worker: str) -> None:
@@ -704,8 +714,9 @@ class TestScalarMultiWorkerEdgeCases:
         s = schema(x=pa.int64())
         zero_row_batch = pa.RecordBatch.from_pydict({"x": []}, schema=s)
 
-        # Force 4 workers even though there's only one batch with zero rows
+        # Allow 4 workers even though there's only one batch with zero rows
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             outputs = list(
                 client.scalar_function(
                     function_name="double",
@@ -715,17 +726,19 @@ class TestScalarMultiWorkerEdgeCases:
                 )
             )
 
-        # Should complete without hanging
+        # Should complete without hanging, on the primary alone
         assert len(outputs) == 1
         assert outputs[0].num_rows == 0
+        assert opened == []
 
     def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
         """Single normal batch with max_workers=4 should complete without hanging."""
         s = schema(x=pa.int64())
         single_batch = pa.RecordBatch.from_pydict({"x": [1, 2, 3]}, schema=s)
 
-        # Force 4 workers even though there's only 1 batch
+        # Allow 4 workers even though there's only 1 batch
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             outputs = list(
                 client.scalar_function(
                     function_name="double",
@@ -737,6 +750,7 @@ class TestScalarMultiWorkerEdgeCases:
 
         # Should complete without hanging and return correct data
         assert_total_rows(outputs, 3)
+        assert opened == []
 
     def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
         """2 batches with max_workers=4 should complete without hanging."""
@@ -744,8 +758,9 @@ class TestScalarMultiWorkerEdgeCases:
         batch1 = pa.RecordBatch.from_pydict({"x": [1, 2]}, schema=s)
         batch2 = pa.RecordBatch.from_pydict({"x": [3, 4, 5]}, schema=s)
 
-        # Force 4 workers even though there are only 2 batches
+        # Allow 4 workers even though there are only 2 batches
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             outputs = list(
                 client.scalar_function(
                     function_name="double",
@@ -757,3 +772,5 @@ class TestScalarMultiWorkerEdgeCases:
 
         # Should complete without hanging and return correct data (5 rows total)
         assert_total_rows(outputs, 5)
+        # The second batch opened the one secondary connection it needed.
+        assert opened == [1]

{vgi_python-0.35.0 → vgi_python-0.36.1}/tests/table_in_out/test_client.py

@@ -10,7 +10,13 @@ import time
 import pyarrow as pa
 import pytest
 
-from tests.conftest import SUBPROCESS_FIXTURE_WORKER, assert_single_result, assert_total_rows, make_schema
+from tests.conftest import (
+    SUBPROCESS_FIXTURE_WORKER,
+    assert_single_result,
+    assert_total_rows,
+    make_schema,
+    record_opened_connections,
+)
 from vgi.client import Client
 from vgi.client.client import ClientError
 
@@ -146,9 +152,12 @@ class TestEdgeCases:
 class TestMultiWorkerEdgeCases:
     """Tests for edge cases with multiple workers.
 
-    These tests expose timeout/hang bugs when:
+    These tests exposed timeout/hang bugs when:
     - Processing parquet with one batch of zero rows
-    - Additional workers spawned but don't receive batches
+    - Additional workers were spawned but received no batches
+
+    The client now opens a secondary connection only when a batch is dealt to
+    it, so no connection sits idle. The tests check that too.
     """
 
     def test_zero_row_batch_single_worker(self, fixture_worker: str) -> None:
@@ -174,8 +183,9 @@ class TestMultiWorkerEdgeCases:
         schema = make_schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
         zero_row_batch = pa.RecordBatch.from_pydict({"id": [], "value": []}, schema=schema)
 
-        # Force 4 workers even though there's only one batch with zero rows
+        # Allow 4 workers even though there's only one batch with zero rows
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             output_batches = list(
                 client.table_in_out_function(
                     function_name="echo",
@@ -184,17 +194,19 @@ class TestMultiWorkerEdgeCases:
                 )
             )
 
-        # Should complete without hanging
+        # Should complete without hanging, on the primary alone
         assert len(output_batches) >= 1
         assert_total_rows(output_batches, 0)
+        assert opened == []
 
     def test_single_batch_multiple_workers(self, fixture_worker: str) -> None:
         """Single normal batch with max_workers=4 should complete without hanging."""
         schema = make_schema([pa.field("id", pa.int64()), pa.field("value", pa.int64())])
         single_batch = pa.RecordBatch.from_pydict({"id": [1, 2, 3], "value": [10, 20, 30]}, schema=schema)
 
-        # Force 4 workers even though there's only 1 batch
+        # Allow 4 workers even though there's only 1 batch
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             output_batches = list(
                 client.table_in_out_function(
                     function_name="echo",
@@ -205,6 +217,7 @@ class TestMultiWorkerEdgeCases:
 
         # Should complete without hanging and return correct data
         assert_total_rows(output_batches, 3)
+        assert opened == []
 
     def test_fewer_batches_than_workers(self, fixture_worker: str) -> None:
         """2 batches with max_workers=4 should complete without hanging."""
@@ -212,8 +225,9 @@ class TestMultiWorkerEdgeCases:
         batch1 = pa.RecordBatch.from_pydict({"id": [1, 2], "value": [10, 20]}, schema=schema)
         batch2 = pa.RecordBatch.from_pydict({"id": [3, 4, 5], "value": [30, 40, 50]}, schema=schema)
 
-        # Force 4 workers even though there are only 2 batches
+        # Allow 4 workers even though there are only 2 batches
         with Client(fixture_worker, worker_limit=4) as client:
+            opened = record_opened_connections(client)
             output_batches = list(
                 client.table_in_out_function(
                     function_name="echo",
@@ -224,6 +238,8 @@ class TestMultiWorkerEdgeCases:
 
         # Should complete without hanging and return correct data (5 rows total)
         assert_total_rows(output_batches, 5)
+        # The second batch opened the one secondary connection it needed.
+        assert opened == [1]
 
 
 class TestWorkerStderrCapture:

vgi_python-0.36.1/tests/test_dynamic_filter_http_turns.py

@@ -0,0 +1,199 @@
+# Copyright 2026 Query Farm LLC - https://query.farm
+
+"""Dynamic-filter state across HTTP turns: bounded replay, exact reconstruction.
+
+An HTTP producer rebuilds its filters from the call and cursor tokens on every
+turn. The cursor used to carry every tick's delta and replay all of them, each
+re-validated through the embedded DuckDB evaluator, so turn ``k`` did ``k``
+parses: a 100-batch Top-N scan spent ~20 s there. These tests drive the real
+``dynamic_filter_echo`` fixture through the real HTTP stack, playing the client
+(the requests are hand-built; every response is the worker's), and pin both
+halves of the fix: a turn's parse work stays constant as ticks accumulate, and
+the state rebuilt from the compacted history is the state the worker had.
+"""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import json
+from collections.abc import Iterator
+from typing import Any
+
+import pyarrow as pa
+import pytest
+from vgi_rpc.rpc import RpcServer
+
+from vgi import filter_v2
+from vgi._test_fixtures.worker import ExampleWorker
+from vgi.arguments import Arguments
+from vgi.invocation import FunctionType
+from vgi.protocol import BindRequest, InitRequest, VgiProtocol
+
+_KEY = b"\x5a" * 32
+_METADATA: dict[bytes | str, bytes | str] = {
+    b"vgi_filter_encoding": b"vgi.filters.v2",
+    b"vgi_filter_version": b"2",
+    b"vgi_evaluation_context": b"vgi.none.v1",
+}
+
+
+def _document_batch(document: dict[str, object], *values: int) -> pa.RecordBatch:
+    fields: list[pa.Field[Any]] = [pa.field("filter_spec", pa.string(), nullable=False)]
+    arrays: list[pa.Array[Any]] = [pa.array([json.dumps(document, separators=(",", ":"))])]
+    for index, value in enumerate(values):
+        fields.append(pa.field(f"value_{index}", pa.int64()))
+        arrays.append(pa.array([value], type=pa.int64()))
+    return pa.RecordBatch.from_arrays(arrays, schema=pa.schema(fields, metadata=_METADATA))
+
+
+def _empty_snapshot() -> pa.RecordBatch:
+    return _document_batch(
+        {"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "snapshot", "predicates": []}
+    )
+
+
+def _upsert(predicate_id: str, revision: int, op: str, value_ref: int) -> dict[str, object]:
+    return {
+        "operation": "upsert",
+        "id": predicate_id,
+        "revision": revision,
+        "mode": "advisory",
+        "source": "top_n",
+        "expression": {
+            "node": "comparison",
+            "op": op,
+            "left": {"node": "column_ref", "column_index": 0, "column_name": "n"},
+            "right": {"node": "literal", "value_ref": value_ref},
+        },
+    }
+
+
+def _remove(predicate_id: str, revision: int) -> dict[str, object]:
+    return {"operation": "remove", "id": predicate_id, "revision": revision}
+
+
+def _tick(updates: list[dict[str, object]], *values: int) -> pa.KeyValueMetadata:
+    """Tick metadata carrying one delta, framed the way the C++ client frames it."""
+    batch = _document_batch(
+        {"encoding": "vgi.filters.v2", "semantics": "vgi.duckdb.standard.v1", "kind": "delta", "updates": updates},
+        *values,
+    )
+    sink = pa.BufferOutputStream()
+    with pa.ipc.new_stream(sink, batch.schema) as writer:
+        writer.write_batch(batch)
+    return pa.KeyValueMetadata({b"vgi_pushdown_filters": base64.b64encode(sink.getvalue().to_pybytes())})
+
+
+@contextlib.contextmanager
+def _echo_stream(count: int, batch_size: int) -> Iterator[Any]:
+    """Open a ``dynamic_filter_echo`` scan over the in-process HTTP stack."""
+    from vgi_rpc.http import http_connect, make_sync_client
+
+    worker = ExampleWorker(quiet=True)
+    worker._signing_key = _KEY
+    client = make_sync_client(RpcServer(VgiProtocol, worker, enable_describe=False), token_key=_KEY)
+    bind = BindRequest(
+        function_name="dynamic_filter_echo",
+        arguments=Arguments(positional=(pa.scalar(count),), named={"batch_size": pa.scalar(batch_size)}),
+        function_type=FunctionType.TABLE,
+        input_schema=None,
+    )
+    with http_connect(VgiProtocol, client=client, compression_level=None) as proxy:  # type: ignore[type-abstract]
+        resp = proxy.bind(request=bind)
+        stream = proxy.init(
+            request=InitRequest(
+                bind_call=bind,
+                output_schema=resp.output_schema,
+                bind_opaque_data=resp.opaque_data,
+                pushdown_filters=_empty_snapshot(),
+            )
+        )
+        try:
+            yield stream
+        finally:
+            stream.close()
+
+
+@pytest.fixture
+def parse_count(monkeypatch: pytest.MonkeyPatch) -> list[int]:
+    """Count delta documents the worker parses (a spy: it records, never answers)."""
+    calls = [0]
+    original = filter_v2._Parser.parse_delta
+
+    def counting(self: Any) -> Any:
+        calls[0] += 1
+        return original(self)
+
+    monkeypatch.setattr(filter_v2._Parser, "parse_delta", counting)
+    return calls
+
+
+def test_a_turn_parses_a_bounded_number_of_deltas(parse_count: list[int]) -> None:
+    """A tightening Top-N bound sends a delta every tick; no turn replays them all.
+
+    Rows descend from 999 in batches of 10 and each tick narrows ``n < bound``,
+    so every tick's delta changes the live predicate -- the case the C++ client
+    cannot skip. Before compaction turn ``k`` parsed ``k`` deltas.
+    """
+    ticks = 60
+    with _echo_stream(count=1000, batch_size=10) as stream:
+        first = stream.tick()  # the init turn's batch; metadata rides continuations only
+        assert first.batch.num_rows == 10
+        per_turn: list[int] = []
+        for revision in range(1, ticks + 1):
+            bound = 1000 - 10 * revision  # rows of this tick are [bound - 10, bound - 1]
+            before = parse_count[0]
+            batch = stream.tick(_tick([_upsert("top_n:0", revision, "lt", 0)], bound)).batch
+            per_turn.append(parse_count[0] - before)
+            assert batch.column("n").to_pylist() == list(range(bound - 1, bound - 11, -1))
+            assert batch.column("pushed_filters")[0].as_py().count("ConstantFilter(n < ") == 1
+            assert f"ConstantFilter(n < {bound})" in batch.column("pushed_filters")[0].as_py()
+
+    # One parse for the new delta, one to replay the single delta the cursor keeps.
+    assert max(per_turn) <= 2, per_turn
+
+
+def test_a_resent_revision_is_stale_and_adds_no_replay(parse_count: list[int]) -> None:
+    """Resending a revision with another value is a stale no-op: nothing applies, nothing accumulates."""
+    with _echo_stream(count=1000, batch_size=10) as stream:
+        stream.tick()
+        stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000))
+        before = parse_count[0]
+        for _ in range(20):
+            batch = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch
+            assert batch.num_rows == 10  # n < 5 would have emptied it
+            assert "ConstantFilter(n < 100000)" in batch.column("pushed_filters")[0].as_py()
+        assert parse_count[0] - before <= 2 * 20
+
+
+def test_rebuilt_state_keeps_the_order_the_worker_had() -> None:
+    """A removed-then-re-added predicate keeps its position across a turn boundary.
+
+    Deltas: {a:1, b:1} -> [a, b]; {remove a:2, b:1} -> [b]; {a:3, b:1} -> [b, a].
+    The compacted history is the deltas that first carried a:3 and b:1 -- the
+    third and the first -- and replaying those alone yields [a, b]. The turn
+    after the third delta has to show the order the third turn itself had.
+    """
+    with _echo_stream(count=1000, batch_size=10) as stream:
+        stream.tick()
+        stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 100000, 5))
+        stream.tick(_tick([_remove("top_n:0", 2), _upsert("top_n:1", 1, "gt", 0)], 5))
+        applied = stream.tick(_tick([_upsert("top_n:0", 3, "lt", 0), _upsert("top_n:1", 1, "gt", 1)], 99999, 5)).batch
+        rebuilt = stream.tick().batch  # no delta: filters come purely from the tokens
+        assert applied.num_rows == rebuilt.num_rows == 10
+        live = applied.column("pushed_filters")[0].as_py()
+        assert live.index("ConstantFilter(n > 5)") < live.index("ConstantFilter(n < 99999)"), live
+        assert rebuilt.column("pushed_filters")[0].as_py() == live
+
+
+def test_a_tombstone_survives_compaction() -> None:
+    """A removal stays in force after its upsert is compacted away: a stale upsert cannot resurrect it."""
+    with _echo_stream(count=1000, batch_size=10) as stream:
+        stream.tick()
+        stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 100000))
+        stream.tick(_tick([_remove("top_n:0", 2)]))
+        stream.tick()  # a turn rebuilt from the compacted tokens
+        stale = stream.tick(_tick([_upsert("top_n:0", 1, "lt", 0)], 5)).batch
+        assert stale.column("pushed_filters")[0].as_py() == "(none)"
+        assert stale.num_rows == 10

{vgi_python-0.35.0 → vgi_python-0.36.1}/uv.lock

@@ -2580,7 +2580,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.35.0"
+version = "0.36.1"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

{vgi_python-0.35.0 → vgi_python-0.36.1}/vgi/_test_fixtures/table/filters.py

@@ -1003,7 +1003,10 @@ class FilterEchoPartitionedFunction(TableFunctionGenerator[_FilterEchoPartitione
             end_idx = min(start_idx + chunk, params.args.count)
             work_items.append(struct.pack(">QQ", start_idx, end_idx))
         params.storage.queue_push(work_items)
-        return GlobalInitResponse()
+        # No more readers than work items. Left at the default (unbounded) the
+        # client opens one stream per DuckDB thread -- 48 on a 48-core host for at
+        # most MAX_PARTITIONS items, every extra one an init plus an empty drain.
+        return GlobalInitResponse(max_workers=max(1, len(work_items)))
 
     @classmethod
     def initial_state(cls, params: ProcessParams[_FilterEchoPartitionedArgs]) -> _FilterEchoPartitionedState:

{vgi_python-0.35.0 → vgi_python-0.36.1}/vgi/_test_fixtures/table/sequence.py

@@ -516,7 +516,10 @@ class PartitionedSequenceFunction(
 
         # Always enqueue (even if empty) to register the invocation
         params.storage.queue_push(work_items)
-        return GlobalInitResponse()
+        # No more readers than work items. Left at the default (unbounded) the
+        # client opens one stream per DuckDB thread -- 48 on a 48-core host for at
+        # most MAX_PARTITIONS items, every extra one an init plus an empty drain.
+        return GlobalInitResponse(max_workers=max(1, len(work_items)))
 
     @classmethod
     def initial_state(cls, params: ProcessParams[PartitionedSequenceFunctionArguments]) -> PartitionedSequenceState:

{vgi_python-0.35.0 → vgi_python-0.36.1}/vgi/_test_fixtures/table/splits.py

@@ -18,8 +18,8 @@ ways it goes right:
   shape that silently truncates a scan if a reader treats an empty split as EOS.
 * ``split_skewed(n)`` — one split ~100x the others, so greedy claiming can be
   told apart from static assignment.
-* ``split_many(n, splits := 1000)`` — far more splits than reader threads, which
-  forces sequential re-init on a reused connection.
+* ``split_many(n, splits)`` — far more splits than reader threads (pass them), which
+  forces sequential re-init on a reused connection. ``splits <= 0`` means 1000.
 * ``split_echo_filters(n)`` — reports the pushdown it saw *per split*, so a
   pushdown regression is visible without inferring it from row counts.
 """
@@ -35,6 +35,7 @@ from vgi_rpc.rpc import OutputCollector
 
 from vgi.arguments import Arg
 from vgi.cache_control import CacheControl
+from vgi.filter_v2 import replaying_accepted_state
 from vgi.metadata import FunctionExample, PartitionKind
 from vgi.protocol import PlanResponse, ScanSplit, VgiOutputCollector
 from vgi.schema_utils import partition_field, schema
@@ -321,7 +322,9 @@ class SplitManyFunction(_SplitBase):
 
     @classmethod
     def _split_count(cls, params: BindParams[SplitSequenceArgs]) -> int:
-        return max(params.args.splits, 1000)
+        # Honour the caller's count, as every other SDK's twin does; a floor here
+        # made a test that asks for 64 splits pay for 1000 HTTP round trips.
+        return params.args.splits if params.args.splits > 0 else 1000
 
 
 @dataclass(frozen=True)
@@ -1055,11 +1058,15 @@ class SplitDynamicFilterFunction(TableFunctionGenerator[SplitSequenceArgs, Split
             init = params.init_call
             merged = None
             if init is not None and init.pushdown_filters is not None:
-                merged = cls.pushdown_filters(
-                    init.pushdown_filters,
-                    join_keys=init.join_keys,
-                    output_schema=init.output_schema,
-                )
+                # The framework validated these at init; re-binding every
+                # predicate through the evaluator on every tick is what made
+                # this fixture cost ~10 ms a batch.
+                with replaying_accepted_state():
+                    merged = cls.pushdown_filters(
+                        init.pushdown_filters,
+                        join_keys=init.join_keys,
+                        output_schema=init.output_schema,
+                    )
             if merged is None:
                 merged = params.current_pushdown_filters
             filter_str = _render_filters_canonical(merged)

{vgi_python-0.35.0 → vgi_python-0.36.1}/vgi/aggregate_function.py

@@ -518,7 +518,14 @@ class AggregateFunction[TState: StreamStateCodec](vgi.function.Function):
         """Compute the aggregate value for one output row.
 
         Args:
-            rid: Partition-local row index being filled.
+            rid: Identifies the output row being filled; do not use it to
+                index the partition. Over the per-row ``aggregate_window``
+                RPC it is DuckDB's own ``rid``: the row's index within the
+                current output chunk (``0 <= rid < 2048``), not its row in
+                the partition. When the default :meth:`window_batch` calls
+                this, it passes the partition row index instead. Either way
+                ``subframes`` are partition-relative: use them to find the
+                rows to aggregate.
             subframes: Frame ranges ``[(begin, end), ...]`` — 1 for the default
                 frame, 3 when ``EXCLUDE`` produces multiple subframes.
             partition: The cached partition data.