vgi-python 0.43.0__tar.gz → 0.43.1__tar.gz

{vgi_python-0.43.0 → vgi_python-0.43.1}/CLAUDE.md

@@ -96,9 +96,13 @@ e.g. to pass an unusual filter to `unittest`):
 ```bash
 cd ~/Development/vgi
 VGI_TEST_WORKER="uv run --project ~/Development/vgi-python vgi-fixture-worker" \
-    ./build/release/test/unittest "test/sql/integration/*"
+    ./build/release/test/unittest --test-config test/configs/no_error_skip.json "test/sql/integration/*"
 ```
 
+Always pass `--test-config test/configs/no_error_skip.json` to `unittest`: without
+it, DuckDB's runner reports any error mentioning "HTTP" or "Unable to connect" as
+a SKIP and exits 0.
+
 **HTTP transport** (worker as an HTTP server):
 ```bash
 cd ~/Development/vgi
@@ -142,7 +146,7 @@ with elapsed milliseconds. This helps identify slow queries and bottlenecks.
 # Timing output goes to stderr, grep for the bracket-prefixed lines:
 cd ~/Development/vgi
 VGI_TEST_WORKER="uv run --project ~/Development/vgi-python vgi-fixture-worker" \
-    ./build/release/test/unittest "test/sql/integration/table/writable_table*" \
+    ./build/release/test/unittest --test-config test/configs/no_error_skip.json "test/sql/integration/table/writable_table*" \
     2>&1 | grep "^\[stmt\|^\[query" | sort -t']' -k2 -rn
 ```
 
@@ -166,7 +170,7 @@ uv run coverage run -m pytest --no-cov -n auto
 # 3. Run DuckDB subprocess integration tests with coverage
 cd ~/Development/vgi
 VGI_TEST_WORKER="/tmp/vgi-coverage-worker.sh" \
-    ./build/release/test/unittest "test/sql/integration/*"
+    ./build/release/test/unittest --test-config test/configs/no_error_skip.json "test/sql/integration/*"
 
 # 4. Run DuckDB HTTP integration tests with coverage
 cd ~/Development/vgi

{vgi_python-0.43.0 → vgi_python-0.43.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.43.0
+Version: 0.43.1
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.43.0 → vgi_python-0.43.1}/ci/run-integration.sh

@@ -336,6 +336,13 @@ EOF
 "$HAYBARN_UNITTEST" "test/_warm.test" >/dev/null 2>&1 || echo "::warning::extension warm step did not fully succeed"
 rm -f "$STAGE/test/_warm.test"
 
+# DuckDB's sqllogictest runner, given no --test-config, turns any error whose
+# text contains "HTTP" or "Unable to connect" into a SKIP and exits 0. Over the
+# http transport every worker error carries "HTTP", so real failures read as
+# skips. The extension ships a config that disables that; every run below uses it.
+TEST_CONFIG="$VGI_SRC/test/configs/no_error_skip.json"
+[ -f "$TEST_CONFIG" ] || { echo "::error::missing $TEST_CONFIG (VGI_REF predates it?)"; exit 1; }
+
 # Run the lane, streaming the native sqllogictest report (a progress line per
 # file + the final "All tests passed (.. N assertions ..)" summary). Out-of-scope
 # tests were dropped at staging; any failed assertion exits non-zero.
@@ -366,7 +373,7 @@ run_unittest() {
   # `set +e` rather than `|| true`: the latter runs before PIPESTATUS is read and
   # overwrites it with true's 0, silently swallowing every real test failure.
   set +e
-  "$HAYBARN_UNITTEST" "$@" 2>&1 | tee "$log"
+  "$HAYBARN_UNITTEST" --test-config "$TEST_CONFIG" "$@" 2>&1 | tee "$log"
   rc="${PIPESTATUS[0]}"
   set -e
   if grep -q 'due to a fatal error condition' "$log"; then

{vgi_python-0.43.0 → vgi_python-0.43.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.43.0"
+version = "0.43.1"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.43.0 → vgi_python-0.43.1}/scripts/run_all_tests.sh

@@ -204,11 +204,14 @@ run_integration_job() {
       return 2
       ;;
   esac
+  # --test-config: without it the runner turns any error mentioning "HTTP" or
+  # "Unable to connect" into a SKIP, so a failure can pass as a skip.
   # `~[.]` excludes Catch2-hidden tests, i.e. `*.test_slow` files. The
   # launcher keeps one Python worker warm while unittest advances serially,
   # avoiding both repeated startup and cross-test state contention.
   VGI_TEST_WORKER="$worker" \
-    timeout 600 ./build/release/test/unittest "test/sql/integration/*" "~[.]" >"$INT_LOG" 2>&1
+    timeout 600 ./build/release/test/unittest --test-config test/configs/no_error_skip.json \
+      "test/sql/integration/*" "~[.]" >"$INT_LOG" 2>&1
   local rc=$?
   summarize_integration "$INT_LOG" "$INT_SUMMARY" "$INT_FAILURES"
   echo "exit=$rc" >>"$INT_SUMMARY"

{vgi_python-0.43.0 → vgi_python-0.43.1}/tests/test_table_function_early_close.py

@@ -6,8 +6,15 @@
 thread. Before the fix, closing the generator early left those threads reading,
 and the ``Client.stop()`` that followed drained the same stream from the main
 thread: two readers on one pipe, and whichever missed the end of stream blocked
-forever (roughly one run in five). Each case here repeats the early close under
-a watchdog, then checks the pooled worker still serves a complete scan.
+forever (roughly one run in five). Each case here repeats the early close, each
+repetition under its own watchdog, then checks the pooled worker still serves a
+complete scan.
+
+The watchdog bounds one scan, not the loop: a hang never finishes, so any bound
+far above a healthy scan catches it, while a bound on the whole loop measures
+throughput. A 24-way scan respawns the workers the pool could not keep idle
+(``max_idle=8``), ~1.2 s idle and up to ~5.4 s with the host's CPUs
+oversubscribed twice over; 25 of those overran a 120 s whole-loop budget.
 """
 
 from __future__ import annotations
@@ -23,7 +30,8 @@ from vgi.arguments import Arguments
 from vgi.client.client import Client, _default_pool
 
 _ITERATIONS = 25
-_WATCHDOG_SECONDS = 120.0
+# Per scan. A healthy one takes ~1.5 s idle, ~6 s at worst under 2x CPU oversubscription.
+_WATCHDOG_SECONDS = 60.0
 
 
 def _run_with_watchdog(body: Callable[[], None]) -> None:
@@ -45,12 +53,11 @@ def _run_with_watchdog(body: Callable[[], None]) -> None:
 
 
 def _close_after_first_batch(function_name: str, arguments: Arguments) -> None:
-    for _ in range(_ITERATIONS):
-        with Client(SUBPROCESS_FIXTURE_WORKER, pool=_default_pool) as client:
-            scan = client.table_function(function_name=function_name, schema_path=["main"], arguments=arguments)
-            first = next(scan)
-            assert first.num_rows > 0
-            scan.close()
+    with Client(SUBPROCESS_FIXTURE_WORKER, pool=_default_pool) as client:
+        scan = client.table_function(function_name=function_name, schema_path=["main"], arguments=arguments)
+        first = next(scan)
+        assert first.num_rows > 0
+        scan.close()
 
 
 def _full_scan_still_works() -> None:
@@ -77,5 +84,32 @@ def _full_scan_still_works() -> None:
 )
 def test_early_close_does_not_hang(function_name: str, arguments: Arguments) -> None:
     """A scan closed after its first batch stops cleanly, and the worker pool stays usable."""
-    _run_with_watchdog(lambda: _close_after_first_batch(function_name, arguments))
+    for _ in range(_ITERATIONS):
+        _run_with_watchdog(lambda: _close_after_first_batch(function_name, arguments))
     _run_with_watchdog(_full_scan_still_works)
+
+
+def test_secondary_workers_stop_concurrently(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Secondary workers stop in parallel, and every one is stopped even when one fails.
+
+    Each stop can wait ~0.2 s on a process exit (its own, or a pool eviction's),
+    so stopping a 24-way scan's workers in turn spent ~4 s closing it.
+    """
+    client = Client(SUBPROCESS_FIXTURE_WORKER, pool=_default_pool)
+    workers = [object() for _ in range(8)]
+    stopped: list[object] = []
+    barrier = threading.Barrier(len(workers), timeout=10)
+
+    def fake_stop(worker: object, *, force: bool = False) -> int:
+        barrier.wait()  # breaks (and fails the test) unless all stops run at once
+        stopped.append(worker)
+        if worker is workers[3]:
+            raise RuntimeError("stop failed")
+        return 0
+
+    monkeypatch.setattr(client, "_stop_worker", fake_stop)
+    client._additional_workers = workers  # type: ignore[assignment]
+    with pytest.raises(RuntimeError, match="stop failed"):
+        client._close_secondary_workers()
+    assert sorted(map(id, stopped)) == sorted(map(id, workers))
+    assert client._additional_workers == []

{vgi_python-0.43.0 → vgi_python-0.43.1}/uv.lock

@@ -2670,7 +2670,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.43.0"
+version = "0.43.1"
 source = { editable = "." }
 dependencies = [
     { name = "click" },

{vgi_python-0.43.0 → vgi_python-0.43.1}/vgi/client/client.py

@@ -90,6 +90,7 @@ import subprocess
 import sys
 import threading
 from collections.abc import Callable, Generator, Iterator, Sequence
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import AbstractContextManager
 from dataclasses import dataclass, field, replace
 from queue import Queue
@@ -1460,13 +1461,29 @@ class Client(CatalogClientMixin, AggregateClientMixin):
     def _close_secondary_workers(self, *, force: bool = False) -> None:
         """Close and stop all secondary (additional) workers.
 
+        The workers are stopped concurrently. Each stop can wait on a process
+        exit: a direct worker's own, or, for a pooled worker, that of the idle
+        worker the pool evicts to make room once it is past ``max_idle``. A
+        graceful exit is mostly interpreter teardown (~0.2 s), so stopping a
+        24-way scan one worker at a time spent ~4 s here.
+
         Args:
             force: Kill direct subprocess workers rather than closing them gracefully
                 (see ``_stop_worker``).
+
+        Raises:
+            Exception: The first error a stop raised, once every worker has been
+                stopped.
         """
-        for worker in self._additional_workers:
-            self._stop_worker(worker, force=force)
-        self._additional_workers = []
+        workers, self._additional_workers = self._additional_workers, []
+        if len(workers) <= 1:
+            for worker in workers:
+                self._stop_worker(worker, force=force)
+            return
+        with ThreadPoolExecutor(max_workers=len(workers), thread_name_prefix="vgi-stop-worker") as executor:
+            futures = [executor.submit(self._stop_worker, worker, force=force) for worker in workers]
+        for future in futures:
+            future.result()
 
     def _join_threads(self, threads: list[threading.Thread]) -> None:
         """Wait for all threads to complete with timeout.