vgi-python 0.22.0__tar.gz → 0.23.0__tar.gz

{vgi_python-0.22.0 → vgi_python-0.23.0}/.github/workflows/integration.yml

@@ -33,6 +33,13 @@ env:
   # The Haybarn release providing the prebuilt haybarn-unittest binary. Must be
   # ABI-compatible with the community-published vgi extension.
   HAYBARN_RELEASE: haybarn-v1.5.4-rc3
+  # CI is not a user: its ATTACHes should not land in Query.Farm's adoption data.
+  # The vgi extension fires a telemetry event per successful ATTACH, and this
+  # suite performs several hundred of them per lane, so opting out keeps the
+  # numbers honest. It also removes telemetry as a variable in lane timings —
+  # the per-ATTACH event was added in Query-farm/vgi 71e2d51 (2026-07-12), and
+  # the stdio lane total stepped from ~690s to ~870s across that window.
+  QUERY_FARM_TELEMETRY_OPT_OUT: "1"
 
 jobs:
   integration:

{vgi_python-0.22.0 → vgi_python-0.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vgi-python
-Version: 0.22.0
+Version: 0.23.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python
@@ -162,7 +162,7 @@ Requires-Dist: httpx>=0.24
 Requires-Dist: platformdirs
 Requires-Dist: pyarrow
 Requires-Dist: typer>=0.9
-Requires-Dist: vgi-rpc>=0.28.2
+Requires-Dist: vgi-rpc>=0.28.3
 Provides-Extra: azure
 Requires-Dist: azure-identity>=1.16.0; extra == 'azure'
 Requires-Dist: pymssql>=2.3.0; extra == 'azure'

{vgi_python-0.22.0 → vgi_python-0.23.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.22.0"
+version = "0.23.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [
@@ -40,7 +40,7 @@ dependencies = [
     "pyarrow",
     "typer>=0.9",
     "platformdirs",
-    "vgi-rpc>=0.28.2",
+    "vgi-rpc>=0.28.3",
     "httpx>=0.24",
 ]
 

{vgi_python-0.22.0 → vgi_python-0.23.0}/tests/test_serve.py

@@ -206,6 +206,57 @@ class TestCreateApp:
 
         assert isinstance(app, falcon.App)
 
+    @staticmethod
+    def _health_proof_header(**env: str) -> str | None:
+        """Serve an app under ``env`` and read the proof capability header.
+
+        Args:
+            **env: ``VGI_PROXY_PROOF_*`` values to set for the duration.
+
+        Returns:
+            The ``VGI-Proxy-Proof-Required`` value, or ``None`` when absent.
+
+        """
+        import falcon.testing
+
+        with pytest.MonkeyPatch.context() as mp:
+            for key in ("VGI_PROXY_PROOF_MODE", "VGI_PROXY_PROOF_ORIGIN_ID", "VGI_PROXY_PROOF_SECRETS"):
+                mp.delenv(key, raising=False)
+            for key, value in env.items():
+                mp.setenv(key, value)
+            app = create_app(_SingleWorker, prefix="/vgi", describe=False)
+            resp = falcon.testing.TestClient(app).simulate_get("/vgi/health")
+        assert resp.status_code == 200
+        return {k.lower(): v for k, v in resp.headers.items()}.get("vgi-proxy-proof-required")
+
+    def test_advertises_proof_required_in_require_mode(self) -> None:
+        """A ``require``-mode worker tells proxies it actually enforces.
+
+        Derived from ``VGI_PROXY_PROOF_MODE`` rather than passed separately,
+        because that env var is also where the gate itself comes from — so the
+        advertisement cannot drift from the posture it describes.
+        """
+        value = self._health_proof_header(
+            VGI_PROXY_PROOF_MODE="require",
+            VGI_PROXY_PROOF_ORIGIN_ID="worker-a",
+            VGI_PROXY_PROOF_SECRETS="k:" + "11" * 32,
+        )
+        assert value == "true"
+
+    @pytest.mark.parametrize("mode", ["off", "allow"])
+    def test_does_not_advertise_below_require(self, mode: str) -> None:
+        """Only ``require`` advertises — ``allow`` never denies, so it must not.
+
+        A proxy reads this to confirm a rollout landed; an ``allow``-mode worker
+        that claimed to require would report the rollout complete while every
+        unproofed direct caller still sailed through.
+        """
+        env = {"VGI_PROXY_PROOF_MODE": mode}
+        if mode != "off":
+            env["VGI_PROXY_PROOF_ORIGIN_ID"] = "worker-a"
+            env["VGI_PROXY_PROOF_SECRETS"] = "k:" + "11" * 32
+        assert self._health_proof_header(**env) is None
+
     def test_signing_key_passed(self) -> None:
         """Explicit signing_key is accepted without warning."""
         import warnings

{vgi_python-0.22.0 → vgi_python-0.23.0}/uv.lock

@@ -2228,7 +2228,7 @@ requires-dist = [
 
 [[package]]
 name = "vgi-python"
-version = "0.22.0"
+version = "0.23.0"
 source = { editable = "." }
 dependencies = [
     { name = "click" },
@@ -2338,7 +2338,7 @@ requires-dist = [
     { name = "vgi-python", extras = ["test-fixtures"], marker = "extra == 'test-fixtures-writable'" },
     { name = "vgi-python", extras = ["test-fixtures", "test-fixtures-writable", "http", "oauth", "otel", "sentry", "azure"], marker = "extra == 'dev'" },
     { name = "vgi-python", extras = ["transactor"], marker = "extra == 'test-fixtures-writable'" },
-    { name = "vgi-rpc", specifier = ">=0.28.2" },
+    { name = "vgi-rpc", specifier = ">=0.28.3" },
     { name = "vgi-rpc", extras = ["conformance", "external", "http", "oauth", "otel", "sentry"], marker = "extra == 'dev'" },
     { name = "vgi-rpc", extras = ["http"], marker = "extra == 'http'" },
     { name = "vgi-rpc", extras = ["oauth"], marker = "extra == 'oauth'" },
@@ -2367,7 +2367,7 @@ docs = [
 
 [[package]]
 name = "vgi-rpc"
-version = "0.28.2"
+version = "0.28.3"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "docstring-parser" },
@@ -2375,9 +2375,9 @@ dependencies = [
     { name = "pywin32", marker = "sys_platform == 'win32'" },
     { name = "tzdata", marker = "sys_platform == 'win32'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/89/a0/00252e2d6986094c13bf35b7f7e1e95b0a3b0497c210b2c88cc72b82b3c7/vgi_rpc-0.28.2.tar.gz", hash = "sha256:4fcfcd5c42a0242aee9d1f40e16fab6559f6401feccf44fe50ad91ee473dc5f4", size = 905341, upload-time = "2026-07-27T13:56:23.482Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/a0/5a/1996a64ca9ecb0a030cee5a4b9944301f20d6269369b39a5d30f07bf71a9/vgi_rpc-0.28.3.tar.gz", hash = "sha256:772f535912926ce47c17c3eba33769ad226171277159c591f441169f642e14d6", size = 908548, upload-time = "2026-07-27T16:45:47.344Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/3c/1d/0a34b627d4f6d3bf83e61a5897fb32e5bf1543708380ee1a6eaeeb56a8d7/vgi_rpc-0.28.2-py3-none-any.whl", hash = "sha256:dc96d11b2ced0fa36e653ed475c653f4124748b2372da1866028da4975eb9b8f", size = 404897, upload-time = "2026-07-27T13:56:21.549Z" },
+    { url = "https://files.pythonhosted.org/packages/70/d1/db1b3aa48d1d0526bc30ad72e51ad8fd7f67f693310de8c143e707a87d0a/vgi_rpc-0.28.3-py3-none-any.whl", hash = "sha256:bf913b9b95ac4abde4e4ddad5b228c1d81bd748268773f9a2e5fa9626cc68bf7", size = 406388, upload-time = "2026-07-27T16:45:45.905Z" },
 ]
 
 [package.optional-dependencies]

{vgi_python-0.22.0 → vgi_python-0.23.0}/vgi/_test_fixtures/http_server.py

@@ -27,6 +27,7 @@ from vgi._test_fixtures.schema_reconcile.worker import SchemaReconcileWorker
 from vgi._test_fixtures.worker import ExampleWorker
 from vgi.logging_config import LogFormat, LogLevel, configure_worker_logging
 from vgi.meta_worker import MetaWorker
+from vgi.profiling import maybe_start_profile
 
 if TYPE_CHECKING:
     from vgi_rpc.external import UploadUrl
@@ -209,12 +210,15 @@ def main() -> None:
 
         env_debug = os.environ.get("VGI_WORKER_DEBUG", "").lower() in ("1", "true", "yes")
         effective_debug = debug or env_debug
+        # VGI_WORKER_LOG_* overrides are applied inside configure_worker_logging,
+        # so this server and `vgi-serve` are configurable the same way.
         effective_level = configure_worker_logging(
             debug=effective_debug,
             log_level=log_level,
             log_loggers=log_logger,
             log_format=log_format,
         )
+        maybe_start_profile()
 
         bucket = s3_bucket or os.environ.get("VGI_HTTP_S3_BUCKET")
         external_location = None

{vgi_python-0.22.0 → vgi_python-0.23.0}/vgi/argument_spec.py

@@ -18,6 +18,7 @@ import json
 import warnings
 from collections.abc import Sequence
 from dataclasses import dataclass
+from functools import cache
 from typing import Annotated, Any, get_args, get_origin, get_type_hints
 
 import pyarrow as pa
@@ -519,6 +520,10 @@ def extract_argument_specs(
     2. Type annotation with `PYTHON_TO_ARROW` mapping
     3. Default to pa.null() with warning for unknown types
 
+    Memoized per class — see :func:`_extract_argument_specs_cached`. A fresh
+    list is returned each call so a caller that mutates it cannot corrupt the
+    cache; the `ArgumentSpec`s inside are frozen dataclasses and safe to share.
+
     Args:
         cls: Function class with `Arg` descriptors.
 
@@ -527,6 +532,52 @@ def extract_argument_specs(
         then named).
 
     """
+    return list(_extract_argument_specs_cached(cls))
+
+
+@cache
+def _extract_argument_specs_cached(cls: type) -> tuple[ArgumentSpec, ...]:
+    """Memoized :func:`_extract_argument_specs`, keyed on the class.
+
+    A function class's argument specs are fixed once the class exists — they are
+    derived from its annotations, its ``FunctionArguments``, and the `Arg`
+    descriptors on its MRO, none of which change at runtime. Deriving them was
+    nonetheless repeated on **every** catalog listing and every overload match:
+    profiling one run of the upstream integration suite through the proxy showed
+    58,058 calls costing 15.8s, which drove 131,653 ``typing.get_type_hints``
+    calls, 2.4M ``_eval_type`` calls and 648k ``compile`` calls — roughly a
+    quarter of the worker's CPU. ``catalog_schema_contents_functions`` was 38ms
+    per call against 0.1-2ms for every other catalog method, entirely because of
+    this.
+
+    Unbounded (``functools.cache``) is the right size: the keys are function
+    classes, which the catalog already holds for the process's lifetime, so the
+    cache adds no retention that did not already exist. It is also thread safe,
+    which matters because workers serve requests on a pool.
+
+    Returns a tuple so the cached value cannot be mutated through a caller's
+    reference.
+
+    Args:
+        cls: Function class with `Arg` descriptors.
+
+    Returns:
+        The class's argument specs, sorted.
+    """
+    return tuple(_extract_argument_specs(cls))
+
+
+def _extract_argument_specs(
+    cls: type,
+) -> list[ArgumentSpec]:
+    """Derive a class's argument specs, without caching.
+
+    Args:
+        cls: Function class with `Arg` descriptors.
+
+    Returns:
+        List of `ArgumentSpec` objects, sorted by position.
+    """
     specs: list[ArgumentSpec] = []
     seen_names: set[str] = set()
 

{vgi_python-0.22.0 → vgi_python-0.23.0}/vgi/logging_config.py

@@ -11,6 +11,7 @@ VGI workers.
 from __future__ import annotations
 
 import logging
+import os
 import sys
 from enum import StrEnum
 
@@ -46,6 +47,68 @@ _KNOWN_LOGGERS: list[tuple[str, str, str]] = [
 ]
 
 
+def _env_overrides(
+    log_level: LogLevel,
+    log_loggers: list[str] | None,
+    log_format: LogFormat,
+) -> tuple[LogLevel, list[str] | None, LogFormat]:
+    """Apply ``VGI_WORKER_LOG_*`` environment overrides to the CLI options.
+
+    The access log (``vgi_rpc.access``) already records ``duration_ms``,
+    ``method`` and row/byte counts for every RPC, but reaching it required
+    passing ``--log-logger vgi_rpc.access --log-format json`` on the command
+    line. Anything that launches a worker without owning its argv — a container
+    image, a test harness spawning workers from a dozen call sites — could not
+    turn it on. These variables make the same configuration reachable the way
+    ``VGI_SIGNING_KEY`` and ``VGI_ENABLE_DESCRIBE`` already are.
+
+    Applied here rather than in a single CLI so that *every* worker entry point
+    honours them identically — ``vgi-serve`` and the fixture servers had
+    byte-identical logging blocks, and an override wired into only one of them
+    would be a trap.
+
+    Each variable wins when set, matching ``VGI_ENABLE_DESCRIBE``: Typer cannot
+    distinguish "user passed the default" from "user passed nothing", so
+    deferring to an explicit environment setting is the only unambiguous rule.
+
+    Args:
+        log_level: The ``--log-level`` value.
+        log_loggers: The ``--log-logger`` values, if any.
+        log_format: The ``--log-format`` value.
+
+    Returns:
+        The effective ``(level, loggers, format)``.
+
+    Raises:
+        SystemExit: If a variable names a level or format that does not exist.
+            A typo would otherwise silently leave logging as it was, which is
+            indistinguishable from the feature not working.
+    """
+    raw_level = os.environ.get("VGI_WORKER_LOG_LEVEL")
+    if raw_level:
+        try:
+            log_level = LogLevel(raw_level.upper())
+        except ValueError:
+            valid = ", ".join(level.value for level in LogLevel)
+            sys.exit(f"VGI_WORKER_LOG_LEVEL={raw_level!r} is not a log level (expected one of: {valid})")
+
+    raw_format = os.environ.get("VGI_WORKER_LOG_FORMAT")
+    if raw_format:
+        try:
+            log_format = LogFormat(raw_format.lower())
+        except ValueError:
+            valid = ", ".join(fmt.value for fmt in LogFormat)
+            sys.exit(f"VGI_WORKER_LOG_FORMAT={raw_format!r} is not a log format (expected one of: {valid})")
+
+    raw_loggers = os.environ.get("VGI_WORKER_LOG_LOGGERS")
+    if raw_loggers:
+        names = [name.strip() for name in raw_loggers.split(",") if name.strip()]
+        if names:
+            log_loggers = names
+
+    return log_level, log_loggers, log_format
+
+
 def configure_worker_logging(
     *,
     debug: bool = False,
@@ -55,6 +118,10 @@ def configure_worker_logging(
 ) -> int:
     """Configure stdlib logging for a VGI worker process.
 
+    ``VGI_WORKER_LOG_LEVEL``, ``VGI_WORKER_LOG_FORMAT`` and
+    ``VGI_WORKER_LOG_LOGGERS`` override the corresponding arguments when set —
+    see :func:`_env_overrides`.
+
     Args:
         debug: If True, force DEBUG on all default loggers (overrides *log_level*).
         log_level: Logging level when *debug* is False.
@@ -65,6 +132,7 @@ def configure_worker_logging(
         The effective numeric log level.
 
     """
+    log_level, log_loggers, log_format = _env_overrides(log_level, log_loggers, log_format)
     effective_level = logging.DEBUG if debug else getattr(logging, log_level.value)
 
     handler = logging.StreamHandler(sys.stderr)

vgi_python-0.23.0/vgi/profiling.py

@@ -0,0 +1,93 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Opt-in CPU profiling for VGI worker processes.
+
+A worker is normally a subprocess someone else launched — a container, a test
+harness, a DuckDB client spawning it over stdio — so the usual ways of profiling
+Python (``python -m cProfile``, an interactive session) are not available to
+whoever needs the answer. Setting one environment variable is.
+
+Shared by every worker entry point rather than living in one CLI, so that
+``vgi-serve`` and the fixture servers profile identically.
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import sys
+
+__all__ = ["maybe_start_profile"]
+
+
+def maybe_start_profile() -> None:
+    """Start a ``cProfile`` profiler when ``VGI_WORKER_PROFILE`` names a path.
+
+    Set ``VGI_WORKER_PROFILE`` to a directory (or a file path) and the worker
+    writes ``pstats`` data on exit, readable with :mod:`pstats`, snakeviz, or
+    ``python -m pstats``. A directory gets one file per process
+    (``worker-<pid>.prof``), which is what you want when the thing under
+    investigation spawns more than one worker.
+
+    **All threads are covered by the single profiler.** That matters because
+    under ``--http`` a worker runs on waitress, which serves every request on a
+    pool thread — a main-thread-only profile would faithfully report time spent
+    in ``select()`` and nothing about the work being measured. Since Python 3.12
+    ``cProfile`` is built on :mod:`sys.monitoring`, whose tool registration is
+    process-global rather than per-thread, so one profiler observes the pool
+    threads too.
+
+    The corollary is that there can be only **one** active profiler in the
+    process: ``sys.monitoring`` allows a given tool id to be claimed once, so a
+    profiler-per-thread raises ``ValueError: Another profiling tool is already
+    active`` in every waitress thread — and because that happens as the thread
+    starts, the server never accepts a connection at all.
+
+    ``atexit`` alone would not be enough: a worker is normally stopped with
+    ``SIGTERM``, whose default action terminates without unwinding, so the
+    profile would be lost on every normal shutdown. A handler turns it into an
+    ordinary exit — but only if nothing else has claimed ``SIGTERM``, since
+    stealing another component's shutdown path would be a worse bug than a
+    missing profile.
+    """
+    dest = os.environ.get("VGI_WORKER_PROFILE")
+    if not dest:
+        return
+
+    import atexit
+    import cProfile
+    import signal
+
+    path = pathlib.Path(dest)
+    if path.is_dir() or dest.endswith(os.sep):
+        path.mkdir(parents=True, exist_ok=True)
+        path = path / f"worker-{os.getpid()}.prof"
+    else:
+        path.parent.mkdir(parents=True, exist_ok=True)
+
+    profiler = cProfile.Profile()
+    profiler.enable()
+    dumped = False
+
+    def _dump() -> None:
+        # atexit and the SIGTERM handler can both reach this; dumping twice
+        # would truncate the file the first call just wrote.
+        nonlocal dumped
+        if dumped:
+            return
+        dumped = True
+        try:
+            profiler.disable()
+            profiler.dump_stats(str(path))
+        except Exception as exc:  # noqa: BLE001 — never fail a shutdown over telemetry
+            sys.stderr.write(f"vgi: could not write profile: {exc}\n")
+            return
+        sys.stderr.write(f"vgi: wrote profile to {path}\n")
+
+    atexit.register(_dump)
+
+    def _on_sigterm(signum: int, frame: object) -> None:
+        raise SystemExit(128 + signum)
+
+    if signal.getsignal(signal.SIGTERM) in (signal.SIG_DFL, None):
+        signal.signal(signal.SIGTERM, _on_sigterm)

{vgi_python-0.22.0 → vgi_python-0.23.0}/vgi/serve.py

@@ -35,6 +35,7 @@ from types import ModuleType
 from typing import TYPE_CHECKING, Any, Literal
 
 from vgi.logging_config import LogFormat, LogLevel
+from vgi.profiling import maybe_start_profile
 
 if TYPE_CHECKING:
     import falcon
@@ -202,6 +203,7 @@ def create_app(
     signing_key: bytes | None = None,
     log_level: int = logging.INFO,
     authenticate: Callable[[falcon.Request], AuthContext] | None = None,
+    proxy_proof_required: bool | None = None,
     oauth_resource_metadata: Any = None,
     otel_config: OtelConfig | None = None,
     max_stream_response_bytes: int | None = None,
@@ -224,6 +226,12 @@ def create_app(
         authenticate: Optional callback that validates each HTTP request
             and returns an `AuthContext`. When ``None``, all requests are
             anonymous.
+        proxy_proof_required: Whether to advertise ``VGI-Proxy-Proof-Required``
+            so a proxy can confirm this worker actually enforces the proof it
+            mints. ``None`` (the default) derives it from
+            ``VGI_PROXY_PROOF_MODE``, which is also where the gate itself comes
+            from — so the advertisement cannot drift from the posture. Pass a
+            bool only when supplying a hand-built gate via ``authenticate``.
         oauth_resource_metadata: Optional `OAuthResourceMetadata` for
             RFC 9728 discovery endpoint.
         otel_config: Optional OpenTelemetry configuration.  When provided,
@@ -259,6 +267,9 @@ def create_app(
     if signing_key is None:
         signing_key = os.urandom(32)
 
+    if proxy_proof_required is None:
+        proxy_proof_required = (os.environ.get("VGI_PROXY_PROOF_MODE") or "").strip().lower() == "require"
+
     worker = worker_cls(quiet=True, log_level=log_level)
     worker._vgi_tracer = VgiTracer.create(otel_config)
     worker._signing_key = signing_key
@@ -271,6 +282,7 @@ def create_app(
         cors_origins=cors_origins,
         token_key=signing_key,
         authenticate=authenticate,
+        proxy_proof_required=proxy_proof_required,
         oauth_resource_metadata=oauth_resource_metadata,
         otel_config=otel_config,
         max_stream_response_bytes=max_stream_response_bytes,
@@ -353,12 +365,17 @@ def main() -> None:
     ) -> None:
         env_debug = os.environ.get("VGI_WORKER_DEBUG", "").lower() in ("1", "true", "yes")
         effective_debug = debug or env_debug
+        # VGI_WORKER_LOG_* overrides are applied inside configure_worker_logging,
+        # so every worker entry point honours them identically.
         effective_level = configure_worker_logging(
             debug=effective_debug,
             log_level=log_level,
             log_loggers=log_logger,
             log_format=log_format,
         )
+        # Before the worker class is imported, so module-import cost lands in
+        # the profile too — for many workers that is the bulk of startup.
+        maybe_start_profile()
 
         # Resolve env var overrides
         describe = _resolve_describe(describe)