vgi-python 0.8.0__py3-none-any.whl

vgi/__init__.py

@@ -0,0 +1,152 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""VGI (Vector Gateway Interface) - Apache Arrow-based protocol for DuckDB extensions.
+
+VGI provides a framework for connecting DuckDB to external programs via
+streaming Arrow IPC. User-defined functions run in worker subprocesses
+and communicate with the database through stdin/stdout.
+
+"""
+
+from vgi_rpc.log import Level, Message
+
+from vgi.aggregate_function import (
+    AggregateBindParams,
+    AggregateFunction,
+)
+from vgi.argument_spec import (
+    VGI_ARG_KEY,
+    VGI_ARG_NAMED,
+    VGI_CONST_KEY,
+    VGI_CONST_TRUE,
+    VGI_TYPE_ANY,
+    VGI_TYPE_KEY,
+    VGI_TYPE_TABLE,
+    VGI_VARARGS_KEY,
+    VGI_VARARGS_TRUE,
+    ArgumentSpec,
+    argument_specs_to_schema,
+    schema_to_argument_specs,
+)
+from vgi.arguments import (
+    AnyArrow,
+    AnyArrowValue,
+    Arg,
+    Arguments,
+    ArgumentValidationError,
+    Auth,
+    ConstParam,
+    Param,
+    Returns,
+    TableInput,
+)
+from vgi.auth import AuthContext, CallContext
+from vgi.metadata import (
+    CatalogFunctionType,
+    FunctionExample,
+    FunctionStability,
+    OrderPreservation,
+    ParameterInfo,
+    ResolvedMetadata,
+    TableInputValidationError,
+    functions_to_arrow,
+)
+
+# Re-export commonly used protocol types
+from vgi.protocol import (
+    BindRequest,
+    InitRequest,
+    VgiOutputCollector,
+)
+from vgi.scalar_function import (
+    RowCountMismatchError,
+    ScalarFunction,
+    ScalarFunctionGenerator,
+    TypeMismatchError,
+)
+from vgi.schema_utils import schema, schema_like
+from vgi.table_filter_pushdown import (
+    ColumnBounds,
+    ColumnRefNode,
+    ComparisonNode,
+    ConjunctionNode,
+    ConstantNode,
+    ExpressionFilter,
+    ExpressionNode,
+    FilterDeserializationError,
+    FilterError,
+    FilterVersionError,
+    FunctionNode,
+    PushdownFilters,
+    deserialize_filters,
+)
+from vgi.table_in_out_function import (
+    TableInOutFunction,
+    TableInOutGenerator,
+)
+from vgi.worker import Worker
+
+__all__ = [
+    "AggregateBindParams",
+    "AggregateFunction",
+    "AnyArrow",
+    "AnyArrowValue",
+    "Arg",
+    "ArgumentSpec",
+    "ArgumentValidationError",
+    "Arguments",
+    "Auth",
+    "AuthContext",
+    "CallContext",
+    "argument_specs_to_schema",
+    "BindRequest",
+    "ColumnBounds",
+    "ColumnRefNode",
+    "ComparisonNode",
+    "ConjunctionNode",
+    "ConstantNode",
+    "ConstParam",
+    "deserialize_filters",
+    "ExpressionFilter",
+    "ExpressionNode",
+    "FilterDeserializationError",
+    "FilterError",
+    "FilterVersionError",
+    "Param",
+    "PushdownFilters",
+    "Returns",
+    # Metadata constants for parsing argument spec schemas
+    "VGI_ARG_KEY",
+    "VGI_ARG_NAMED",
+    "VGI_CONST_KEY",
+    "VGI_CONST_TRUE",
+    "VGI_TYPE_KEY",
+    "VGI_TYPE_TABLE",
+    "VGI_TYPE_ANY",
+    "VGI_VARARGS_KEY",
+    "VGI_VARARGS_TRUE",
+    "FunctionNode",
+    "FunctionExample",
+    "FunctionStability",
+    "CatalogFunctionType",
+    "InitRequest",
+    "Level",
+    "VgiOutputCollector",
+    "Message",
+    "OrderPreservation",
+    "ParameterInfo",
+    "ResolvedMetadata",
+    "RowCountMismatchError",
+    "ScalarFunction",
+    "ScalarFunctionGenerator",
+    "TableInOutFunction",
+    "TableInOutGenerator",
+    "TableInput",
+    "TableInputValidationError",
+    "TypeMismatchError",
+    "Worker",
+    "functions_to_arrow",
+    "schema",
+    "schema_like",
+    "schema_to_argument_specs",
+]

vgi/_duckdb.py

@@ -0,0 +1,62 @@
+# Copyright 2026 Query Farm LLC - https://query.farm
+
+"""DuckDB-compatible engine resolution: prefer ``haybarn``, fall back to ``duckdb``.
+
+vgi does not hard-depend on either engine. Code that needs an in-process
+engine (expression-filter evaluation, the transactor, the statistics
+fixtures) resolves it through this module, which prefers Query Farm's
+``haybarn`` distribution when installed and falls back to stock ``duckdb``.
+
+Install one via the extras: ``pip install vgi[haybarn]`` (preferred) or
+``pip install vgi[duckdb]``.
+"""
+
+from __future__ import annotations
+
+import importlib
+from types import ModuleType
+from typing import TYPE_CHECKING, Any, cast
+
+if TYPE_CHECKING:
+    import duckdb
+
+#: Engine modules in preference order. Both expose the duckdb-python API
+#: (``connect()``, ``DuckDBPyConnection``, ...).
+_ENGINE_MODULES = ("haybarn", "duckdb")
+
+_engine: ModuleType | None = None
+
+
+def engine_module() -> ModuleType:
+    """Return the resolved engine module (``haybarn`` preferred, ``duckdb`` fallback).
+
+    The result is cached for the life of the process.
+
+    Raises:
+        ImportError: if neither engine is installed.
+
+    """
+    global _engine
+    if _engine is None:
+        for name in _ENGINE_MODULES:
+            try:
+                _engine = importlib.import_module(name)
+                break
+            except ImportError:
+                continue
+        else:
+            raise ImportError(
+                "No DuckDB-compatible engine is installed. Install 'haybarn' (preferred) "
+                "or 'duckdb', e.g. `pip install vgi[haybarn]` or `pip install vgi[duckdb]`."
+            )
+    return _engine
+
+
+def connect(*args: Any, **kwargs: Any) -> duckdb.DuckDBPyConnection:
+    """Open a connection via the resolved engine's ``connect()``.
+
+    Typed as ``duckdb.DuckDBPyConnection`` for the type-checker; at runtime
+    the object is whichever engine module won resolution (haybarn's
+    connection class implements the same API).
+    """
+    return cast("duckdb.DuckDBPyConnection", engine_module().connect(*args, **kwargs))

vgi/_storage_profile.py

@@ -0,0 +1,132 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Backend-agnostic per-shard storage round-trip profiler (opt-in).
+
+Enable with ``VGI_STORAGE_PROFILE=1``. Recording happens at the
+``BoundStorage`` facade so it works against *any* backend — in-process sqlite
+included — which lets the suite be profiled locally without a Cloudflare
+deployment. Backends that already self-profile at their transport layer (the
+cloudflare-do ``_post``, which sees per-page round-trips) set
+``_profiles_at_transport = True`` so ``BoundStorage`` defers to them and the
+two layers never double-count.
+
+Stats are keyed by ``(shard_key, op)``. Each test run does one ATTACH with a
+fresh random-nonce sealed envelope, so one ``shard_key`` == one test run — the
+per-shard summary is effectively a per-test storage profile. A daemon flusher
+logs a JSON summary periodically (and atexit) under ``vgi.storage.profile``.
+"""
+
+from __future__ import annotations
+
+import atexit
+import json
+import logging
+import os
+import threading
+import time
+from typing import Any
+
+_PROFILE_ON = os.environ.get("VGI_STORAGE_PROFILE") == "1"
+_profile_logger = logging.getLogger("vgi.storage.profile")
+
+
+class _StorageProfiler:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        # (shard, op) -> [count, total_s, max_s, total_bytes, max_bytes]
+        # ``max_bytes`` is the largest single-call HTTP body (max of request
+        # and response direction) — the number that matters for provider
+        # request/response size limits.
+        self._stats: dict[tuple[str, str], list[float]] = {}
+        self._started = False
+
+    def record(self, shard: str, op: str, seconds: float, io_bytes: int) -> None:
+        with self._lock:
+            e = self._stats.setdefault((shard, op), [0.0, 0.0, 0.0, 0.0, 0.0])
+            e[0] += 1
+            e[1] += seconds
+            e[2] = max(e[2], seconds)
+            e[3] += io_bytes
+            e[4] = max(e[4], io_bytes)
+
+    def dump(self) -> None:
+        with self._lock:
+            by_shard: dict[str, dict[str, list[float]]] = {}
+            for (shard, op), e in self._stats.items():
+                by_shard.setdefault(shard, {})[op] = list(e)
+        for shard, ops in by_shard.items():
+            round_trips = int(sum(e[0] for e in ops.values()))
+            storage_s = sum(e[1] for e in ops.values())
+            op_summary = {
+                op: {
+                    "n": int(e[0]),
+                    "total_ms": round(e[1] * 1000, 1),
+                    "avg_ms": round(e[1] / e[0] * 1000, 1) if e[0] else 0,
+                    "max_ms": round(e[2] * 1000, 1),
+                    "io_kb": round(e[3] / 1024, 1),
+                    # largest single HTTP body (req or resp) — headroom vs
+                    # provider size caps.
+                    "max_call_kb": round(e[4] / 1024, 1),
+                }
+                for op, e in sorted(ops.items(), key=lambda kv: -kv[1][1])
+            }
+            _profile_logger.warning(
+                json.dumps(
+                    {
+                        "msg": "storage_profile",
+                        "shard": shard,
+                        "round_trips": round_trips,
+                        "storage_s": round(storage_s, 2),
+                        "ops": op_summary,
+                    }
+                )
+            )
+
+    def start(self, interval: float = 5.0) -> None:
+        if self._started:
+            return
+        self._started = True
+
+        def _loop() -> None:
+            while True:
+                time.sleep(interval)
+                self.dump()
+
+        threading.Thread(target=_loop, daemon=True, name="vgi-storage-profile").start()
+        atexit.register(self.dump)
+
+
+_profiler = _StorageProfiler()
+if _PROFILE_ON:
+    _profiler.start()
+
+
+def _byte_size(obj: object) -> int:
+    """Best-effort serialized size of a storage payload, without consuming generators.
+
+    bytes -> len; list/tuple -> summed size of elements (recursing one level so
+    ``[(k, v), ...]`` and ``[item, ...]`` are both counted); everything else
+    (ints, None, namespaces, generators/iterators) -> 0.
+    """
+    if isinstance(obj, (bytes, bytearray)):
+        return len(obj)
+    if isinstance(obj, (list, tuple)):
+        total = 0
+        for item in obj:
+            if isinstance(item, (bytes, bytearray)):
+                total += len(item)
+            elif isinstance(item, tuple):
+                total += sum(len(x) for x in item if isinstance(x, (bytes, bytearray)))
+        return total
+    return 0
+
+
+def io_call_bytes(args: tuple[Any, ...], kwargs: dict[str, Any], result: object) -> int:
+    """Largest single-direction HTTP body for one BoundStorage call.
+
+    Writes carry the payload in the args (items / value / appended item);
+    reads carry it in the result. Provider request/response size caps apply
+    per direction, so the relevant figure is the max of the two.
+    """
+    request = sum(_byte_size(a) for a in args) + sum(_byte_size(v) for v in kwargs.values())
+    return max(request, _byte_size(result))

vgi/_test_fixtures/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Example implementations for VGI.
+
+This package contains example functions and servers for testing and reference.
+"""
+
+from vgi._test_fixtures.table_in_out import (
+    BufferInputFunction,
+    EchoFunction,
+    RepeatInputsFunction,
+    SumAllColumnsFunction,
+)
+
+__all__ = [
+    "EchoFunction",
+    "BufferInputFunction",
+    "RepeatInputsFunction",
+    "SumAllColumnsFunction",
+]

vgi/_test_fixtures/accumulate/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Accumulate fixture worker: persistent named row collections over BoundStorage.
+
+Exposes ``accumulate`` / ``accumulate_read`` / ``accumulate_clear`` table
+functions that persist rows through the framework's ``FunctionStorage``,
+scoped per ATTACH session. The fixture is the end-to-end exercise of the
+``BoundStorage`` interfaces: persistent attach-scoped K/V segments (including
+ranged scans/deletes for TTL eviction), atomic counters, and execution-scoped
+staging logs.
+
+Hosted inside the consolidated ``vgi-fixture-worker`` alongside the other
+reproducer catalogs; driven by ``test/sql/integration/accumulate/*.test`` in
+``~/Development/vgi`` and mirrored by ``tests/conformance/test_accumulate.py``.
+"""
+
+from vgi._test_fixtures.accumulate.worker import AccumulateWorker
+
+__all__ = ["AccumulateWorker"]