vgi-python 0.8.11__tar.gz → 0.10.0__tar.gz

{vgi_python-0.8.11 → vgi_python-0.10.0}/.github/workflows/integration.yml

@@ -39,7 +39,9 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:
-      fail-fast: false
+      # If any transport leg fails, cancel the rest — one broken transport means
+      # the build is broken; no value in burning ~45 min finishing the others.
+      fail-fast: true
       matrix:
         include:
           # stdio:  the subprocess transport (whole suite — the primary lane).

{vgi_python-0.8.11 → vgi_python-0.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: vgi-python
-Version: 0.8.11
+Version: 0.10.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.8.11 → vgi_python-0.10.0}/packages/vgi-fixtures/pyproject.toml

@@ -38,6 +38,7 @@ dependencies = ["vgi-python[test-fixtures-writable]", "vgi-python[duckdb]"]
 
 [project.scripts]
 vgi-fixture-worker = "vgi._test_fixtures.worker:main"
+vgi-fixture-companion-worker = "vgi._test_fixtures.companion:main"
 vgi-fixture-http = "vgi._test_fixtures.http_server:main"
 vgi-fixture-versioned-worker = "vgi._test_fixtures.versioned:main"
 vgi-fixture-versioned-tables-worker = "vgi._test_fixtures.versioned_tables:main"

{vgi_python-0.8.11 → vgi_python-0.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.8.11"
+version = "0.10.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/catalog/test_declarative.py

@@ -41,8 +41,10 @@ from vgi.catalog import (
     View,
     ViewInfo,
 )
+from vgi.invocation import BindResponse
 from vgi.schema_utils import schema
 from vgi.table_function import (
+    BindParams,
     ProcessParams,
     TableFunctionGenerator,
     bind_fixed_schema,
@@ -81,6 +83,31 @@ class UsersFunction(TableFunctionGenerator[EmptyArgs]):
         out.finish()
 
 
+@init_single_worker
+class SecretLookupFunction(TableFunctionGenerator[EmptyArgs]):
+    """Backing function whose ``on_bind`` performs a two-phase secret lookup.
+
+    This exercises the bug where ``Table(function=...)`` schema derivation took
+    the empty secret-scope-request schema instead of retrying the bind.
+    """
+
+    class Meta:  # noqa: D106
+        name = "secret_scan"
+        description = "Backing function that looks up a secret at bind time"
+
+    FIXED_SCHEMA: ClassVar[pa.Schema] = schema({"id": pa.int64(), "name": pa.string()})
+
+    @classmethod
+    def on_bind(cls, params: BindParams[EmptyArgs]) -> BindResponse:  # noqa: D102
+        params.secrets.get("some_secret")  # unresolved -> registers a two-phase lookup
+        return BindResponse(output_schema=cls.FIXED_SCHEMA)
+
+    @classmethod
+    def process(cls, params: ProcessParams[EmptyArgs], state: None, out: OutputCollector) -> None:  # noqa: D102
+        out.emit(pa.RecordBatch.from_pydict({"id": [1], "name": ["x"]}, schema=params.output_schema))
+        out.finish()
+
+
 @init_single_worker
 @bind_fixed_schema
 class EventsFunction(TableFunctionGenerator[EmptyArgs]):
@@ -186,6 +213,18 @@ class TestTableWithFunction:
         assert resolved.field(0).name == "id"
         assert resolved.field(1).name == "name"
 
+    def test_function_backed_table_derives_schema_despite_secret_lookup(self) -> None:
+        """A backing function that calls secrets.get() in on_bind still yields its schema.
+
+        Regression: on_bind's two-phase secret lookup makes the first bind return an
+        empty secret-scope-request schema; schema derivation must retry with resolved
+        secrets rather than adopt the empty schema (which produced 0-column tables).
+        """
+        table = Table(name="things", function=SecretLookupFunction)
+        resolved = table.resolved_columns
+        assert [f.name for f in resolved] == ["id", "name"]
+        assert len(resolved) == 2
+
     def test_table_function_backed_with_constraints(self) -> None:
         """Function-backed table can have constraints validated."""
         table = Table(
@@ -336,7 +375,6 @@ class TestTableWithFunction:
         from typing import Any as _Any
 
         from vgi.invocation import BindResponse
-        from vgi.table_function import BindParams
 
         @init_single_worker
         class _ManualBindFn(TableFunctionGenerator[EmptyArgs]):

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/catalog/test_serialization.py

@@ -1110,7 +1110,7 @@ class TestArrowSchemaCorrectness:
     def test_catalog_attach_result_schema(self) -> None:
         """Verify CatalogAttachResult Arrow schema."""
         schema = CatalogAttachResult.ARROW_SCHEMA
-        assert len(schema) == 14
+        assert len(schema) == 15
         assert schema.field("attach_opaque_data").type == pa.binary()
         assert schema.field("supports_transactions").type == pa.bool_()
         assert schema.field("supports_time_travel").type == pa.bool_()
@@ -1120,6 +1120,7 @@ class TestArrowSchemaCorrectness:
         assert schema.field("default_schema").type == pa.string()
         assert schema.field("settings").type == pa.list_(pa.binary())
         assert schema.field("secret_types").type == pa.list_(pa.binary())
+        assert schema.field("attach_catalogs").type == pa.list_(pa.binary())
         assert schema.field("resolved_data_version").type == pa.string()
         assert schema.field("resolved_implementation_version").type == pa.string()
 

vgi_python-0.10.0/tests/conformance/test_const_param_enforcement.py

@@ -0,0 +1,69 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Conformance test: ConstParam value constraints are enforced at bind.
+
+A ``ConstParam`` may declare ``choices``/``ge``/``le``/``gt``/``lt``/``pattern``.
+Those are surfaced for agent discovery (via ``vgi_function_arguments()``) AND
+must be enforced: a const value that violates a declared constraint fails at
+bind with ``ArgumentValidationError``, rather than silently reaching
+``compute()``. This mirrors the legacy ``Arg`` descriptor path.
+"""
+
+from __future__ import annotations
+
+from typing import Annotated
+
+import pyarrow as pa
+import pytest
+
+from vgi.arguments import Arguments, ArgumentValidationError, ConstParam, Param, Returns
+from vgi.protocol import BindRequest, FunctionType
+from vgi.scalar_function import BindResult, ScalarFunction
+
+
+class _Measure(ScalarFunction):
+    class Meta:
+        name = "measure"
+
+    @classmethod
+    def on_bind(cls, params: object) -> BindResult:
+        return BindResult(pa.string())
+
+    @classmethod
+    def compute(
+        cls,
+        unit: Annotated[str, ConstParam("Output unit", choices=["mm", "cm", "m"])],
+        precision: Annotated[int, ConstParam("Decimals", ge=0, le=10)],
+        value: Annotated[pa.DoubleArray, Param(doc="Value")],
+    ) -> Annotated[pa.StringArray, Returns()]:
+        return pa.array([str(value)], type=pa.string())
+
+
+def _bind(unit: str, precision: int) -> None:
+    _Measure.bind(
+        BindRequest(
+            function_name="measure",
+            arguments=Arguments(positional=(pa.scalar(unit), pa.scalar(precision))),
+            function_type=FunctionType.SCALAR,
+            input_schema=pa.schema([("value", pa.float64())]),
+        )
+    )
+
+
+def test_valid_const_values_bind() -> None:
+    """A const value inside every declared constraint binds cleanly."""
+    _bind("cm", 2)
+
+
+def test_choice_violation_raises_at_bind() -> None:
+    """A const value outside the choice set fails at bind."""
+    with pytest.raises(ArgumentValidationError) as exc:
+        _bind("xx", 2)
+    assert "unit" in str(exc.value)
+
+
+def test_range_violation_raises_at_bind() -> None:
+    """A const value outside the numeric range fails at bind."""
+    with pytest.raises(ArgumentValidationError) as exc:
+        _bind("cm", 99)
+    assert "precision" in str(exc.value)

vgi_python-0.10.0/tests/conformance/test_container.py

@@ -0,0 +1,17 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Conformance stub for ``vgi/test/sql/integration/container/``."""
+
+from __future__ import annotations
+
+from tests.conformance._stub import skip_area
+
+skip_area(
+    "container",
+    [
+        "errors.test",
+        "shared_http.test",
+        "shared_tcp.test",
+        "sklearn_smoke.test",
+    ],
+)

vgi_python-0.10.0/tests/conformance/test_copy_from.py

@@ -0,0 +1,16 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Conformance stub for ``vgi/test/sql/integration/copy_from/``."""
+
+from __future__ import annotations
+
+from tests.conformance._stub import skip_area
+
+skip_area(
+    "copy_from",
+    [
+        "basic.test",
+        "options.test",
+        "secrets.test",
+    ],
+)

vgi_python-0.10.0/tests/conformance/test_copy_to.py

@@ -0,0 +1,21 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Conformance stub for ``vgi/test/sql/integration/copy_to/``."""
+
+from __future__ import annotations
+
+from tests.conformance._stub import skip_area
+
+skip_area(
+    "copy_to",
+    [
+        "basic.test",
+        "failure.test",
+        "options.test",
+        "ordered.test",
+        "parallel.test",
+        "secrets.test",
+        "tmp_file.test",
+        "types.test",
+    ],
+)

vgi_python-0.10.0/tests/conformance/test_github.py

@@ -0,0 +1,15 @@
+# Copyright 2025, 2026 Query Farm LLC - https://query.farm
+
+"""Conformance stub for ``vgi/test/sql/integration/github/``."""
+
+from __future__ import annotations
+
+from tests.conformance._stub import skip_area
+
+skip_area(
+    "github",
+    [
+        "download.test",
+        "errors.test",
+    ],
+)

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/conformance/test_secret.py

@@ -9,10 +9,13 @@ from tests.conformance._stub import skip_area
 skip_area(
     "secret",
     [
+        "secret_aggregate.test",
+        "secret_function_backed_table.test",
         "secret_no_secret.test",
         "secret_registration.test",
         "secret_scalar.test",
         "secret_scoped.test",
         "secret_table_function.test",
+        "secret_table_in_out.test",
     ],
 )

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/test_aggregate_function.py

@@ -13,7 +13,14 @@ import pytest
 from vgi_rpc import ArrowSerializableDataclass, ArrowType
 
 from vgi.aggregate_function import AggregateFunction
-from vgi.arguments import ConstParam, Param, Returns
+from vgi.arguments import (
+    Arguments,
+    ArgumentValidationError,
+    ConstParam,
+    Param,
+    Returns,
+    validate_const_arg_constraints,
+)
 from vgi.table_function import ProcessParams
 
 # ---------------------------------------------------------------------------
@@ -199,6 +206,49 @@ class TestInitSubclassParameterExtraction:
         assert arg.arrow_type == pa.float64()
         assert arg.const is True
 
+    def test_const_param_constraints_enforced(self) -> None:
+        """A ConstParam's declared constraints are enforced at bind (same as scalar)."""
+
+        class BoundedAgg(AggregateFunction[SimpleState]):
+            class Meta:
+                name = "test_bounded_agg"
+
+            @classmethod
+            def initial_state(cls, params: ProcessParams[Any]) -> SimpleState:
+                return SimpleState()
+
+            @classmethod
+            def update(
+                cls,
+                states: dict[int, SimpleState],
+                group_ids: pa.Int64Array,
+                value: Annotated[pa.DoubleArray, Param(doc="Values")],
+                mode: Annotated[str, ConstParam("Mode", choices=["min", "max"])] = "min",
+            ) -> None:
+                pass
+
+            @classmethod
+            def combine(cls, source: SimpleState, target: SimpleState, params: ProcessParams[Any]) -> SimpleState:
+                return SimpleState()
+
+            @classmethod
+            def finalize(
+                cls,
+                group_ids: pa.Int64Array,
+                states: dict[int, SimpleState],
+                params: ProcessParams[Any],
+            ) -> Annotated[pa.RecordBatch, Returns(pa.float64())]:
+                return pa.record_batch({"result": pa.array([], type=pa.float64())})
+
+        # The ConstParam carries its choices (threaded via _const_param_to_arg)...
+        const_params = BoundedAgg._const_params
+        assert const_params["mode"].choices == ["min", "max"]
+        # ...and the shared bind-time validator (called by aggregate_bind) enforces it.
+        validate_const_arg_constraints(const_params, Arguments(positional=(pa.scalar("max"),)))
+        with pytest.raises(ArgumentValidationError) as exc:
+            validate_const_arg_constraints(const_params, Arguments(positional=(pa.scalar("median"),)))
+        assert "mode" in str(exc.value)
+
     def test_const_param_phases(self) -> None:
         """ConstParam phase values are stored correctly."""
 

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/test_argument_spec.py

@@ -12,13 +12,18 @@ import pytest
 from vgi.argument_spec import (
     VGI_ARG_KEY,
     VGI_ARG_NAMED,
+    VGI_CHOICES_KEY,
+    VGI_DEFAULT_KEY,
     VGI_DOC_KEY,
+    VGI_PATTERN_KEY,
+    VGI_RANGE_KEY,
     VGI_TYPE_ANY,
     VGI_TYPE_KEY,
     VGI_TYPE_TABLE,
     VGI_VARARGS_KEY,
     VGI_VARARGS_TRUE,
     ArgumentSpec,
+    _format_range,
     argument_specs_to_schema,
     extract_argument_specs,
     schema_to_argument_specs,
@@ -722,3 +727,99 @@ class TestArgumentDoc:
         specs = {s.name: s for s in extract_argument_specs(FunctionWithDocs)}
         assert specs["unit"].doc == "Unit string, e.g. 'mi'"
         assert specs["scale"].doc == ""
+
+
+class TestArgumentConstraints:
+    """Per-argument constraint metadata (choices / range / pattern / default)."""
+
+    def test_format_range_notation(self) -> None:
+        """Interval notation covers inclusive, exclusive, open, and empty bounds."""
+        assert _format_range(0, 100, None, None) == "[0, 100]"
+        assert _format_range(None, None, 0, None) == "(0, +inf)"
+        assert _format_range(1, None, None, 10) == "[1, 10)"
+        assert _format_range(None, 5, None, None) == "(-inf, 5]"
+        assert _format_range(None, None, None, None) is None
+
+    def test_writer_emits_constraint_keys(self) -> None:
+        """A constrained spec writes value-encoded constraint metadata keys."""
+        spec = ArgumentSpec(
+            name="unit",
+            position="unit",
+            arrow_type=pa.utf8(),
+            default_json='"mm"',
+            choices_json='["mm", "cm", "m"]',
+            range_notation="[0, 100]",
+            pattern="^[A-Z]{2}$",
+        )
+        meta = argument_specs_to_schema([spec])[0].metadata
+        assert meta[VGI_DEFAULT_KEY] == b'"mm"'
+        assert meta[VGI_CHOICES_KEY] == b'["mm", "cm", "m"]'
+        assert meta[VGI_RANGE_KEY] == b"[0, 100]"
+        assert meta[VGI_PATTERN_KEY] == b"^[A-Z]{2}$"
+
+    def test_unconstrained_arg_omits_keys(self) -> None:
+        """An arg without constraints yields none of the constraint keys."""
+        meta = argument_specs_to_schema([ArgumentSpec(name="v", position=0, arrow_type=pa.int64())])[0].metadata
+        for key in (VGI_DEFAULT_KEY, VGI_CHOICES_KEY, VGI_RANGE_KEY, VGI_PATTERN_KEY):
+            assert meta is None or key not in meta
+
+    def test_constraints_round_trip(self) -> None:
+        """Constraint metadata survives the schema round-trip."""
+        spec = ArgumentSpec(
+            name="mode",
+            position="mode",
+            arrow_type=pa.utf8(),
+            default_json='"add"',
+            choices_json='["add", "mul"]',
+            range_notation="[0, 10)",
+            pattern="^x+$",
+        )
+        out = schema_to_argument_specs(argument_specs_to_schema([spec]))[0]
+        assert out.default_json == '"add"'
+        assert out.choices_json == '["add", "mul"]'
+        assert out.range_notation == "[0, 10)"
+        assert out.pattern == "^x+$"
+
+    def test_extract_constraints_from_arg(self) -> None:
+        """extract_argument_specs reads choices/default/bounds/pattern off Arg."""
+
+        class FunctionWithConstraints(TableInOutFunction):  # type: ignore[type-arg]
+            mode: str = Arg[str](  # type: ignore[assignment]
+                "mode", doc="Blend mode", choices=["add", "mul"], default="add"
+            )
+            level: int = Arg[int](0, ge=1, le=10)  # type: ignore[assignment]
+            code: str = Arg[str](1, pattern="^[A-Z]{2}$")  # type: ignore[assignment]
+
+        specs = {s.name: s for s in extract_argument_specs(FunctionWithConstraints)}
+        assert specs["mode"].choices_json == '["add", "mul"]'
+        assert specs["mode"].default_json == '"add"'
+        assert specs["level"].range_notation == "[1, 10]"
+        assert specs["code"].pattern == "^[A-Z]{2}$"
+
+    def test_const_param_constraints_surface(self) -> None:
+        """The modern Param/ConstParam API surfaces choices / range / pattern."""
+        from vgi.arguments import ConstParam, Param, Returns
+        from vgi.scalar_function import ScalarFunction
+
+        class ConstrainedScalar(ScalarFunction):
+            class Meta:
+                name = "constrained_scalar"
+                description = "constraint demo"
+
+            @classmethod
+            def compute(
+                cls,
+                unit: Annotated[str, ConstParam("Output unit", choices=["mm", "cm", "m"])],
+                precision: Annotated[int, ConstParam("Decimals", ge=0, le=10)],
+                code: Annotated[str, ConstParam("Label code", pattern="^[A-Z]{2}$")],
+                value: Annotated[pa.DoubleArray, Param(doc="Value")],
+            ) -> Annotated[pa.StringArray, Returns()]:
+                return value  # never executed — the test only reads arg specs
+
+        specs = {s.name: s for s in extract_argument_specs(ConstrainedScalar)}
+        assert specs["unit"].choices_json == '["mm", "cm", "m"]'
+        assert specs["precision"].range_notation == "[0, 10]"
+        assert specs["code"].pattern == "^[A-Z]{2}$"
+        # The columnar Param carries no constraints.
+        assert specs["value"].choices_json is None
+        assert specs["value"].range_notation is None

{vgi_python-0.8.11 → vgi_python-0.10.0}/tests/test_copy_to_function.py

@@ -9,8 +9,14 @@ import types
 
 import pyarrow as pa
 
-from vgi._test_fixtures.copy_to import ExampleLinesCopyToArgs, ExampleLinesCopyToFunction
+from vgi._test_fixtures.copy_to import (
+    ExampleLinesCopyToArgs,
+    ExampleLinesCopyToFunction,
+    SecretLinesCopyToArgs,
+    SecretLinesCopyToFunction,
+)
 from vgi._test_fixtures.worker import ExampleCatalog
+from vgi.table_function import ResolvedSecrets, SecretsAccessor
 
 SCHEMA = pa.schema([("a", pa.int64()), ("b", pa.string())])
 
@@ -100,6 +106,81 @@ def test_close_empty_input_with_header_writes_header_only() -> None:
     assert _read(out_name) == "a,b\n"
 
 
+# ---------------------------------------------------------------------------
+# Secret forwarding (CopyToFunction.on_secrets hook)
+# ---------------------------------------------------------------------------
+
+
+def _secret_params(store: _Store, secrets: ResolvedSecrets) -> types.SimpleNamespace:
+    """A TableBufferingParams-shaped stub carrying resolved secrets for write/close."""
+    bind_call = types.SimpleNamespace(input_schema=SCHEMA)
+    init_call = types.SimpleNamespace(bind_call=bind_call)
+    return types.SimpleNamespace(storage=store, init_call=init_call, execution_id=b"x", secrets=secrets)
+
+
+def test_on_secrets_requests_destination_scoped_secret() -> None:
+    """on_secrets() registers a pending lookup scoped to the COPY destination path."""
+    accessor = SecretsAccessor(None)  # nothing resolved yet → first-call behavior
+    params = types.SimpleNamespace(
+        args=SecretLinesCopyToArgs(),  # default secret_type='vgi_example'
+        bind_call=types.SimpleNamespace(copy_to=types.SimpleNamespace(file_path="s3://bucket/out.bin")),
+        secrets=accessor,
+    )
+    SecretLinesCopyToFunction.on_secrets(params)
+    # The hook must have asked the framework for a scoped two-phase resolution.
+    assert accessor.needs_resolution
+    pending = accessor.pending_lookups
+    assert len(pending) == 1
+    assert pending[0].secret_type == "vgi_example"
+    assert pending[0].scope == "s3://bucket/out.bin"
+
+
+def test_close_forwards_resolved_secret_api_key() -> None:
+    """close() writes the resolved (destination-scoped) secret's api_key + row count."""
+    store = _Store()
+    out_name = _tmp_path()
+    secrets = ResolvedSecrets(
+        {
+            "writer_creds": {
+                "type": pa.scalar("vgi_example"),
+                "scope": pa.scalar(out_name),
+                "api_key": pa.scalar("WRITER_KEY"),
+            }
+        }
+    )
+    params = _secret_params(store, secrets)
+    opts = SecretLinesCopyToArgs()
+
+    SecretLinesCopyToFunction.write(
+        batch=pa.record_batch({"a": [1, 2], "b": ["x", "y"]}, schema=SCHEMA),
+        options=opts,
+        file_path=out_name,
+        params=params,
+    )
+    n = SecretLinesCopyToFunction.close(options=opts, file_path=out_name, params=params)
+    assert n == 2
+    assert _read(out_name) == "api_key=WRITER_KEY\nrows=2\n"
+
+
+def test_close_writes_none_when_secret_absent() -> None:
+    """A genuinely missing secret resolves to 'NONE' (silent miss, not an error)."""
+    store = _Store()
+    out_name = _tmp_path()
+    params = _secret_params(store, ResolvedSecrets())
+    opts = SecretLinesCopyToArgs()
+    n = SecretLinesCopyToFunction.close(options=opts, file_path=out_name, params=params)
+    assert n == 0
+    assert _read(out_name) == "api_key=NONE\nrows=0\n"
+
+
+def test_catalog_advertises_secret_lines_out_format() -> None:
+    """The secret-forwarding writer is advertised like any other COPY TO format."""
+    formats = ExampleCatalog().copy_from_formats(attach_opaque_data=b"", transaction_opaque_data=None)
+    by = {(f.direction, f.format_name): f for f in formats}
+    assert ("to", "secret_lines_out") in by
+    assert by[("to", "secret_lines_out")].handler == "secret_lines_writer"
+
+
 def test_catalog_advertises_copy_to_format() -> None:
     """The example catalog advertises example_lines_out with direction='to'."""
     formats = ExampleCatalog().copy_from_formats(attach_opaque_data=b"", transaction_opaque_data=None)