vgi-python 0.26.0__py3-none-any.whl → 0.28.0__py3-none-any.whl

vgi/_test_fixtures/attach_options.py

@@ -31,7 +31,12 @@ from vgi_rpc import ArrowSerializableDataclass
 from vgi_rpc.rpc import CallContext, OutputCollector
 from vgi_rpc.utils import deserialize_record_batch, serialize_record_batch_bytes
 
-from vgi.catalog.attach_option import AttachOption, AttachOptionSpec, extract_attach_option_specs
+from vgi.catalog.attach_option import (
+    AttachOption,
+    AttachOptionSpec,
+    extract_attach_option_specs,
+    validate_required_attach_options,
+)
 from vgi.catalog.catalog_interface import (
     AttachOpaqueData,
     CatalogAttachResult,
@@ -53,6 +58,10 @@ __all__ = [
 
 
 CATALOG_NAME = "attach_options"
+# A second catalog from the same worker, declaring one option the caller must
+# supply. Kept separate from CATALOG_NAME so the defaults-and-round-trip
+# coverage above can keep attaching with no options at all.
+REQUIRED_CATALOG_NAME = "attach_options_required"
 _ATTACH_ID_SEP = b"\x00"
 _UUID_BYTES = 16
 
@@ -109,9 +118,19 @@ class AttachOptions:
     ] = {"a": 1, "b": "x"}
 
 
+class RequiredAttachOptions:
+    """Attach-time options for the catalog that refuses an anonymous ATTACH."""
+
+    # No class-level assignment: there is no default to fall back on, so the
+    # caller has to supply it.
+    api_key: Annotated[str, AttachOption(desc="API key", required=True)]
+    region: Annotated[str, AttachOption(desc="Region")] = "us-east-1"
+
+
 # Resolve once at import time; used both to build the echo function's output schema
 # and to backfill defaults in catalog_attach.
 _ATTACH_OPTION_SPECS: list[AttachOptionSpec] = extract_attach_option_specs(AttachOptions)
+_REQUIRED_OPTION_SPECS: list[AttachOptionSpec] = extract_attach_option_specs(RequiredAttachOptions)
 
 _ECHO_SCHEMA: pa.Schema = schema({spec.name: spec.type for spec in _ATTACH_OPTION_SPECS})
 
@@ -244,8 +263,14 @@ class AttachOptionsCatalog(ReadOnlyCatalogInterface):
         ctx: CallContext | None = None,
     ) -> CatalogAttachResult:
         del data_version_spec, implementation_version, ctx
-        if name != CATALOG_NAME:
-            raise ValueError(f"Unknown catalog: {name!r}. Available: {CATALOG_NAME}")
+        if name not in (CATALOG_NAME, REQUIRED_CATALOG_NAME):
+            raise ValueError(f"Unknown catalog: {name!r}. Available: {CATALOG_NAME}, {REQUIRED_CATALOG_NAME}")
+
+        if name == REQUIRED_CATALOG_NAME:
+            # This interface overrides catalog_attach, so the base class's check
+            # never runs — an overrider that declares required options calls the
+            # shared validator itself.
+            validate_required_attach_options(name, _REQUIRED_OPTION_SPECS, options)
 
         attach_opaque_data = _encode_attach_opaque_data(options)
 
@@ -270,6 +295,12 @@ class AttachOptionsCatalog(ReadOnlyCatalogInterface):
                 data_version_spec=None,
                 attach_option_specs=[spec.serialize() for spec in _ATTACH_OPTION_SPECS],
             ),
+            CatalogInfo(
+                name=REQUIRED_CATALOG_NAME,
+                implementation_version=None,
+                data_version_spec=None,
+                attach_option_specs=[spec.serialize() for spec in _REQUIRED_OPTION_SPECS],
+            ),
         ]
 
     def catalog_version(

vgi/_test_fixtures/http_server.py

@@ -412,25 +412,16 @@ def main() -> None:
                     mw._exempt_prefixes = (*mw._exempt_prefixes, f"{prefix}/__blobs__")
 
         if describe:
-            # Standardized landing surface (shared static page + JSON contract),
-            # describing the reference ExampleWorker catalog.
-            from vgi.http.landing_page import (
-                ColumnsResource,
-                DescribeJsonResource,
-                LandingPageResource,
-            )
+            # Standardized landing surface: the shared static page plus the
+            # browser client build it reads the catalog with.
+            from vgi.http.landing_page import ClientBundleResource, LandingPageResource
 
             oauth_active = oauth_metadata is not None and getattr(oauth_metadata, "client_id", None) is not None
             server_id = getattr(server, "server_id", "")
-            wsgi_app.add_route(prefix or "/", LandingPageResource(server_id=server_id))
-            wsgi_app.add_route(
-                f"{prefix}/describe.json",
-                DescribeJsonResource(ExampleWorker, oauth=oauth_active, server_id=server_id),
-            )
             wsgi_app.add_route(
-                f"{prefix}/describe/{{catalog}}/{{schema}}/{{table}}.json",
-                ColumnsResource(ExampleWorker),
+                prefix or "/", LandingPageResource(ExampleWorker, server_id=server_id, oauth=oauth_active)
             )
+            wsgi_app.add_route(f"{prefix}/vgi-client.js", ClientBundleResource())
 
         # vgi_rpc's make_wsgi_app already advertises VGI-Max-Request-Bytes and
         # installs a Falcon middleware enforcing it with a structured 413 that

vgi/catalog/__init__.py

@@ -16,6 +16,7 @@ from vgi.catalog.catalog_interface import (
     CatalogInterface,
     CatalogObject,
     CatalogSchemaObject,
+    ColumnStatistics,
     CopyFromFormatInfo,
     FunctionInfo,
     FunctionType,
@@ -35,6 +36,8 @@ from vgi.catalog.catalog_interface import (
     TableInfo,
     TransactionOpaqueData,
     ViewInfo,
+    deserialize_column_statistics,
+    serialize_column_statistics,
 )
 from vgi.catalog.descriptors import Catalog, ForeignKeyDef, Index, Macro, Schema, Sql, Table, View
 from vgi.catalog.secret_type import SecretTypeSpec
@@ -60,6 +63,7 @@ __all__ = [
     "CatalogInfo",
     "CatalogObject",
     "CatalogSchemaObject",
+    "ColumnStatistics",
     "IndexInfo",
     "MacroInfo",
     "SchemaInfo",
@@ -89,4 +93,7 @@ __all__ = [
     # Storage
     "CatalogStorage",
     "CatalogStorageSqlite",
+    # Column statistics wire format
+    "serialize_column_statistics",
+    "deserialize_column_statistics",
 ]

vgi/catalog/_descriptor_spec.py

@@ -5,9 +5,16 @@
 [`Setting`][vgi.catalog.setting.Setting] (session-level, resent every call) and
 [`AttachOption`][vgi.catalog.attach_option.AttachOption] (delivered once at
 ``catalog_attach``) are declared the same way — an `Annotated`-hint descriptor
-plus a serializable spec with an identical Arrow IPC wire format. This module
+plus a serializable spec sharing the same four Arrow IPC columns. This module
 holds the machinery both share; the two public modules subclass it so their
-names, wire format, and behaviour are unchanged.
+names and behaviour are unchanged.
+
+A subclass may widen its wire format with additional columns — `AttachOption`
+adds ``required`` — via :meth:`_SpecBase._extra_row` /
+:meth:`_SpecBase._extra_kwargs` and :meth:`_DescriptorBase.extra_spec_kwargs`.
+Because both directions read by column name, the widening is compatible in both
+directions: a reader ignores columns it doesn't know, and a reader whose peer
+never wrote the column falls back to the field default.
 """
 
 from __future__ import annotations
@@ -97,6 +104,24 @@ class _SpecBase:
         ]  # type: ignore[arg-type]
     )
 
+    def _extra_row(self) -> dict[str, Any]:
+        """Extra wire columns this spec contributes beyond the shared four.
+
+        Subclasses that widen :data:`ARROW_SCHEMA` return the matching values
+        here. Keys must exist in the subclass's schema.
+        """
+        return {}
+
+    @classmethod
+    def _extra_kwargs(cls, row: dict[str, object]) -> dict[str, Any]:
+        """Constructor kwargs read from the extra wire columns.
+
+        ``row`` is built from the *batch's* schema, so a column a peer never
+        wrote is simply absent — read with ``.get()`` and supply the default.
+        """
+        del row
+        return {}
+
     def serialize(self) -> bytes:
         """Serialize to Arrow IPC bytes."""
         # Serialize type as a single-field schema
@@ -116,6 +141,7 @@ class _SpecBase:
                     "description": self.desc,
                     "type": type_bytes,
                     "default_value": default_bytes,
+                    **self._extra_row(),
                 }
             ],
             schema=self.ARROW_SCHEMA,
@@ -147,6 +173,7 @@ class _SpecBase:
             desc=cast(str, row["description"]),
             type=data_type,
             default=default,
+            **cls._extra_kwargs(row),
         )
 
 
@@ -179,6 +206,15 @@ class _DescriptorBase:
             return self
         return getattr(type(obj), self._name, None)
 
+    def extra_spec_kwargs(self) -> dict[str, Any]:
+        """Extra keyword arguments this descriptor passes to its spec factory.
+
+        The base declaration carries only ``name``/``desc``/``type``/``default``;
+        a subclass adding a field returns it here so ``_extract_specs`` stays
+        shared.
+        """
+        return {}
+
 
 def _extract_specs[D: _DescriptorBase, S: _SpecBase](
     declaring_cls: type,
@@ -240,6 +276,7 @@ def _extract_specs[D: _DescriptorBase, S: _SpecBase](
                 desc=descriptor.desc,
                 type=arrow_type,
                 default=default,
+                **descriptor.extra_spec_kwargs(),
             )
         )
 

vgi/catalog/attach_option.py

@@ -12,13 +12,18 @@ the machinery in ``vgi.catalog._descriptor_spec``.
 """
 
 from dataclasses import dataclass
+from typing import Any, ClassVar
+
+import pyarrow as pa
 
 from vgi.catalog._descriptor_spec import _DescriptorBase, _extract_specs, _SpecBase
 
 __all__ = [
     "AttachOption",
     "AttachOptionSpec",
+    "MissingAttachOptionsError",
     "extract_attach_option_specs",
+    "validate_required_attach_options",
 ]
 
 
@@ -27,10 +32,48 @@ class AttachOptionSpec(_SpecBase):
     """Extracted attach-option metadata for catalog discovery serialization.
 
     The resolved form of an `AttachOption`. See
-    `_SpecBase` (in `vgi.catalog._descriptor_spec`) for the field and
-    wire-format definition.
+    `_SpecBase` (in `vgi.catalog._descriptor_spec`) for the shared fields and
+    wire format.
+
+    Attributes:
+        required: The caller must supply this option at ``ATTACH`` time. A
+            catalog that cannot be attached without it advertises that fact at
+            discovery, so a client can say so before attempting the attach
+            rather than surfacing a failure that reads like an empty catalog.
+            Mutually exclusive with ``default`` — an option that falls back to
+            a value is by definition satisfiable without the caller.
+        ARROW_SCHEMA: The shared four columns plus ``required``.
+
     """
 
+    required: bool = False
+
+    ARROW_SCHEMA: ClassVar[pa.Schema] = pa.schema(
+        [
+            *_SpecBase.ARROW_SCHEMA,
+            # Nullable, and appended last: a peer that predates this column
+            # reads the batch by name and simply doesn't see it.
+            pa.field("required", pa.bool_(), nullable=True),
+        ]
+    )
+
+    def __post_init__(self) -> None:
+        """Reject the contradictory ``required`` + ``default`` combination."""
+        if self.required and self.default is not None:
+            raise ValueError(
+                f"Attach option {self.name!r} is required but also declares a default "
+                f"({self.default!r}); an option with a default is always satisfiable "
+                "without the caller. Drop one."
+            )
+
+    def _extra_row(self) -> dict[str, Any]:
+        return {"required": self.required}
+
+    @classmethod
+    def _extra_kwargs(cls, row: dict[str, object]) -> dict[str, Any]:
+        # Absent column (older peer) and explicit null both mean "not required".
+        return {"required": bool(row.get("required") or False)}
+
 
 @dataclass
 class AttachOption(_DescriptorBase):
@@ -41,7 +84,72 @@ class AttachOption(_DescriptorBase):
     in the Annotated hint. See
     `_DescriptorBase` (in `vgi.catalog._descriptor_spec`) for the
     ``desc`` and ``arrow_type`` attributes.
+
+    Declare a required option by omitting the class-level assignment that would
+    otherwise supply its default::
+
+        class AttachOptions:
+            region: Annotated[str, AttachOption(desc="AWS region")] = "us-east-1"
+            api_key: Annotated[str, AttachOption(desc="API key", required=True)]
+
+    Attributes:
+        required: The caller must supply this option at ``ATTACH`` time.
+
+    """
+
+    required: bool = False
+
+    def extra_spec_kwargs(self) -> dict[str, Any]:
+        """Carry ``required`` through to the `AttachOptionSpec`."""
+        return {"required": self.required}
+
+
+class MissingAttachOptionsError(ValueError):
+    """Raised when ``catalog_attach`` omits options declared ``required``.
+
+    Carries the option names in :attr:`missing` so a caller can act on them
+    without parsing the message.
+
+    Attributes:
+        missing: Names of the required options that were not supplied.
+
+    """
+
+    missing: list[str]
+
+    def __init__(self, catalog_name: str, missing: list[str]) -> None:
+        """Build the error from the catalog name and the missing option names."""
+        self.missing = missing
+        joined = ", ".join(repr(name) for name in missing)
+        super().__init__(
+            f"Catalog {catalog_name!r} cannot be attached without the required "
+            f"option{'s' if len(missing) > 1 else ''} {joined}."
+        )
+
+
+def validate_required_attach_options(
+    catalog_name: str,
+    specs: list[AttachOptionSpec],
+    options: dict[str, Any],
+) -> None:
+    """Raise if any ``required`` spec has no corresponding entry in ``options``.
+
+    Option names are matched case-insensitively, mirroring DuckDB's handling of
+    ``ATTACH`` option keys.
+
+    Args:
+        catalog_name: Catalog being attached, for the error message.
+        specs: The attach-option specs the catalog declares.
+        options: Options the caller supplied at ``ATTACH`` time.
+
+    Raises:
+        MissingAttachOptionsError: If a required option was not supplied.
+
     """
+    supplied = {key.lower() for key in options}
+    missing = [spec.name for spec in specs if spec.required and spec.name.lower() not in supplied]
+    if missing:
+        raise MissingAttachOptionsError(catalog_name, missing)
 
 
 def extract_attach_option_specs(options_cls: type) -> list[AttachOptionSpec]:

vgi/catalog/catalog_interface.py

@@ -1414,6 +1414,71 @@ def serialize_column_statistics(
     return serialize_record_batch_bytes(batch, custom_metadata=custom_metadata)
 
 
+def _union_child_scalar(scalar: pa.Scalar | None) -> pa.Scalar | None:  # type: ignore[type-arg]
+    """Unwrap the typed child scalar from a sparse-union ``min``/``max`` cell.
+
+    A union cell is valid but wraps a null child whenever that row's statistic was
+    unknown, so both levels have to be checked.
+
+    Args:
+        scalar: The union cell to unwrap, or None.
+
+    Returns:
+        The typed child scalar, or None when the cell or its child is null.
+    """
+    if scalar is None or not scalar.is_valid:
+        return None
+    value = getattr(scalar, "value", None)
+    if value is None or not value.is_valid:
+        return None
+    return value  # type: ignore[no-any-return]
+
+
+def deserialize_column_statistics(data: bytes) -> list[ColumnStatistics]:
+    """Deserialize column statistics — the inverse of [`serialize_column_statistics`][].
+
+    Reads the sparse-union wire batch back into typed [`ColumnStatistics`][], so a client
+    can consume the bytes returned by ``catalog_table_column_statistics_get`` without
+    re-implementing the union layout. See
+    [`Client.table_column_statistics`][vgi.client.CatalogClientMixin.table_column_statistics] for the
+    wrapper that fetches and decodes in one call.
+
+    The optional ``cache_max_age_seconds`` travels in the IPC batch's custom metadata
+    rather than in the statistics themselves; read it with ``deserialize_record_batch``
+    if you need it.
+
+    Args:
+        data: IPC-serialized bytes of a statistics RecordBatch.
+
+    Returns:
+        Per-column statistics, in the order the worker serialized them. An empty list
+        when the batch carries no rows.
+    """
+    batch, _ = deserialize_record_batch(data)
+    names = batch.column("column_name")
+    mins = batch.column("min")
+    maxs = batch.column("max")
+    has_null = batch.column("has_null")
+    has_not_null = batch.column("has_not_null")
+    distinct = batch.column("distinct_count")
+    unicode_col = batch.column("contains_unicode")
+    max_len = batch.column("max_string_length")
+
+    return [
+        ColumnStatistics(
+            column_name=names[i].as_py(),
+            min=_union_child_scalar(mins[i]),
+            max=_union_child_scalar(maxs[i]),
+            has_null=bool(has_null[i].as_py()),
+            has_not_null=bool(has_not_null[i].as_py()),
+            distinct_count=distinct[i].as_py(),
+            contains_unicode=unicode_col[i].as_py(),
+            max_string_length=max_len[i].as_py(),
+        )
+        for i in range(batch.num_rows)
+    ]
+
+
 class CatalogInterface(ABC):
     """Provides an interface to manage catalogs, schemas, tables, and views for VGI.
 
@@ -2591,10 +2656,13 @@ class ReadOnlyCatalogInterface(CatalogInterface):
         ctx: "CallContext | None" = None,
     ) -> CatalogAttachResult:
         """Attach to the catalog. Version constraints are ignored by default."""
+        from vgi.catalog.attach_option import validate_required_attach_options
+
         del data_version_spec, implementation_version, ctx
         effective_name = self._effective_catalog_name
         if name != effective_name:
             raise ValueError(f"Unknown catalog: {name!r}. Available: {effective_name}")
+        validate_required_attach_options(effective_name, self.attach_option_specs, options)
 
         # Serialize settings and secret types for the attach result
         serialized_settings = [s.serialize() for s in self.settings]

vgi/client/catalog_mixin.py

@@ -53,6 +53,7 @@ from vgi.catalog import (
     AttachOpaqueData,
     CatalogAttachResult,
     CatalogInfo,
+    ColumnStatistics,
     FunctionInfo,
     MacroInfo,
     MacroType,
@@ -66,6 +67,7 @@ from vgi.catalog import (
     TransactionOpaqueData,
     ViewInfo,
 )
+from vgi.catalog.catalog_interface import deserialize_column_statistics
 from vgi.protocol import (
     CatalogAttachRequest,
     CatalogCreateRequest,
@@ -100,17 +102,30 @@ class CatalogClientMixin:
     factory).
 
     Attributes:
-        server_path: Worker shell command used for subprocess transport.
+        server_path: Worker command used for subprocess transport — a string to
+            be split, or an argv sequence taken as-is.
     """
 
     # Type hints for attributes expected from Client
-    server_path: str
+    server_path: str | Sequence[str]
     _transport: Literal["subprocess", "http", "tcp"]
     _base_url: str | None
     _tcp_host: str | None
     _tcp_port: int | None
     _external_location: Any | None
 
+    def _worker_argv(self) -> list[str]:
+        """The worker command as an argv list, however ``server_path`` was given.
+
+        A sequence is taken verbatim; a string is split. ``posix=False`` on Windows
+        keeps backslashes in paths from being read as escapes — at the cost of
+        leaving quotes attached to the token, which is why an argument carrying
+        spaces or quotes should be passed as a sequence element instead.
+        """
+        if isinstance(self.server_path, str):
+            return shlex.split(self.server_path, posix=sys.platform != "win32")
+        return list(self.server_path)
+
     def _get_or_create_httpx_client(self) -> Any:  # implemented by Client
         raise NotImplementedError
 
@@ -156,7 +171,7 @@ class CatalogClientMixin:
                 ) as proxy:
                     yield proxy
             else:
-                cmd = shlex.split(self.server_path, posix=sys.platform != "win32")
+                cmd = self._worker_argv()
                 with _catalog_pool.connect(VgiProtocol, cmd) as proxy:  # type: ignore[type-abstract]
                     yield proxy
         except RpcError as e:
@@ -572,6 +587,41 @@ class CatalogClientMixin:
                 transaction_opaque_data=transaction_opaque_data,
             ).to_optional()
 
+    def table_column_statistics(
+        self,
+        *,
+        attach_opaque_data: AttachOpaqueData,
+        transaction_opaque_data: TransactionOpaqueData | None = None,
+        schema_name: str,
+        name: str,
+    ) -> list[ColumnStatistics]:
+        """Fetch a table's column statistics, decoded.
+
+        Workers may inline statistics on [`TableInfo`][] (see
+        ``TableInfo.column_statistics``) or serve them lazily through this per-table
+        call, which is what ``TableInfo.supports_column_statistics`` advertises. Prefer
+        the inlined copy when present and fall back to this.
+
+        Args:
+            attach_opaque_data: The attachment ID from catalog_attach.
+            transaction_opaque_data: Optional transaction ID for transactional reads.
+            schema_name: The schema containing the table.
+            name: The table name.
+
+        Returns:
+            Per-column statistics, or an empty list when the worker has none for
+            this table.
+
+        """
+        with self._catalog_connect() as proxy:
+            data = proxy.catalog_table_column_statistics_get(
+                attach_opaque_data=attach_opaque_data,
+                schema_name=schema_name,
+                name=name,
+                transaction_opaque_data=transaction_opaque_data,
+            )
+        return deserialize_column_statistics(data) if data else []
+
     def table_create(
         self,
         *,