vgi-python 0.45.0__py3-none-any.whl → 0.46.0__py3-none-any.whl

vgi/_test_fixtures/catalog_publication.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Serve standard fixtures with contents_probe published through a local HTTP object store.
+
+Run with the normal fixture worker arguments (stdio, --http, --unix). Object
+storage lives for this worker's lifetime; durable S3 restart coverage lives in
+tests/catalog/test_catalog_publication.py. Set VGI_CATALOG_PUBLICATION_PORT to
+inspect /stats from a separate process. Only use this local fixture in tests.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+import threading
+import uuid
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Any
+
+import pyarrow as pa
+from vgi_rpc.external import ClientExternalConfig, Compression
+
+from vgi._test_fixtures.catalog_contents import ContentsProbeCatalog
+from vgi.catalog.snapshot import CatalogSnapshotPolicy, MemoryCatalogPublicationRepository, SnapshotIdentity
+
+
+class _Storage:
+    def __init__(self) -> None:
+        self.objects: dict[str, tuple[bytes, str | None]] = {}
+        self.gets = 0
+        self.uploads = 0
+        self.base = ""
+
+    def upload(self, data: bytes, schema: pa.Schema, *, content_encoding: str | None = None) -> str:
+        del schema
+        key = "/" + uuid.uuid4().hex
+        self.objects[key] = data, content_encoding
+        self.uploads += 1
+        return self.base + key
+
+
+def main() -> None:
+    """Configure publication before invoking the ordinary fixture CLI."""
+    storage = _Storage()
+
+    class Handler(BaseHTTPRequestHandler):
+        def do_OPTIONS(self) -> None:
+            self.send_response(204)
+            self.send_header("Access-Control-Allow-Origin", "*")
+            self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")
+            self.send_header("Access-Control-Allow-Headers", "X-VGI-Accept-Encoding, Content-Type")
+            self.send_header("Cross-Origin-Resource-Policy", "cross-origin")
+            self.end_headers()
+
+        def do_GET(self) -> None:
+            if self.path == "/stats":
+                data = json.dumps({"uploads": storage.uploads, "object_gets": storage.gets}).encode()
+                encoding = None
+            elif self.path in storage.objects:
+                storage.gets += 1
+                data, encoding = storage.objects[self.path]
+            else:
+                self.send_error(404)
+                return
+            self.send_response(200)
+            self.send_header("Access-Control-Allow-Origin", "*")
+            self.send_header("Cross-Origin-Resource-Policy", "cross-origin")
+            self.send_header("Content-Length", str(len(data)))
+            if encoding:
+                self.send_header("Content-Encoding", encoding)
+            self.end_headers()
+            self.wfile.write(data)
+
+        def log_message(self, *args: Any) -> None:
+            pass
+
+    server = ThreadingHTTPServer(("127.0.0.1", int(os.environ.get("VGI_CATALOG_PUBLICATION_PORT", "0"))), Handler)
+    storage.base = f"http://127.0.0.1:{server.server_port}"
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    config = ClientExternalConfig(max_retries=0, url_validator=None)
+    ContentsProbeCatalog.catalog_snapshot_policy = CatalogSnapshotPolicy(
+        delivery="published",
+        repository=MemoryCatalogPublicationRepository(storage, profile="fixture", stable_urls=True),
+        identity=lambda request: SnapshotIdentity(
+            catalog_namespace="contents_probe",
+            source_revision="fixture-2.2",
+            visibility_scope="public-fixture",
+            catalog_version=request.catalog_version,
+        ),
+        compression=Compression(algorithm="gzip"),
+        fetch_config=config,
+    )
+    print(f"Catalog publication fixture storage: {storage.base}", file=sys.stderr)
+    try:
+        from vgi._test_fixtures.worker import main as fixture_main
+
+        fixture_main()
+    finally:
+        server.shutdown()
+        server.server_close()
+        thread.join()
+        config.fetch_config.close()
+
+
+if __name__ == "__main__":
+    main()

vgi/_test_fixtures/narrow_bind/worker.py

@@ -168,7 +168,6 @@ class NarrowBindCatalog(ReadOnlyCatalogInterface):
         for i, info in enumerate(infos):
             if info.path == ["main"]:
                 infos[i] = SchemaInfo(
-                    attach_opaque_data=info.attach_opaque_data,
                     path=list(info.path),
                     comment=info.comment,
                     tags=info.tags,

vgi/_test_fixtures/projection_repro/worker.py

@@ -381,7 +381,6 @@ class ProjReproCatalog(ReadOnlyCatalogInterface):
         for i, info in enumerate(infos):
             if info.path == ["main"]:
                 infos[i] = SchemaInfo(
-                    attach_opaque_data=info.attach_opaque_data,
                     path=list(info.path),
                     comment=info.comment,
                     tags=info.tags,

vgi/_test_fixtures/schema_reconcile/worker.py

@@ -544,7 +544,6 @@ class SchemaReconcileCatalog(ReadOnlyCatalogInterface):
         for i, info in enumerate(infos):
             if info.path == [_SCHEMA_NAME]:
                 infos[i] = SchemaInfo(
-                    attach_opaque_data=info.attach_opaque_data,
                     path=list(info.path),
                     comment=info.comment,
                     tags=info.tags,

vgi/_test_fixtures/writable/worker.py

@@ -566,7 +566,6 @@ class WritableCatalog(ReadOnlyCatalogInterface):
         # default of 1 (unknown / eager-load), which restores correct lookups.
         static_schemas = [
             SchemaInfo(
-                attach_opaque_data=s.attach_opaque_data,
                 path=list(s.path),
                 comment=s.comment,
                 tags=s.tags,

vgi/catalog/catalog_interface.py

@@ -10,7 +10,7 @@ import dataclasses
 import threading
 from abc import ABC, abstractmethod
 from collections.abc import Mapping, Sequence
-from dataclasses import dataclass, field
+from dataclasses import InitVar, dataclass, field
 from datetime import datetime
 from enum import Enum
 from typing import (
@@ -32,6 +32,7 @@ if TYPE_CHECKING:
     from vgi.catalog.descriptors import Catalog, Index, Macro, Schema, Table, View
     from vgi.catalog.secret_type import SecretTypeSpec
     from vgi.catalog.setting import SettingSpec
+    from vgi.catalog.snapshot import CatalogSnapshotPolicy
 
 import pyarrow as pa
 from vgi_rpc import ArrowSerializableDataclass, ArrowType
@@ -361,8 +362,10 @@ class SchemaInfo(CatalogObject, ArrowSerializableDataclass):
     """Information about a schema in a catalog.
 
     Attributes:
-        attach_opaque_data: The unique id for the attached catalog.
         path: Raw identifier components from the outermost schema to this schema.
+        attach_opaque_data: Deprecated constructor-only compatibility argument.
+            Discarded and never serialized; attachment state belongs to
+            ``CatalogAttachResult`` and subsequent request envelopes.
         estimated_object_count: Approximate population per object kind, keyed by
             the same names the C++ extension uses for its set-cache
             instrumentation: ``"table"``, ``"view"``, ``"scalar_function"``,
@@ -388,8 +391,9 @@ class SchemaInfo(CatalogObject, ArrowSerializableDataclass):
             current view does not.
     """
 
-    attach_opaque_data: AttachOpaqueData
     path: SchemaPath
+    # Constructor-only compatibility for existing catalog authors; never metadata.
+    attach_opaque_data: InitVar[AttachOpaqueData | None] = field(default=None, kw_only=True)
     estimated_object_count: dict[str, int] | None = None
 
 
@@ -800,11 +804,11 @@ class FunctionInfo(CatalogSchemaObject, ArrowSerializableDataclass):
 
     function_type: FunctionType
 
-    arguments: SerializedSchema
+    arguments: Annotated[SerializedSchema, ArrowType(pa.dictionary(pa.int32(), pa.binary()))]
 
-    output_schema: SerializedSchema
+    output_schema: Annotated[SerializedSchema, ArrowType(pa.dictionary(pa.int32(), pa.binary()))]
 
-    parameter_default_values: Annotated[pa.RecordBatch | None, ArrowType(pa.binary())] = None
+    parameter_default_values: Annotated[pa.RecordBatch | None, ArrowType(pa.dictionary(pa.int32(), pa.binary()))] = None
 
     stability: FunctionStability | None = None
     null_handling: NullHandling | None = None
@@ -1754,6 +1758,9 @@ class CatalogInterface(ABC):
     #: ``False``.
     catalog_contents_attach_independent: ClassVar[bool] = False
 
+    #: Optional framework publication policy; ordinary delivery when unset.
+    catalog_snapshot_policy: ClassVar["CatalogSnapshotPolicy | None"] = None
+
     @property
     def interface_feature_flags(self) -> set[str]:
         """Get the feature flags supported by this [`CatalogInterface`][].

vgi/catalog/contents_codec.py

@@ -0,0 +1,223 @@
+"""Catalog-wide metadata collections for the 2.2 wire contract.
+
+The author-facing inventory stays grouped by schema. Collection encoders reuse
+the ordinary record conversion, and readers consume every row of every batch.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Iterable, Sequence
+from typing import TYPE_CHECKING, Any
+
+import pyarrow as pa
+from vgi_rpc import ArrowSerializableDataclass
+from vgi_rpc.utils import new_ipc_stream
+
+from vgi.catalog.catalog_interface import (
+    FunctionInfo,
+    FunctionType,
+    IndexInfo,
+    MacroInfo,
+    MacroType,
+    SchemaContentsInfo,
+    SchemaInfo,
+    TableInfo,
+    ViewInfo,
+)
+from vgi.schema_path import schema_path_key
+
+if TYPE_CHECKING:
+    from vgi.protocol import CatalogContentsResponse
+
+CATEGORY_TYPES: dict[str, type[Any]] = {
+    "schemas": SchemaInfo,
+    "tables": TableInfo,
+    "views": ViewInfo,
+    "functions": FunctionInfo,
+    "macros": MacroInfo,
+    "indexes": IndexInfo,
+}
+
+
+def encode_collection[T: ArrowSerializableDataclass](
+    record_type: type[T], records: Iterable[T | bytes], *, batch_size: int = 1024
+) -> bytes | None:
+    """Encode records directly, accepting legacy singleton IPC bytes as an adapter.
+
+    Each batch builds its own dictionaries; IPC dictionary replacements are
+    supported. Raw IPC byte strings are never concatenated.
+    """
+    from vgi.protocol import _with_sorted_maps
+
+    if batch_size <= 0:
+        raise ValueError("batch_size must be positive")
+    sink = pa.BufferOutputStream()
+    rows: list[dict[str, object]] = []
+    count = 0
+    with new_ipc_stream(sink, record_type.ARROW_SCHEMA) as writer:
+        for record in records:
+            if isinstance(record, bytes):
+                decoded = decode_collection(record_type, record)
+                if len(decoded) != 1:
+                    raise ValueError("legacy catalog item must contain exactly one row")
+                record = decoded[0]
+            if not isinstance(record, record_type):
+                raise TypeError(f"expected {record_type.__name__} catalog record")
+            rows.append(_with_sorted_maps(record)._to_row_dict())
+            count += 1
+            if len(rows) >= batch_size:
+                writer.write_batch(pa.RecordBatch.from_pylist(rows, schema=record_type.ARROW_SCHEMA))
+                rows.clear()
+        if rows:
+            writer.write_batch(pa.RecordBatch.from_pylist(rows, schema=record_type.ARROW_SCHEMA))
+    return sink.getvalue().to_pybytes() if count else None
+
+
+def decode_collection[T: ArrowSerializableDataclass](record_type: type[T], data: bytes | None) -> list[T]:
+    """Validate one typed collection and decode all rows, including later batches."""
+    if data is None:
+        return []
+    result: list[T] = []
+    source = pa.BufferReader(data)
+    with pa.ipc.open_stream(source) as reader:
+        if not reader.schema.equals(record_type.ARROW_SCHEMA, check_metadata=False):
+            raise ValueError(f"catalog {record_type.__name__} collection has an incompatible schema")
+        for batch in reader:
+            batch.validate(full=True)
+            for column, field in zip(batch.columns, batch.schema, strict=True):
+                for value in column.to_pylist():
+                    _validate_nullability(value, field, record_type.__name__)
+            result.extend(record_type.deserialize_from_batch(batch.slice(i, 1)) for i in range(batch.num_rows))
+    if source.tell() != len(data):
+        raise ValueError("catalog collection has trailing bytes after its IPC stream")
+    return result
+
+
+def _validate_nullability(value: Any, field: pa.Field[pa.DataType], path: str) -> None:
+    if value is None:
+        if not field.nullable:
+            raise ValueError(f"catalog {path}.{field.name} contains nulls")
+        return
+    dtype = field.type
+    if pa.types.is_dictionary(dtype):
+        return
+    if pa.types.is_struct(dtype):
+        for child in dtype:
+            _validate_nullability(value.get(child.name), child, path + "." + field.name)
+    elif pa.types.is_list(dtype) or pa.types.is_large_list(dtype) or pa.types.is_fixed_size_list(dtype):
+        for item in value:
+            _validate_nullability(item, dtype.value_field, path + "." + field.name)
+    elif pa.types.is_map(dtype):
+        for key, item in value:
+            _validate_nullability(key, dtype.key_field, path + "." + field.name)
+            _validate_nullability(item, dtype.item_field, path + "." + field.name)
+
+
+def encode_inventory(infos: Sequence[SchemaContentsInfo]) -> dict[str, bytes | None]:
+    """Validate schema membership and encode all six categories across schemas."""
+    ordered = sorted(infos, key=lambda c: len(c.schema.path))
+    paths = [schema_path_key(c.schema.path) for c in ordered]
+    if len(paths) != len(set(paths)):
+        raise ValueError("catalog_contents returned duplicate schema paths")
+    for path in paths:
+        if len(path) > 1 and path[:-1] not in paths:
+            raise ValueError(f"catalog_contents returned schema path {list(path)!r} without its parent")
+    grouped: dict[str, list[Any]] = {kind: [] for kind in CATEGORY_TYPES}
+    source_kinds = {
+        "tables": "tables",
+        "views": "views",
+        "indexes": "indexes",
+        "scalar_functions": "functions",
+        "aggregate_functions": "functions",
+        "table_functions": "functions",
+        "scalar_macros": "macros",
+        "table_macros": "macros",
+    }
+    for info in ordered:
+        grouped["schemas"].append(info.schema)
+        for source, target in source_kinds.items():
+            for item in getattr(info, source):
+                if isinstance(item, bytes):
+                    decoded = decode_collection(CATEGORY_TYPES[target], item)
+                    if len(decoded) != 1:
+                        raise ValueError("legacy catalog item must contain exactly one row")
+                    item = decoded[0]
+                expected_types: dict[str, set[Any]] = {
+                    "scalar_functions": {FunctionType.SCALAR},
+                    "aggregate_functions": {FunctionType.AGGREGATE},
+                    "table_functions": {FunctionType.TABLE, FunctionType.TABLE_BUFFERING},
+                    "scalar_macros": {MacroType.SCALAR},
+                    "table_macros": {MacroType.TABLE},
+                }
+                expected = expected_types.get(source)
+                actual = getattr(item, "function_type", getattr(item, "macro_type", None))
+                if expected is not None and actual not in expected:
+                    raise ValueError(f"catalog_contents {source} item has the wrong subtype")
+                if schema_path_key(item.schema_path) != schema_path_key(info.schema.path):
+                    raise ValueError(f"catalog_contents {source} item belongs to a different schema")
+                grouped[target].append(item)
+    return {kind: encode_collection(cls, grouped[kind]) for kind, cls in CATEGORY_TYPES.items()}
+
+
+def decode_inventory(
+    response: CatalogContentsResponse, *, if_none_match: str | None = None
+) -> list[SchemaContentsInfo]:
+    """Validate the response and restore the SDK's grouped inventory."""
+    if response.not_modified:
+        if response.etag is None or response.etag != if_none_match:
+            raise ValueError("catalog_contents not_modified requires a matching non-null ETag")
+        if any(getattr(response, kind) is not None for kind in CATEGORY_TYPES):
+            raise ValueError("catalog_contents not_modified must have null categories")
+        return []
+    schemas = decode_collection(SchemaInfo, response.schemas)
+    result: dict[tuple[str, ...], SchemaContentsInfo] = {}
+    for schema in schemas:
+        path = schema_path_key(schema.path)
+        if path in result:
+            raise ValueError("catalog_contents returned duplicate schema paths")
+        if len(path) > 1 and path[:-1] not in result:
+            raise ValueError("catalog_contents schema parent must precede its child")
+        result[path] = SchemaContentsInfo(
+            schema=schema,
+            tables=[],
+            views=[],
+            scalar_functions=[],
+            aggregate_functions=[],
+            table_functions=[],
+            scalar_macros=[],
+            table_macros=[],
+            indexes=[],
+        )
+    for category, cls in CATEGORY_TYPES.items():
+        if category == "schemas":
+            continue
+        for item in decode_collection(cls, getattr(response, category)):
+            path = schema_path_key(item.schema_path)
+            if path not in result:
+                raise ValueError(f"catalog_contents {category} references an unknown schema")
+            destination = category
+            if category == "functions":
+                destination = {
+                    FunctionType.SCALAR: "scalar_functions",
+                    FunctionType.AGGREGATE: "aggregate_functions",
+                    FunctionType.TABLE: "table_functions",
+                    FunctionType.TABLE_BUFFERING: "table_functions",
+                }[item.function_type]
+            elif category == "macros":
+                destination = {MacroType.SCALAR: "scalar_macros", MacroType.TABLE: "table_macros"}[item.macro_type]
+            getattr(result[path], destination).append(item)
+    return list(result.values())
+
+
+def contents_digest(categories: dict[str, bytes | None]) -> str:
+    """Hash category identity and complete stream bytes with unambiguous lengths."""
+    digest = hashlib.sha256(b"vgi.catalog.contents.2.2\0")
+    for kind in CATEGORY_TYPES:
+        data = categories[kind]
+        digest.update(kind.encode() + b"\0")
+        digest.update(b"\0" if data is None else b"\1")
+        if data is not None:
+            digest.update(len(data).to_bytes(8, "little"))
+            digest.update(data)
+    return digest.hexdigest()

vgi/catalog/contract.py

@@ -0,0 +1,22 @@
+"""Canonical fingerprint of the nested catalog wire contract for all SDKs."""
+
+import hashlib
+from functools import lru_cache
+
+from vgi_rpc.rpc import rpc_methods
+from vgi_rpc.rpc._protocol_hash import canonical_json, compute_protocol_hash
+from vgi_rpc.rpc._type_tokens import schema_tokens
+
+from vgi.catalog.contents_codec import CATEGORY_TYPES
+from vgi.protocol import CatalogContentsResponse, VgiProtocol
+
+
+@lru_cache(maxsize=1)
+def catalog_contract_hash() -> str:
+    """Include nested binary schemas, which the RPC method hash alone cannot see."""
+    description = {
+        "rpc": compute_protocol_hash(VgiProtocol.protocol_name, rpc_methods(VgiProtocol)),
+        "response": schema_tokens(CatalogContentsResponse.ARROW_SCHEMA),
+        "categories": {kind: schema_tokens(cls.ARROW_SCHEMA) for kind, cls in CATEGORY_TYPES.items()},
+    }
+    return hashlib.sha256(b"vgi.catalog.contract.v1|" + canonical_json(description)).hexdigest()

vgi/catalog/descriptors.py

@@ -899,6 +899,9 @@ class Schema:
     def to_schema_info(self, attach_opaque_data: AttachOpaqueData) -> SchemaInfo:
         """Convert to [`SchemaInfo`][] for catalog response.
 
+        ``attach_opaque_data`` is accepted for compatibility and discarded;
+        schema metadata does not contain attachment state.
+
         Populates ``estimated_object_count`` from the declared population so
         the C++ extension's eager-load gate can choose between bulk
         ``LoadEntries`` and per-name single-entry RPCs without an extra round
@@ -923,7 +926,6 @@ class Schema:
         for func in self.functions:
             function_counts[func.get_metadata().function_type] += 1
         return SchemaInfo(
-            attach_opaque_data=attach_opaque_data,
             path=list(self.path),
             comment=self.comment,
             tags=dict(self.tags),