vgi-python 0.40.0__py3-none-any.whl → 0.41.0__py3-none-any.whl

vgi/_test_fixtures/catalog_contents.py

@@ -42,7 +42,9 @@ at least once (tables, a view, scalar / aggregate / table functions, scalar and
 table macros), split over two schemas (``main`` and ``extra``), so RPC
 accounting in ``vgi/test/sql/integration/catalog/catalog_contents*.test`` is
 deterministic. Served by ``vgi-fixture-worker`` and ``vgi-fixture-http``
-(MetaWorker), so the same tests run over every transport. Their functions
+(MetaWorker), so the same tests run over every transport. Every other SDK's
+fixture worker serves these six catalogs too, and the same ``.test`` files run
+against each of them, so the semantics here are the cross-SDK contract. Their functions
 re-use the example catalog's classes (``main.double`` ...), so every worker
 here sets ``route_unattached_calls = False``: they are reachable only through
 their own attach, and an unattached ``main.double`` still routes to the

vgi/client/__init__.py

@@ -45,7 +45,7 @@ from vgi.client.aggregate import (
     AggregateSession,
     AggregateStreamingSession,
 )
-from vgi.client.catalog_mixin import CatalogClientMixin, CatalogContents
+from vgi.client.catalog_mixin import CatalogClientMixin, CatalogContents, CatalogSnapshot
 from vgi.client.client import Client, ClientError, ResumableTableScan, ResumeUnsupported
 
 if TYPE_CHECKING:
@@ -57,6 +57,7 @@ __all__ = [
     "AggregateStreamingSession",
     "CatalogClientMixin",
     "CatalogContents",
+    "CatalogSnapshot",
     "Client",
     "ClientError",
     "OutputWriter",

vgi/client/catalog_mixin.py

@@ -42,7 +42,7 @@ import shlex
 import sys
 from collections.abc import Iterator, Sequence
 from contextlib import contextmanager
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from typing import Any, Literal, overload
 
 import pyarrow as pa
@@ -129,6 +129,55 @@ class CatalogContents:
     schemas: list[SchemaContentsInfo]
 
 
+@dataclass(frozen=True)
+class CatalogSnapshot:
+    """A whole catalog — every schema and everything in it — from [`CatalogClientMixin.load_catalog`][].
+
+    Hold on to it and pass it back as ``previous`` to revalidate: when it came
+    from ``catalog_contents`` with an etag, the next load sends
+    ``if_none_match`` and a ``not_modified`` answer returns this same content
+    without re-downloading it.
+
+    Attributes:
+        schemas: One [`SchemaContentsInfo`][] per schema, parents first.
+        catalog_version: The version the snapshot was taken at, or ``None``
+            when it was assembled from the per-schema RPCs (which carry none).
+        etag: The ``catalog_contents`` validator, or ``None`` when the worker
+            does not revalidate or the snapshot did not come from
+            ``catalog_contents``.
+        source: ``"catalog_contents"`` (one RPC) or ``"per_schema"``
+            (``catalog_schemas`` plus the per-schema ``catalog_schema_contents_*``
+            RPCs — the worker did not advertise ``catalog_contents``, the load
+            ran inside a transaction, or ``catalog_contents`` failed).
+        not_modified: ``True`` when this is ``previous`` confirmed current by a
+            ``not_modified`` answer.
+        fallback_reason: Why ``catalog_contents`` was advertised but not used
+            (its error message, or a protocol violation), else ``None``.
+    """
+
+    schemas: list[SchemaContentsInfo]
+    catalog_version: int | None
+    etag: str | None
+    source: Literal["catalog_contents", "per_schema"]
+    not_modified: bool = False
+    fallback_reason: str | None = None
+
+
+# ``SchemaInfo.estimated_object_count`` key for each per-schema kind; a count of
+# exactly 0 is a hard "none of this kind" guarantee, so the RPC is skipped (the
+# same rule the DuckDB extension and ``CatalogInterface.catalog_contents`` apply).
+_PER_SCHEMA_KINDS: tuple[tuple[str, SchemaObjectType, str], ...] = (
+    ("tables", SchemaObjectType.TABLE, "table"),
+    ("views", SchemaObjectType.VIEW, "view"),
+    ("scalar_functions", SchemaObjectType.SCALAR_FUNCTION, "scalar_function"),
+    ("aggregate_functions", SchemaObjectType.AGGREGATE_FUNCTION, "aggregate_function"),
+    ("table_functions", SchemaObjectType.TABLE_FUNCTION, "table_function"),
+    ("scalar_macros", SchemaObjectType.SCALAR_MACRO, "macro"),
+    ("table_macros", SchemaObjectType.TABLE_MACRO, "macro"),
+    ("indexes", SchemaObjectType.INDEX, "index"),
+)
+
+
 class CatalogClientError(Exception):
     """Error raised by catalog operations."""
 
@@ -492,13 +541,123 @@ class CatalogClientMixin:
             one [`SchemaContentsInfo`][] per schema, parents first.
 
         """
+        # Decode inside the connection block so a malformed item surfaces as
+        # CatalogClientError (which load_catalog falls back on), not a bare
+        # decode error.
         with self._catalog_connect() as proxy:
             response = proxy.catalog_contents(attach_opaque_data=attach_opaque_data, if_none_match=if_none_match)
-        return CatalogContents(
-            catalog_version=response.catalog_version,
-            etag=response.etag,
-            not_modified=response.not_modified,
-            schemas=[_decode_schema_contents(entry) for entry in response.schemas],
+            return CatalogContents(
+                catalog_version=response.catalog_version,
+                etag=response.etag,
+                not_modified=response.not_modified,
+                schemas=[_decode_schema_contents(entry) for entry in response.schemas],
+            )
+
+    def load_catalog(
+        self,
+        *,
+        attach: CatalogAttachResult,
+        previous: CatalogSnapshot | None = None,
+        transaction_opaque_data: TransactionOpaqueData | None = None,
+    ) -> CatalogSnapshot:
+        """Load every schema and all of its contents, in as few RPCs as the worker allows.
+
+        This is the client's whole-catalog enumeration. When the attach result
+        advertises ``supports_catalog_contents`` it is one ``catalog_contents``
+        call; otherwise — or inside a transaction, or when that call fails — it
+        is ``catalog_schemas`` plus the per-schema ``catalog_schema_contents_*``
+        RPCs (skipping a kind whose ``estimated_object_count`` is exactly 0).
+        ``catalog_contents`` is never sent to a worker that did not advertise it.
+
+        Revalidation: pass the snapshot from a previous load as ``previous``.
+        If it came from ``catalog_contents`` with an etag, the call sends
+        ``if_none_match``; a ``not_modified`` answer returns ``previous``'s
+        content (with ``not_modified=True`` and the current version), and a full
+        answer replaces it. A snapshot older than ``previous`` (a lagging
+        replica) is retried once, then the per-schema RPCs are used.
+
+        Args:
+            attach: The result of [`catalog_attach`][vgi.client.CatalogClientMixin.catalog_attach]
+                for this catalog.
+            previous: A snapshot already held, to revalidate instead of reload.
+            transaction_opaque_data: Load inside this transaction. ``catalog_contents``
+                returns only the committed catalog, so a transactional load always
+                uses the (transaction-aware) per-schema RPCs.
+
+        Returns:
+            A [`CatalogSnapshot`][]. ``source`` says which path served it and
+            ``fallback_reason`` why an advertised ``catalog_contents`` was not used.
+        """
+        attach_opaque_data = attach.attach_opaque_data
+        if not attach.supports_catalog_contents or transaction_opaque_data is not None:
+            return self._load_catalog_per_schema(attach_opaque_data, transaction_opaque_data, None)
+
+        if_none_match = previous.etag if previous is not None and previous.source == "catalog_contents" else None
+        known_version = previous.catalog_version if previous is not None else None
+        reason: str
+        for _attempt in range(2):
+            try:
+                contents = self.contents(attach_opaque_data=attach_opaque_data, if_none_match=if_none_match)
+            except CatalogClientError as e:
+                reason = str(e)
+                break
+            if contents.not_modified:
+                if previous is None or if_none_match is None or contents.schemas:
+                    reason = "catalog_contents answered not_modified to a request it could not match"
+                    break
+                return replace(
+                    previous,
+                    catalog_version=contents.catalog_version,
+                    etag=contents.etag if contents.etag is not None else previous.etag,
+                    not_modified=True,
+                )
+            # Versions are monotonic within a session; 0 means "unknown".
+            if known_version and contents.catalog_version and contents.catalog_version < known_version:
+                reason = (
+                    f"catalog_contents returned version {contents.catalog_version}, "
+                    f"older than the known version {known_version}"
+                )
+                continue
+            return CatalogSnapshot(
+                schemas=contents.schemas,
+                catalog_version=contents.catalog_version,
+                etag=contents.etag,
+                source="catalog_contents",
+            )
+        return self._load_catalog_per_schema(attach_opaque_data, None, reason)
+
+    def _load_catalog_per_schema(
+        self,
+        attach_opaque_data: AttachOpaqueData,
+        transaction_opaque_data: TransactionOpaqueData | None,
+        fallback_reason: str | None,
+    ) -> CatalogSnapshot:
+        """Assemble a snapshot from ``catalog_schemas`` and the per-schema RPCs."""
+        result: list[SchemaContentsInfo] = []
+        for schema in self.schemas(
+            attach_opaque_data=attach_opaque_data, transaction_opaque_data=transaction_opaque_data
+        ):
+            counts = schema.estimated_object_count or {}
+            kinds: dict[str, list[Any]] = {}
+            for attr, object_type, count_key in _PER_SCHEMA_KINDS:
+                if counts.get(count_key, 1) == 0:
+                    kinds[attr] = []
+                    continue
+                kinds[attr] = list(
+                    self.schema_contents(
+                        attach_opaque_data=attach_opaque_data,
+                        transaction_opaque_data=transaction_opaque_data,
+                        path=list(schema.path),
+                        type=object_type,
+                    )
+                )
+            result.append(SchemaContentsInfo(schema=schema, **kinds))
+        return CatalogSnapshot(
+            schemas=result,
+            catalog_version=None,
+            etag=None,
+            source="per_schema",
+            fallback_reason=fallback_reason,
         )
 
     def schema_get(

vgi/client/cli_catalog.py

@@ -11,6 +11,7 @@ catalog
 ├── create <name>           # Create a new catalog
 ├── drop <name>             # Drop a catalog
 ├── version                 # Get catalog version (--attach-opaque-data or --catalog)
+├── contents                # Every schema and its contents (catalog_contents when advertised)
 ├── schema                  # Schema operations
 │   ├── list/get/create/drop/contents
 ├── table                   # Table operations
@@ -163,6 +164,52 @@ def catalog_version(
     output_json({"version": version, "attach_opaque_data": bytes_to_hex(resolved_attach_opaque_data)})
 
 
+@catalog.command("contents")
+@click.option("--catalog", "catalog_name", required=True, help="Catalog name to attach")
+@click.option("--attach-options", default="{}", help="Attach options as JSON")
+@click.option("--worker", "-w", required=True, help="VGI worker command")
+def catalog_contents(catalog_name: str, attach_options: str, worker: str) -> None:
+    """List every schema and the names of everything in it.
+
+    Uses one ``catalog_contents`` call when the catalog advertises it, else
+    ``catalog_schemas`` plus the per-schema RPCs (see ``Client.load_catalog``).
+    """
+    client = Client(worker)
+    attach = client.catalog_attach(
+        name=catalog_name,
+        options=parse_json_option(attach_options, "--attach-options"),
+        data_version_spec=None,
+        implementation_version=None,
+    )
+    snapshot = client.load_catalog(attach=attach)
+    kinds = (
+        "tables",
+        "views",
+        "scalar_functions",
+        "aggregate_functions",
+        "table_functions",
+        "scalar_macros",
+        "table_macros",
+        "indexes",
+    )
+    output_json(
+        {
+            "source": snapshot.source,
+            "catalog_version": snapshot.catalog_version,
+            "etag": snapshot.etag,
+            "fallback_reason": snapshot.fallback_reason,
+            "schemas": [
+                {
+                    "path": list(entry.schema.path),
+                    "comment": entry.schema.comment,
+                    **{kind: [item.name for item in getattr(entry, kind)] for kind in kinds},
+                }
+                for entry in snapshot.schemas
+            ],
+        }
+    )
+
+
 # Add nested subcommand groups
 catalog.add_command(schema)
 catalog.add_command(table)

vgi/client/cli_utils.py

@@ -389,6 +389,7 @@ def catalog_attach_result_to_dict(result: CatalogAttachResult) -> dict[str, Any]
         "catalog_version": result.catalog_version,
         "attach_opaque_data_required": result.attach_opaque_data_required,
         "default_schema": result.default_schema,
+        "supports_catalog_contents": result.supports_catalog_contents,
         "settings": [bytes_to_hex(s) for s in result.settings],
         "resolved_data_version": result.resolved_data_version,
         "resolved_implementation_version": result.resolved_implementation_version,

vgi/client/client.py

@@ -54,6 +54,7 @@ Key methods
     client.catalog_attach(...)    — open a catalog
     client.schemas(...)           — list schemas
     client.schema_contents(...)   — list tables/views/functions/macros
+    client.load_catalog(...)      — whole catalog (catalog_contents when advertised)
     client.scalar_function(...)   — invoke a scalar
     client.table_function(...)    — invoke a table function
     client.table_function_plan(...) — plan a table function into named, redeemable splits

{vgi_python-0.40.0.dist-info → vgi_python-0.41.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.40.0
+Version: 0.41.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.40.0.dist-info → vgi_python-0.41.0.dist-info}/RECORD

@@ -45,7 +45,7 @@ vgi/_test_fixtures/bad_enum.py,sha256=b8l-afmXgGR6iqgIUiVeqHu0C0F8Of8ahEmdQGtqd1
 vgi/_test_fixtures/bad_protocol.py,sha256=fPU7rpXEgBxRtPHfuCa0O2AHNRRbZDllLfydHsAKPEg,2992
 vgi/_test_fixtures/cancellable.py,sha256=hetTRLyT4kkazPNcxAdfzj0pYzEmegMksamJbR0w3Ho,12151
 vgi/_test_fixtures/catalog.py,sha256=zjBKBgg4TKoDQ9SYIFj7sx6bOL3_aXHMJHSzTgk1fdA,28526
-vgi/_test_fixtures/catalog_contents.py,sha256=Hvt98vLMhdWwk9FvdtPx5GoZJ1ZLgiCFBA2NLEJohVY,11765
+vgi/_test_fixtures/catalog_contents.py,sha256=C9vxHil2kmsqoELiTq2jZINBOwONJ2KIIYrreNMEN6c,11934
 vgi/_test_fixtures/companion.py,sha256=Ya_rg2J4tXOYC7zBFOjTTh-4AYuRhJN5BT4tVZULQ-Q,6037
 vgi/_test_fixtures/copy_from.py,sha256=YAWametIhrKE_K_kRwx37QpKMY0yBWAordE5-cNVn4M,6134
 vgi/_test_fixtures/copy_to.py,sha256=fFHxPYVd0wfAfMEvpeBRijqUasBeoAGREY3kHmm_oK8,9667
@@ -133,17 +133,17 @@ vgi/catalog/duckdb_statistics.py,sha256=Bzw4UGO-zdKrontaCcRV2palO8G3XLCWh-AhZozv
 vgi/catalog/secret_type.py,sha256=1DoEMyQun9K5z53817rUuhG1YpHSEAsazEmubkRLVL0,3745
 vgi/catalog/setting.py,sha256=06QfgaAR-0BKflIJ0du6PGqA5BPMdrkwr6u7f4nddII,1846
 vgi/catalog/storage.py,sha256=DhngzywbhQUfLF7NjFyiPp4Sw2ovOhcUXAuW3rkBQD8,11935
-vgi/client/__init__.py,sha256=SXS_95QN2c68ssxEWwSvS7yfyq_2ORtDpf5YqSEwC0w,2507
+vgi/client/__init__.py,sha256=QOZGVkJWOfB8r4jjiwfU2q8Q97eWkLTnT6BK1jPZ2RE,2547
 vgi/client/aggregate.py,sha256=EAaI-4d1mnNjGORZvLf3wfVuu8RXBTFGVbMek0DeGDg,38482
-vgi/client/catalog_mixin.py,sha256=ft6S0vfhvG77qvXixF-7g5jjfDE9ClwL0TM2h0mvv_w,61691
+vgi/client/catalog_mixin.py,sha256=CMUl4AUDCFDerRLc6mieZau1JCx9CIAbG_hIRHsZRJs,69620
 vgi/client/cli.py,sha256=nahuRCjday0z74Gk9zxOxIOJX9n6fHrP7-CGwJoyTzU,22754
-vgi/client/cli_catalog.py,sha256=twjlhHGl7n2mNN4jc7DnDRQr-Zj9KLAF2eATIQnITHQ,5577
+vgi/client/cli_catalog.py,sha256=RVhSJLFKSBfVwLfEb_GLT8vnuliOwU6FzlsVUF7Rpsw,7308
 vgi/client/cli_schema.py,sha256=9G2vZiaAyqINItcDCoARdM04qNlDyJ3IXw3SuA6jgU8,7818
 vgi/client/cli_table.py,sha256=H1xrKZF5yYmSe55YSrLWXGIXPmuGuzQcncEBvqL43zs,26498
 vgi/client/cli_transaction.py,sha256=TDW0ZrBy8XM_eYSkYk5ewEsnozfzDJOlopd66Fm7OOE,3142
-vgi/client/cli_utils.py,sha256=_JoNiVu12NV4Pg2b1wtWFt0L0uIWJ9p9mBIfIu-QIuM,17387
+vgi/client/cli_utils.py,sha256=ykh7XK-yzcW6WGED2_DhPLIVPSyVG3KUdllV5ztBicw,17458
 vgi/client/cli_view.py,sha256=0qhxoS91-zFMjq8V_Fb0PNW-YDxERu8-zPpxSFEE-G8,8490
-vgi/client/client.py,sha256=cvBQHzM2Z2lamC-fkBAvZD4HuRxB03aUkEKFVo_E9GU,167312
+vgi/client/client.py,sha256=GSkcVXQzI0VCmeUf7WY4kR0ZqFtYmeb6EkenarBvX5c,167399
 vgi/client/errors.py,sha256=cSN65cyQMk3crwZCz1IdPErK3mtlEkTNpu-nuQUDOQE,1633
 vgi/http/__init__.py,sha256=hlOwOVcoZqmEKVGk7ganOdr5ryRSpEhLBF9sRD7BkYc,608
 vgi/http/demo_storage.py,sha256=830s-H61thozC3eEuqdnwCpYFfvoJRq9vjyuXa5OURo,8769
@@ -156,8 +156,8 @@ vgi/transactor/_duckdb_compat.py,sha256=sXVZ9JLKAQyGR1BjWczSwdQEavtr-TcZPoVZZnTr
 vgi/transactor/client.py,sha256=7DTeMksogsw6ANjQjGOPpKYrV76rg4_kGjktMJf54jg,4486
 vgi/transactor/protocol.py,sha256=v59IvrKnuvwOXvn_HEcBAbCzDVcx0akgKN0R1mChXg0,5034
 vgi/transactor/server.py,sha256=nzsZQxJZdgappdYX8okrFIrjFtUbdM_Hhskl1fZ2nDY,32553
-vgi_python-0.40.0.dist-info/METADATA,sha256=lw7UWSpVsGOISgdZ3GYe4Y0cQoL59BlTUsqiXlstBng,25953
-vgi_python-0.40.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
-vgi_python-0.40.0.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
-vgi_python-0.40.0.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
-vgi_python-0.40.0.dist-info/RECORD,,
+vgi_python-0.41.0.dist-info/METADATA,sha256=cYzmCmk6poHPPaMQ9OiIidOEntlJdzlD_Nr-RLbjoP4,25953
+vgi_python-0.41.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+vgi_python-0.41.0.dist-info/entry_points.txt,sha256=3Kz1vgodw3pOL_xjtSyDB55-ZRy-U2X-X_Bdr582x0Q,165
+vgi_python-0.41.0.dist-info/licenses/LICENSE,sha256=pbJb4zZasP6n5ifEV81wFu017TarjydaYVmGbHcehtY,6103
+vgi_python-0.41.0.dist-info/RECORD,,