vgi-python 0.44.0__tar.gz → 0.46.0__tar.gz

{vgi_python-0.44.0 → vgi_python-0.46.0}/.github/workflows/ci.yml

@@ -129,7 +129,7 @@ jobs:
   hosted-protocols:
     # vgi-rpc's cross-SDK hosted-protocols group: the fixture workers must host
     # vgi.v2 then conformance.Secondary.v1 on stdio, unix and HTTP, and pass the
-    # Identity.v1 groups over HTTP (vgi-fixture-http --identity).
+    # Identity.v2 groups over HTTP (vgi-fixture-http --identity).
     name: Hosted-protocols conformance
     runs-on: ubuntu-latest
     steps:

{vgi_python-0.44.0 → vgi_python-0.46.0}/.github/workflows/integration.yml

@@ -1,13 +1,12 @@
 # Runs the canonical Query-farm/vgi integration sqllogictest suite against the
-# Python example worker on every push / PR. Rather than building the C++ extension
-# from source, it drives a prebuilt standalone `haybarn-unittest` and installs the
-# SIGNED vgi extension from the Haybarn community channel + its deps from core.
-# The .test files come from a pinned Query-farm/vgi checkout; ci/run-integration.sh
-# + ci/preprocess-require.awk adapt them for the standalone runner. See ci/README.md.
+# Python example worker on every push / PR. The native runner comes from the
+# Query-farm/vgi Actions build of the same commit as the test suite and statically
+# links the matching extension. Public package releases remain separate.
+# ci/run-integration.sh also supports the published standalone runner locally.
 #
 # (The separate ci.yml covers lint / type-check / unit + conformance tests / docs.
 # Those drive the pure-Python in-process Client; this workflow is the only place
-# the worker is exercised through the real published DuckDB extension over the wire.)
+# the worker is exercised through the real DuckDB extension over the wire.)
 name: Integration suite
 
 on:
@@ -19,20 +18,16 @@ on:
 
 permissions:
   contents: read
+  actions: read
 
 concurrency:
   group: integration-${{ github.ref }}
   cancel-in-progress: true
 
 env:
-  # The Query-farm/vgi ref whose test/sql/integration suite we run. For now this
-  # tracks the latest `main` (resolved at checkout time) so the Python port is
-  # always validated against the newest C++ test suite. Pin to a specific commit
-  # SHA here if/when reproducible, deliberate bumps are wanted (see ci/README.md).
-  VGI_REF: main
-  # The Haybarn release providing the prebuilt haybarn-unittest binary. Must be
-  # ABI-compatible with the community-published vgi extension.
-  HAYBARN_RELEASE: haybarn-v1.5.5-rc1
+  # The test suite and native Actions artifact must come from this exact ref.
+  # The downloader verifies its provenance against the checked-out commit.
+  VGI_REF: 3a2249f815ff6e2a8b5289a3e6284326084e3630
   # CI is not a user: its ATTACHes should not land in Query.Farm's adoption data.
   # The vgi extension fires a telemetry event per successful ATTACH, and this
   # suite performs several hundred of them per lane, so opting out keeps the
@@ -81,18 +76,13 @@ jobs:
       - name: Install dependencies (vgi + fixtures + all extras)
         run: uv sync --all-extras --python 3.14
 
-      - name: Download haybarn-unittest
+      - name: Download the matching native integration runner
         run: |
-          gh release download "$HAYBARN_RELEASE" \
-            --repo Query-farm-haybarn/haybarn \
-            --pattern 'haybarn_unittest-linux-amd64.zip' \
-            --output /tmp/haybarn-unittest.zip --clobber
-          unzip -o -q /tmp/haybarn-unittest.zip -d /tmp/haybarn-unittest
-          UNITTEST=$(find /tmp/haybarn-unittest -name 'haybarn-unittest' -type f | head -1)
-          chmod +x "$UNITTEST"
-          echo "HAYBARN_UNITTEST=$UNITTEST" >> "$GITHUB_ENV"
+          python3 "$VGI_SRC/scripts/ci_fetch_native_runner.py" \
+            --source "$VGI_SRC" --output "$RUNNER_TEMP/vgi-native"
         env:
           GH_TOKEN: ${{ github.token }}
+          VGI_SRC: ${{ github.workspace }}/vgi-upstream
 
       - name: Run integration suite (${{ matrix.lane }})
         run: ci/run-integration.sh

{vgi_python-0.44.0 → vgi_python-0.46.0}/.github/workflows/tailnet-integration.yml

@@ -33,12 +33,12 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - name: Checkout unreleased vgi-rpc identity layer
+      - name: Checkout vgi-rpc (Identity.v2)
         uses: actions/checkout@v7
         with:
           repository: Query-farm/vgi-rpc-python
           path: .tailnet/vgi-rpc
-          ref: 04f19147625b4f643b38a687151d8233daa8997c
+          ref: e04ddfa56c19b4cd7cc1e3729d165c763b0d98c7  # v0.52.0 (Identity.v2)
 
       - name: Check kernel networking support
         run: |

{vgi_python-0.44.0 → vgi_python-0.46.0}/CLAUDE.md

@@ -302,7 +302,7 @@ vgi-client --input data.parquet --function sum_all_columns --worker vgi-fixture-
 | `VGI_OAUTH_DEVICE_CODE_CLIENT_SECRET` | Client secret for device-code flow (optional, URL-safe chars only) |
 | `VGI_OAUTH_USE_ID_TOKEN` | When `1`/`true`/`yes`, clients use OIDC `id_token` as Bearer instead of `access_token` |
 | `VGI_RPC_GRANT_KEYS` | Comma-separated base64 32-byte sealed-grant keys (first mints, all verify); enables grants as HTTP bearers. Also `VGI_RPC_GRANT_AUDIENCE`, `VGI_RPC_GRANT_MAX_TTL_SECONDS` |
-| `VGI_INTROSPECT_PRINCIPALS` | Comma-separated principals permitted to call `vgi_rpc.Identity.v1`'s `introspect_token`. Required — no permissive default — whenever the worker implements `resolve_token()` (see below) |
+| `VGI_INTROSPECT_PRINCIPALS` | Comma-separated principals permitted to call `vgi_rpc.Identity.v2`'s `introspect_token`. Required — no permissive default — whenever the worker implements `resolve_token()` (see below) |
 | `VGI_WORKER_ACCESS_LOG_SAMPLE` | Fraction of *successful* calls to keep in the access log, `0.0`–`1.0`. Errors are always kept; the decision is per call, so every record of one stream shares a fate |
 | `VGI_WORKER_ACCESS_LOG_ASYNC` | When `1`/`true`/`yes`, emit access-log records from a listener thread. Bounded queue; full means drop, and a crash loses whatever is queued |
 | `VGI_WORKER_ACCESS_LOG_QUEUE_SIZE` | Bound on the async access-log queue (default 10000) |
@@ -426,7 +426,7 @@ Also available on `Worker.main --http` and the fixture server. Under
 
 ### Token Introspection (`resolve_token`)
 
-A worker may optionally host the `vgi_rpc.Identity.v1` protocol, whose
+A worker may optionally host the `vgi_rpc.Identity.v2` protocol, whose
 `introspect_token` method resolves an opaque bearer credential to a principal —
 for a reverse proxy that terminates the only public listener and must know the
 caller's identity before it can authorize anything.
@@ -509,7 +509,9 @@ one. Allowlist rules are unchanged. See `docs/authentication.md` and
 
 ### Attach Tickets (`vgi.attach_tickets.v1`)
 
-`seal_attach` seals a user's ATTACH (catalog name, options incl. secrets,
+Protocol version `2.0.0`: `AttachTicket.expires_at` is nullable
+`timestamp[us, tz="UTC"]` (`datetime | None`, null = no expiry); `1.0.0` was a
+float64 with `+inf`. `seal_attach` seals a user's ATTACH (catalog name, options incl. secrets,
 version specs) into a `vgia1.` ticket under `VGI_SIGNING_KEY`, AAD
 `"vgi.attach_ticket.v1\0" + principal` (principal only, not domain). A runner
 holding the user's grant reattaches with the single option `vgi_attach_ticket`;

{vgi_python-0.44.0 → vgi_python-0.46.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.44.0
+Version: 0.46.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python
@@ -162,7 +162,7 @@ Requires-Dist: httpx2>=2.12.0
 Requires-Dist: platformdirs
 Requires-Dist: pyarrow
 Requires-Dist: typer>=0.9
-Requires-Dist: vgi-rpc>=0.51.1
+Requires-Dist: vgi-rpc>=0.52.0
 Provides-Extra: azure
 Requires-Dist: azure-identity>=1.16.0; extra == 'azure'
 Requires-Dist: pymssql>=2.3.0; extra == 'azure'
@@ -189,6 +189,8 @@ Requires-Dist: vgi-rpc[otel]; extra == 'dev'
 Requires-Dist: vgi-rpc[sentry]; extra == 'dev'
 Provides-Extra: duckdb
 Requires-Dist: duckdb==1.5.5; extra == 'duckdb'
+Provides-Extra: external
+Requires-Dist: vgi-rpc[external]; extra == 'external'
 Provides-Extra: fixtures
 Requires-Dist: vgi-fixtures; extra == 'fixtures'
 Provides-Extra: granian

{vgi_python-0.44.0 → vgi_python-0.46.0}/ci/README.md

@@ -4,19 +4,18 @@
 runs the canonical [Query-farm/vgi](https://github.com/Query-farm/vgi)
 integration sqllogictest suite against this repo's Python example worker on
 every push / PR. The same `.test` files run against the Python, Java, and Go
-ports, so a green run here is real wire-compatibility evidence — it exercises
-the worker through the *published* DuckDB extension, not a mock, and not the
-pure-Python in-process `Client` that the conformance tests in `ci.yml` use.
+ports, so a green run here verifies the worker through the native DuckDB
+extension over the wire. The native runner and test suite come from the same
+VGI commit. Public package releases remain separate from this development check.
 
 (The separate [`ci.yml`](../.github/workflows/ci.yml) covers lint / mypy / unit
 + conformance tests / docs / LocalStack S3 offload.)
 
 ## How it works (no C++ build)
 
-Rather than building the vgi DuckDB extension from source (which needs the
-Haybarn vcpkg pipeline), CI drives a **prebuilt** standalone `haybarn-unittest`
-(the DuckDB/Haybarn sqllogictest runner, published in Haybarn's releases) and
-installs the **signed** vgi extension from the Haybarn community channel:
+CI downloads the native integration runner built by Query-farm/vgi Actions.
+It links vgi, httpfs, json and parquet, so the worker's wire contract is checked
+against the matching client without waiting for a community extension release.
 
 1. **Install the workers** — `uv sync --all-extras` installs the `vgi-fixtures`
    workspace member, which puts the fixture console scripts
@@ -27,17 +26,27 @@ installs the **signed** vgi extension from the Haybarn community channel:
    `--http`, and the launcher's `--unix`, so one install covers every lane.
 2. **Checkout the test suite** — `Query-farm/vgi` at a pinned commit; its
    `test/sql/integration/*.test` files are the suite.
-3. **Download the runner** — `haybarn_unittest-linux-amd64.zip` from the pinned
-   Haybarn release.
-4. **Preprocess** — the standalone runner links none of the extensions the tests
-   gate on, so [`preprocess-require.awk`](preprocess-require.awk) rewrites each
-   `require <ext>` into an explicit signed `INSTALL <ext> FROM {community,core};
-   LOAD <ext>;`. `require-env` and everything else pass through.
-5. **Run** — [`run-integration.sh`](run-integration.sh) stages the preprocessed
-   tree, wires the `VGI_*_WORKER` env vars at the `.venv/bin` scripts for the
-   selected transport, `FORCE INSTALL`s the vgi extension (so the run uses what
-   users can install today), then runs the suite in a single `haybarn-unittest`
-   invocation. Any failed assertion exits non-zero and fails the job.
+3. **Download the runner** — the pinned checkout's `ci_fetch_native_runner.py`
+   retrieves its Actions artifact and checks its provenance against that exact
+   source commit. It exports `HAYBARN_UNITTEST`, `HAYBARN_CLI` and
+   `RUNNER_LINKS_EXTENSIONS=1` for the next step.
+4. **Stage** — [`run-integration.sh`](run-integration.sh) copies the selected
+   tests unchanged when extensions are linked into the runner. Their normal
+   `require` gates load linked extensions; unavailable optional extensions keep
+   their normal skip behavior.
+5. **Run** — the harness wires the `VGI_*_WORKER` environment variables to the
+   `.venv/bin` scripts for the selected transport and executes the suite, with
+   `simple_writable` in its own process. Native mode passes `--test-dir` so the
+   runner discovers the staged tree instead of its compiled source directory.
+   Failed assertions, empty runs, fatal signals and dead shared HTTP workers
+   fail the job. The no-error-skip configuration keeps HTTP errors from silently
+   skipping tests.
+
+For local checks against the published extension, leave
+`RUNNER_LINKS_EXTENSIONS` unset or set it to `0`. This preserves the standalone
+runner path: [`preprocess-require.awk`](preprocess-require.awk) converts
+`require` gates to signed `INSTALL` and `LOAD` statements, and the harness
+warms the community/core extension cache before running.
 
 The env wiring mirrors the vgi repo's Makefile `test_subprocess` / `test_shm` /
 `test_launcher` / `test_http` targets — the canonical local way to run this
@@ -77,9 +86,9 @@ matrix; `stdio` remains available for local compatibility testing:
   `test_launcher`), so those tests skip here; run the local stdio lane to cover
   them. One deliberate divergence from `test_launcher`: see
   `filter_echo_partitioned.test` below.
-- **`http`** — the whole suite over the stateless HTTP transport. Staging
-  injects `LOAD httpfs` before each worker ATTACH (the prebuilt binary doesn't
-  statically link httpfs).
+- **`http`** — the whole suite over the stateless HTTP transport. The native
+  runner links httpfs; standalone mode injects `LOAD httpfs` before worker
+  attachments.
 
 The `simple_writable/*.test` write-path tests (INSERT/UPDATE/DELETE/RETURNING)
 run on **every** lane against `VGI_SIMPLE_WRITABLE_WORKER`
@@ -147,36 +156,32 @@ even though the fixture (`vgi-fixture-bad-enum-worker`) is installed.
 ```bash
 uv sync --all-extras
 VGI_SRC=~/Development/vgi \
-HAYBARN_UNITTEST=/path/to/haybarn-unittest \
+HAYBARN_UNITTEST=~/Development/vgi/build/release/test/unittest \
+RUNNER_LINKS_EXTENSIONS=1 \
 TRANSPORT=stdio \
   ci/run-integration.sh
 ```
 
-Download `haybarn-unittest` for your platform from the pinned Haybarn release:
+To check the published extension instead, download `haybarn-unittest` for your
+platform from a compatible Haybarn release and leave `RUNNER_LINKS_EXTENSIONS=0`:
 
 ```bash
 gh release download "$HAYBARN_RELEASE" --repo Query-farm-haybarn/haybarn \
   --pattern 'haybarn_unittest-*.zip'
 ```
 
-For day-to-day local work against a hand-built C++ extension, prefer the vgi
-repo's own targets (`make test_subprocess`, `make test_http`, …) /
-`scripts/run_all_tests.sh` as described in `CLAUDE.md`; this harness exists for
-CI, where building the extension from source is impractical.
+The vgi repo's own targets (`make test_subprocess`, `make test_http`, …) and
+`scripts/run_all_tests.sh` remain available as described in `CLAUDE.md`.
 
 ## Version pins (and their coupling)
 
-Two pins live in the workflow's `env:` block:
+The workflow's `VGI_REF` pins the test suite and the native Actions artifact:
 
 | Pin | What | Why |
 |-----|------|-----|
-| `VGI_REF` | the `Query-farm/vgi` ref supplying the `.test` files | **for now tracks `main`** (latest at checkout time) so the port is always validated against the newest suite; switch to a commit SHA for reproducible, deliberate bumps |
-| `HAYBARN_RELEASE` | the Haybarn release supplying `haybarn-unittest` | must be ABI-compatible with the community vgi extension |
-
-**The coupling to know about:** the vgi extension is pulled live from the
-community channel (`FORCE INSTALL vgi FROM community`), which always serves the
-*currently published* build — it is not version-pinned here. So CI verifies the
-worker against what users can install today. If `VGI_REF` points at a commit
-whose tests exercise a protocol feature the published extension doesn't yet ship
-(or vice-versa), that test can fail or skip — bump `VGI_REF` deliberately and
-re-validate against the current community extension.
+| `VGI_REF` | the `Query-farm/vgi` commit supplying the tests and native runner | the downloader requires an artifact built from that exact commit |
+
+The optional standalone mode still installs the currently published community
+extension. Use a compatible test suite and worker when checking that release;
+an unreleased minor wire revision requires the matching native development
+runner. Changing the CI pin does not publish an extension or an SDK package.

{vgi_python-0.44.0 → vgi_python-0.46.0}/ci/run-integration.sh

@@ -2,8 +2,8 @@
 # Copyright 2025, 2026 Query Farm LLC - https://query.farm
 #
 # Run the canonical Query-farm/vgi integration sqllogictest suite against the
-# Python example worker, using a prebuilt standalone `haybarn-unittest` and the
-# signed community vgi extension — no C++ build from source. See ci/README.md.
+# Python example worker, using either a native runner with linked extensions or
+# standalone `haybarn-unittest` and the signed community extension. See ci/README.md.
 #
 # Ported from vgi-go's harness. The Python differences:
 #   * The "worker binaries" are console scripts installed into the project venv
@@ -29,6 +29,8 @@
 # Optional:
 #   BIN_DIR           dir holding the fixture console scripts (default: $REPO/.venv/bin)
 #   TRANSPORT         stdio | shm | launch | http   (default: launch)
+#   RUNNER_LINKS_EXTENSIONS  1 for a native runner linking vgi/httpfs/json/parquet;
+#                     0 for the published standalone runner (default)
 #   VGI_RPC_SHM_SIZE_BYTES  shm side-channel segment size (the shm lane)
 #   STAGE             scratch dir for the preprocessed test tree (default: mktemp)
 set -euo pipefail
@@ -41,6 +43,7 @@ REPO="$(cd "$HERE/.." && pwd)"
 BIN_DIR="${BIN_DIR:-$REPO/.venv/bin}"
 STAGE="${STAGE:-$(mktemp -d)}"
 TRANSPORT="${TRANSPORT:-launch}"
+RUNNER_LINKS_EXTENSIONS="${RUNNER_LINKS_EXTENSIONS:-0}"
 INTEGRATION="$VGI_SRC/test/sql/integration"
 [ -d "$INTEGRATION" ] || { echo "::error::no test/sql/integration under VGI_SRC=$VGI_SRC"; exit 1; }
 
@@ -118,7 +121,11 @@ mkdir -p "$STAGE/test/sql/integration"
        -not -name 'nested_type_combinations.test' \
        "${EXTRA_SKIP[@]}" | while read -r f; do
     mkdir -p "$STAGE/test/sql/integration/$(dirname "$f")"
-    awk -v http="$AWK_HTTP" -f "$HERE/preprocess-require.awk" "$f" > "$STAGE/test/sql/integration/$f"
+    if [ "$RUNNER_LINKS_EXTENSIONS" = "1" ]; then
+      cp "$f" "$STAGE/test/sql/integration/$f"
+    else
+      awk -v http="$AWK_HTTP" -f "$HERE/preprocess-require.awk" "$f" > "$STAGE/test/sql/integration/$f"
+    fi
   done )
 
 # Some integration cases package an executable from the upstream support tree.
@@ -310,6 +317,9 @@ esac
 
 cd "$STAGE"
 
+if [ "$RUNNER_LINKS_EXTENSIONS" = "1" ]; then
+  echo "Using extensions linked into the matching native runner."
+else
 echo "Warming the extension cache (vgi from community, deps from core) ..."
 mkdir -p "$STAGE/test"
 # FORCE INSTALL vgi re-downloads the currently-published community build,
@@ -335,6 +345,7 @@ INSTALL spatial FROM core;
 EOF
 "$HAYBARN_UNITTEST" "test/_warm.test" >/dev/null 2>&1 || echo "::warning::extension warm step did not fully succeed"
 rm -f "$STAGE/test/_warm.test"
+fi
 
 # DuckDB's sqllogictest runner, given no --test-config, turns any error whose
 # text contains "HTTP" or "Unable to connect" into a SKIP and exits 0. Over the
@@ -343,6 +354,14 @@ rm -f "$STAGE/test/_warm.test"
 TEST_CONFIG="$VGI_SRC/test/configs/no_error_skip.json"
 [ -f "$TEST_CONFIG" ] || { echo "::error::missing $TEST_CONFIG (VGI_REF predates it?)"; exit 1; }
 
+# Native unittest defaults to its compiled source directory even after `cd`.
+# Select the staged tree explicitly so discovery respects the exclusions above
+# and spawned workers resolve relative fixture paths from the same directory.
+RUNNER_ARGS=()
+if [ "$RUNNER_LINKS_EXTENSIONS" = "1" ]; then
+  RUNNER_ARGS=(--test-dir "$STAGE")
+fi
+
 # Run the lane, streaming the native sqllogictest report (a progress line per
 # file + the final "All tests passed (.. N assertions ..)" summary). Out-of-scope
 # tests were dropped at staging; any failed assertion exits non-zero.
@@ -373,7 +392,7 @@ run_unittest() {
   # `set +e` rather than `|| true`: the latter runs before PIPESTATUS is read and
   # overwrites it with true's 0, silently swallowing every real test failure.
   set +e
-  "$HAYBARN_UNITTEST" --test-config "$TEST_CONFIG" "$@" 2>&1 | tee "$log"
+  "$HAYBARN_UNITTEST" "${RUNNER_ARGS[@]}" --test-config "$TEST_CONFIG" "$@" 2>&1 | tee "$log"
   rc="${PIPESTATUS[0]}"
   set -e
   if grep -q 'due to a fatal error condition' "$log"; then
@@ -382,6 +401,10 @@ run_unittest() {
          "otherwise have passed. This is invisible to the exit code by construction."
     rc=1
   fi
+  if grep -Eq 'No tests ran|No test cases matched' "$log"; then
+    echo "::error::the runner did not execute the selected tests"
+    rc=1
+  fi
   rm -f "$log"
   return "$rc"
 }

{vgi_python-0.44.0 → vgi_python-0.46.0}/docs/api/auth.md

@@ -18,7 +18,7 @@ require `pip install vgi-python[http]`; JWT authentication additionally requires
 `principal` plus an optional `token_name`. It lives in a private `vgi-rpc` module, so `vgi.auth`
 re-exports it — a worker that implements `resolve_token` never has to name a private import path.
 
-As of vgi-rpc 0.46.0 introspection is the `vgi_rpc.Identity.v1` protocol hosted on the RPC server,
+As of vgi-rpc 0.46.0 introspection is the `vgi_rpc.Identity.v2` protocol hosted on the RPC server,
 not an HTTP JSON route, so clients discover it through reflection. VGI hosts it on HTTP only, the
 transport that authenticates callers: its allowlist names principals, which stdin/stdout, AF_UNIX
 and TCP do not have. `TokenIdentity` and `TokenResolver` are no longer gated on the `[http]` extra.

{vgi_python-0.44.0 → vgi_python-0.46.0}/docs/api/worker.md

@@ -72,7 +72,7 @@ overriding the same hook stop it at startup.
 
 Reflection (`list_protocols`) reports protocols in this order:
 
-| Transport | `vgi.v2` | `hosted_protocols()` | `vgi_rpc.Reflection.v1` | `vgi_rpc.Identity.v1` |
+| Transport | `vgi.v2` | `hosted_protocols()` | `vgi_rpc.Reflection.v1` | `vgi_rpc.Identity.v2` |
 | --- | --- | --- | --- | --- |
 | stdin/stdout | yes | yes | yes | no |
 | AF_UNIX / named pipe | yes | yes | yes | no |

{vgi_python-0.44.0 → vgi_python-0.46.0}/docs/authentication.md

@@ -41,7 +41,7 @@ lookup.
 
 ## Grants and resolved tokens as bearer credentials
 
-`vgi_rpc.Identity.v1`'s `issue_grant` mints a standing credential that
+`vgi_rpc.Identity.v2`'s `issue_grant` mints a standing credential that
 unattended automation later presents as an ordinary bearer. Over HTTP the
 worker accepts two kinds of identity credential after the deployment's own
 authenticator:

vgi_python-0.46.0/docs/catalog_publication.md

@@ -0,0 +1,171 @@
+# Batched catalog contents and reusable publications
+
+VGI wire revision **2.2.0** retains the `vgi.v2` route and replaces
+`catalog_contents` with six catalog-wide Arrow IPC streams: schemas, tables,
+views, functions, macros and indexes. Each stream can contain multiple batches.
+Null means an empty category. Function and macro subtype fields remain intact.
+Existing schema-level RPCs retain their previous collection encoding.
+
+Function metadata stores `arguments`, `output_schema` and
+`parameter_default_values` as `dictionary<int32, binary>` columns. Repeated
+serialized schemas and typed defaults share dictionary entries by their byte
+content. Python callers continue to use bytes for schemas and an optional
+`RecordBatch` for defaults. Dictionary replacements across record batches are
+supported; values remain opaque binary data rather than text.
+Single-object and schema-level function RPCs use these same updated rows.
+
+`SchemaInfo` no longer contains attachment data on the wire. Existing Python
+authors may still pass `attach_opaque_data=` to its constructor during migration;
+the argument is discarded. Use `CatalogAttachResult.attach_opaque_data` for
+subsequent requests. Worker functions still receive their attachment state.
+
+Every minor wire revision is an explicit contract boundary. The server checks
+major and minor exactly, so clients and workers must both use 2.2. A patch bump
+does not permit schema changes. Serving 2.1 and 2.2 during migration requires
+separate deployment URLs or an explicit compatibility implementation.
+
+## Enable publication
+
+Publication is opt-in through `CatalogInterface.catalog_snapshot_policy`.
+Ordinary delivery and the existing frozen-catalog response cache remain the
+default. Published delivery uses the existing vgi-rpc `ExternalRef` mechanism;
+clients resolve the external object on all transports, including subprocess.
+
+Install `vgi-python[external]` on publishing workers and clients that resolve
+publications; add `http` when serving or connecting over HTTP, for example
+`pip install 'vgi-python[http,external]'`. The `external` extra supplies the RPC
+HTTP downloader, retry and compression dependencies. Supply your own configured
+S3 client and its SDK when using the S3 repository.
+
+The catalog implementation uses APIs available in published **vgi-rpc 0.50.0**.
+An isolated check against that release passed 157 functional catalog, client,
+serialization, binary dictionary and publication tests. The package retains its
+declared RPC minimum of 0.52.0 for Identity.v2 support; catalog publication does
+not introduce a dependency on unpublished RPC APIs.
+
+```python
+from vgi.catalog.snapshot import (
+    CatalogSnapshotPolicy,
+    S3CatalogPublicationRepository,
+    SnapshotIdentity,
+)
+from vgi_rpc.external import Compression
+
+# s3 is your configured boto3-compatible S3/R2 client. The client must support
+# conditional PutObject with IfNoneMatch and IfMatch.
+repository = S3CatalogPublicationRepository(
+    s3,
+    bucket="catalog-snapshots",
+    profile="production-r2-gzip",
+    url_ttl_seconds=3600,
+    retention_seconds=86400,
+)
+
+class MyCatalog(ExistingCatalog):
+    catalog_snapshot_policy = CatalogSnapshotPolicy(
+        delivery="published",
+        repository=repository,
+        identity=lambda request: SnapshotIdentity(
+            catalog_namespace="my-catalog",
+            source_revision=DEPLOYMENT_REVISION,
+            catalog_version=request.catalog_version,
+            # Omit this for the safe attachment-local default. Sharing requires
+            # an explicit guarantee that metadata is identical for this scope.
+            visibility_scope="public-default-options",
+        ),
+        compression=Compression(algorithm="gzip"),
+    )
+```
+
+The identity hook must be cheap and must change `source_revision` whenever
+metadata changes. A dynamic catalog can use a schema generation rather than a
+deployment revision. Include options, resolved implementation/data versions,
+tenant or principal, and any other metadata visibility differences in its scope.
+Do not put credentials into names or scope strings. Omitting `visibility_scope`
+isolates publications by a hash of the attachment envelope. Catalog version zero
+still requires a meaningful source revision.
+
+An optional `validator(request) -> str | None` can answer a matching ETag before
+repository access or inventory construction. Every request unwraps and authorizes
+its attachment first, even on a cache hit. The framework also checks that version
+and source identity remain unchanged while it builds a publication.
+
+## Publication lifecycle
+
+The framework writes a complete unary RPC envelope with one `result: binary`
+row. That binary contains the catalog response and its six nested streams.
+The stored object is compressed according to the policy and includes a mandatory
+SHA-256 over the raw envelope. Before committing its manifest, the framework
+fetches the actual HTTP URL and validates the digest, envelope, nested schemas,
+all rows/batches, schema membership, subtype fields and metadata equality.
+Raw and stored byte limits apply before upload and during readback, including
+when the caller supplies a custom fetch policy.
+
+The repository stores small trusted manifests separately from immutable objects.
+Keys include routing name, major/minor wire revision, generated canonical catalog
+contract hash, codec revision, namespace, source revision, visibility scope,
+catalog version, and storage/compression profile. A new worker process looks up
+the manifest before collecting metadata, then validates it with one HTTP fetch.
+A warm hit performs no inventory build, serialization, compression, upload or
+object GET. A client receiving the pointer still downloads the result as needed.
+
+`S3CatalogPublicationRepository` uses conditional manifest writes and tombstones
+for invalidation. It can re-sign a retained object without rebuilding it.
+Configure object lifecycle rules to retain objects for at least the declared
+retention interval; keep manifest retention separate. Signed URLs in manifests
+are credentials, so the manifest store must be access controlled. Newly changed
+retrieval URLs are validated before serving them.
+
+`MemoryCatalogPublicationRepository` supports a supplied RPC uploader for local
+use. It requires an explicit URL lifetime or stable-location declaration.
+It does not provide persistence across process restarts. A process coordinates
+one build per key; conditional repository commit makes separate publishers
+converge but can permit duplicate initial uploads across processes. Storage
+lifecycle rules should collect unused losing uploads.
+
+The manager bounds its in-memory manifest cache. URL or object-retention expiry
+causes renewal or rebuilding. Recovery of a corrupt or missing object
+conditionally invalidates that manifest and attempts a fresh publication.
+Publication errors fail the RPC by default. Set
+`on_publication_failure="ordinary"` to explicitly permit a newly validated
+ordinary full response when publication fails.
+
+Workers expose `warm_catalog_contents(attach_opaque_data)`,
+`import_catalog_publication(attach_opaque_data, manifest)` and
+`invalidate_catalog_publication(attach_opaque_data, manifest_id)` for authorized
+operational use. Import uses the same HTTP validation as recovery. A failed client
+download does not automatically notify the worker; use conditional invalidation
+to repair a known bad publication without deleting a concurrent replacement.
+
+## Validation
+
+Run the focused Python checks with the project environment on `PATH`:
+
+```sh
+uv run pytest tests/catalog/test_catalog_contents.py \
+  tests/catalog/test_catalog_publication.py \
+  tests/catalog/test_client_load_catalog.py tests/catalog/test_serialization.py
+```
+
+The publication tests use real HTTP object retrieval with uncompressed, gzip and
+zstd results. They exercise restart recovery, singleflight, default attachment
+isolation, corrupt publication, explicit fallback, authentication before cached
+delivery, caps, generation changes, manifest import and conditional invalidation.
+The S3 fixture verifies discovery and conditional writes across repository
+instances. Configure a real S3/R2 bucket and deployment CORS before production
+rollout; the in-memory conditional object fixture does not validate a vendor's
+credentials, CORS settings or lifecycle configuration.
+
+The installable native/browser fixture is:
+
+```sh
+python -m vgi._test_fixtures.catalog_publication
+```
+
+It accepts ordinary fixture worker options, including `--http`. Only
+`contents_probe` uses publication; other standard fixture catalogs retain their
+normal behavior. Set `VGI_CATALOG_PUBLICATION_PORT` to expose the local object
+server at a predictable port; `/stats` reports uploads and object GETs. Its CORS
+headers and preflight support permit browser conformance tests. This fixture is
+intended for local testing and its object storage lasts only for the worker's
+lifetime.

{vgi_python-0.44.0 → vgi_python-0.46.0}/docs/protocol/vgi-attach-tickets.md

@@ -1,7 +1,11 @@
 # VGI Attach Tickets
 
 **Status:** Normative. Reference implementation: vgi-python `vgi/attach_ticket.py`.
-**Protocol:** `vgi.attach_tickets.v1`, version `1.0.0`. No change to `vgi.v2`.
+**Protocol:** `vgi.attach_tickets.v1`, version `2.0.0`. No change to `vgi.v2`.
+**Revision 2026-10-17 (`2.0.0`):** `AttachTicket.expires_at` changed from
+`float64` unix seconds (`+inf` = no expiry) to nullable
+`timestamp[us, tz="UTC"]` (null = no expiry). The sealed payload (§2) is
+unchanged; the exact major.minor version check rejects a `1.0.0` peer.
 **Vectors:** [`vgi/_test_fixtures/attach_ticket_vectors.json`](https://github.com/Query-farm/vgi-python/blob/main/vgi/_test_fixtures/attach_ticket_vectors.json)
 
 An SDK implements attach tickets from this page alone. The words MUST, MUST
@@ -17,7 +21,7 @@ this worker can open. Later a runner attaches with one option,
 `vgi_attach_ticket`, and the worker restores the original attach before any
 catalog code runs.
 
-A ticket says *what* to attach. A sealed grant (`vgi_rpc.Identity.v1`
+A ticket says *what* to attach. A sealed grant (`vgi_rpc.Identity.v2`
 `issue_grant`) says *who* attaches. The two are separate on purpose:
 
 - **A ticket carries no authority.** It opens only under the caller's
@@ -133,7 +137,7 @@ from `vgi_rpc.Reflection.v1` `list_protocols`):
    is useless without a grant.
 
 It is hosted beside `vgi.v2` and never changes how a `vgi.v2` request is
-dispatched. Wire name `vgi.attach_tickets.v1`, `protocol_version = "1.0.0"`.
+dispatched. Wire name `vgi.attach_tickets.v1`, `protocol_version = "2.0.0"`.
 
 ### 5.2 Shapes
 
@@ -154,7 +158,7 @@ One unary method, `seal_attach(request: SealAttachRequest) -> AttachTicket`.
 | Field | Arrow type | Meaning |
 | --- | --- | --- |
 | `ticket` | `utf8` not null | The `vgia1.` text |
-| `expires_at` | `float64` not null | Unix seconds; `+inf` when the ticket has no expiry |
+| `expires_at` | `timestamp[us, tz="UTC"]` nullable | The UTC instant the ticket expires (the sealed `expires_at` seconds × 1 000 000 µs); null when the ticket has no expiry |
 
 ### 5.3 Rules
 
@@ -194,7 +198,7 @@ In this order:
 
    So the two halves of a held entry (grant and ticket) expire together.
 5. **Seal** per §2 with a fresh `ticket_id` and nonce. Return the text and
-   `expires_at` (`+inf` for `0`).
+   `expires_at` (null for `0`).
 
 A worker MAY refuse by policy with `action_denied`.
 

{vgi_python-0.44.0 → vgi_python-0.46.0}/packages/vgi-fixtures/pyproject.toml

@@ -8,7 +8,7 @@
 # and the conformance suite import it by its normal `vgi._test_fixtures` path.
 [project]
 name = "vgi-fixtures"
-version = "0.8.2"
+version = "0.46.0"
 description = "Example/test VGI worker modules and entry points (not for production use)"
 readme = "README.md"
 # Local copy of the repo-root LICENSE (kept in sync): the build can't reach