vgi-python 0.37.3__tar.gz → 0.38.0__tar.gz

{vgi_python-0.37.3 → vgi_python-0.38.0}/CLAUDE.md

@@ -392,6 +392,8 @@ Set `SENTRY_DSN` (and install `vgi[sentry]`) to forward unhandled exceptions to
 
 **Attach options redaction:** by default *no* options are logged in the `catalog.attach` / `catalog.create` breadcrumbs because options routinely carry credentials. Implementers opt in via `CatalogInterface.loggable_attach_options(options) -> Mapping`, which returns a redacted, safe-to-log subset (host, region, bucket — never password/token/secret). When the override returns an empty mapping (the default), the `options` field is omitted from the breadcrumb entirely. See `docs/catalog-interface.md` for details.
 
+**Secret attach options:** credential options (API keys, tokens, passwords) MUST be declared `AttachOption(..., secret=True)`. `AttachOptionSpec.secret` rides as a nullable `bool` column named `secret` appended after `required` (an absent column reads as `False`); clients and the DuckDB extension mask such options, keep them out of cache keys and logs, and can supply them from a `vgi_attach` DuckDB secret (`CREATE SECRET (TYPE vgi_attach, SCOPE '<worker url>', api_key '…')`). Secret + required is fine; secret + default is allowed but discouraged.
+
 **Releases.** `vgi-serve` populates `release` from `SENTRY_RELEASE` if set, otherwise from `importlib.metadata.version("vgi-python")`. Production deploys should set `SENTRY_RELEASE=$(git rev-parse HEAD)` (or a tagged release identifier) so Sentry's commit-tracking and regression-detection features can correlate events to specific commits. Publishing GitHub releases makes the release-comparison UI more useful.
 
 The same enrichment applies to OTel spans when `VGI_OTEL_ENABLED=1` — both backends read from the same `VgiTracer.set_current_span_attributes()` call sites in `vgi/otel.py`. Either, neither, or both can be active in a process.

{vgi_python-0.37.3 → vgi_python-0.38.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: vgi-python
-Version: 0.37.3
+Version: 0.38.0
 Summary: Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow
 Project-URL: Homepage, https://query.farm
 Project-URL: Repository, https://github.com/Query-farm/vgi-python

{vgi_python-0.37.3 → vgi_python-0.38.0}/docs/catalog-interface.md

@@ -246,6 +246,42 @@ class MyCatalog(CatalogInterface):
 
 ---
 
+## Declaring Attach Options
+
+A worker declares the options its catalogs accept at `ATTACH` time in an `AttachOptions` inner class. Each option is advertised to clients at discovery (`CatalogInfo.attach_option_specs`), so DuckDB can cast and validate it, and frontends can render an options form.
+
+```python test="skip"
+from typing import Annotated
+from vgi import Worker
+from vgi.catalog.attach_option import AttachOption
+
+class MyWorker(Worker):
+    class AttachOptions:
+        region: Annotated[str, AttachOption(desc="AWS region")] = "us-east-1"
+        # A credential: required, secret, and with no default.
+        api_key: Annotated[str, AttachOption(desc="API key", required=True, secret=True)]
+```
+
+| Flag | Meaning |
+|------|---------|
+| `required=True` | The caller must supply the option. Declare it with no class-level default; `required` plus a default is rejected. |
+| `secret=True` | The option carries a credential. |
+
+**Credential options (API keys, tokens, passwords) MUST be declared `secret=True`.** Clients and the DuckDB extension then mask the value, keep it out of result-cache keys and logs, and can supply it from a `vgi_attach` DuckDB secret, so the `ATTACH` statement never carries it:
+
+```sql
+CREATE SECRET (TYPE vgi_attach, SCOPE 'https://worker.example.com', api_key '...');
+ATTACH 'mydb' AS mydb (TYPE vgi, LOCATION 'https://worker.example.com');
+```
+
+`secret` combines with `required`. It is allowed together with a default, but a secret option should normally have none: the default is advertised in plain text to every client at discovery.
+
+On the wire, `required` and `secret` are nullable boolean columns appended after the four shared spec columns (`name`, `description`, `type`, `default_value`), in that order. Readers look columns up by name, so a spec from a peer that predates either column reads it as `false`, and older peers ignore columns they don't know.
+
+`secret` does not change what the worker itself logs: attach options are never logged unless the catalog opts in through `loggable_attach_options()` (below), which must never return a secret option.
+
+---
+
 ## Logging Attach Options Safely
 
 The worker emits structured `_logger.info` records and Sentry breadcrumbs for catalog lifecycle events (`catalog.attach`, `catalog.detach`, `catalog.create`, `catalog.transaction.begin`, `catalog.transaction.commit`, `catalog.transaction.rollback`). `attach_opaque_data` and `transaction_opaque_data` are short-hashed (12-char SHA-256 prefixes) before they reach the log record, the breadcrumb data, or the Sentry scope tags — the raw values never appear in observability output, since they may carry secrets. An operator correlates the short hash back to the catalog via these breadcrumbs.

{vgi_python-0.37.3 → vgi_python-0.38.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "vgi-python"
-version = "0.37.3"
+version = "0.38.0"
 description = "Vector Gateway Interface - Connect DuckDB to external programs via Apache Arrow"
 readme = "README.md"
 keywords = [

{vgi_python-0.37.3 → vgi_python-0.38.0}/tests/catalog/test_attach_option.py

@@ -1,6 +1,6 @@
 # Copyright 2025, 2026 Query Farm LLC - https://query.farm
 
-"""Tests for the AttachOption descriptor, its ``required`` flag, and validation."""
+"""Tests for the AttachOption descriptor, its ``required`` and ``secret`` flags, and validation."""
 
 from typing import Annotated
 
@@ -25,7 +25,11 @@ class _Options:
 
     region: Annotated[str, AttachOption(desc="AWS region")] = "us-east-1"
     # No class-level assignment: nothing to fall back to, so the caller must supply it.
-    api_key: Annotated[str, AttachOption(desc="API key", required=True)]
+    api_key: Annotated[str, AttachOption(desc="API key", required=True, secret=True)]
+    # A credential that may be omitted, with no default (the recommended shape).
+    token: Annotated[str | None, AttachOption(desc="Optional token", arrow_type=pa.string(), secret=True)] = None
+    # Allowed, though discouraged: a secret option with a default.
+    password: Annotated[str, AttachOption(desc="Password", secret=True)] = "changeme"
 
 
 def _roundtrip(spec: AttachOptionSpec) -> AttachOptionSpec:
@@ -83,6 +87,7 @@ class TestWireFormat:
             "type",
             "default_value",
             "required",
+            "secret",
         ]
 
     def test_reads_batch_without_required_column(self) -> None:
@@ -124,6 +129,120 @@ class TestWireFormat:
         assert AttachOptionSpec.deserialize(restored).required is False
 
 
+class TestSecretExtraction:
+    """Extraction of the ``secret`` flag from Annotated declarations."""
+
+    def test_defaults_to_false(self) -> None:
+        """An option that isn't declared secret isn't secret."""
+        specs = {s.name: s for s in extract_attach_option_specs(_Options)}
+        assert specs["region"].secret is False
+
+    def test_secret_with_required(self) -> None:
+        """``secret`` and ``required`` combine: a credential the caller must supply."""
+        specs = {s.name: s for s in extract_attach_option_specs(_Options)}
+        assert specs["api_key"].secret is True
+        assert specs["api_key"].required is True
+        assert specs["api_key"].default is None
+
+    def test_secret_without_required(self) -> None:
+        """A secret option need not be required."""
+        specs = {s.name: s for s in extract_attach_option_specs(_Options)}
+        assert specs["token"].secret is True
+        assert specs["token"].required is False
+
+    def test_secret_with_default_allowed(self) -> None:
+        """A secret option may declare a default (discouraged, but not an error)."""
+        specs = {s.name: s for s in extract_attach_option_specs(_Options)}
+        assert specs["password"].secret is True
+        assert specs["password"].default == "changeme"
+
+    def test_secret_required_and_default_still_rejected(self) -> None:
+        """``secret`` doesn't relax the required + default contradiction."""
+        with pytest.raises(ValueError, match="required but also declares a default"):
+            AttachOptionSpec(name="k", desc="", type=pa.string(), default="x", required=True, secret=True)
+
+
+class TestSecretWireFormat:
+    """Serialization of the added ``secret`` column."""
+
+    @pytest.mark.parametrize("secret", [True, False], ids=["secret", "plain"])
+    def test_roundtrip_preserves_secret(self, secret: bool) -> None:
+        """``secret`` survives an Arrow IPC round trip in both states."""
+        spec = AttachOptionSpec(name="api_key", desc="API key", type=pa.string(), default=None, secret=secret)
+        restored = _roundtrip(spec)
+        assert restored.secret is secret
+        assert restored.required is False
+
+    @pytest.mark.parametrize(
+        ("required", "secret"),
+        [(True, True), (True, False), (False, True), (False, False)],
+    )
+    def test_roundtrip_required_and_secret_independent(self, required: bool, secret: bool) -> None:
+        """The two flags occupy separate columns and never bleed into each other."""
+        spec = AttachOptionSpec(
+            name="api_key", desc="API key", type=pa.string(), default=None, required=required, secret=secret
+        )
+        restored = _roundtrip(spec)
+        assert (restored.required, restored.secret) == (required, secret)
+
+    def test_secret_column_follows_required(self) -> None:
+        """``secret`` is a nullable bool appended after ``required``."""
+        schema = AttachOptionSpec.ARROW_SCHEMA
+        assert schema.get_field_index("secret") == schema.get_field_index("required") + 1
+        assert schema.get_field_index("secret") == len(schema) - 1
+        field = schema.field("secret")
+        assert field.type == pa.bool_()
+        assert field.nullable
+
+    def test_reads_batch_without_secret_column(self) -> None:
+        """A peer that has ``required`` but predates ``secret`` yields ``secret=False``."""
+        type_bytes = pa.schema([pa.field("value", pa.string())]).serialize().to_pybytes()
+        older_schema = pa.schema([*_SpecBase.ARROW_SCHEMA, pa.field("required", pa.bool_(), nullable=True)])
+        older = pa.RecordBatch.from_pylist(
+            [{"name": "api_key", "description": "", "type": type_bytes, "default_value": None, "required": True}],
+            schema=older_schema,
+        )
+        restored_batch, _ = deserialize_record_batch(serialize_record_batch_bytes(older))
+        restored = AttachOptionSpec.deserialize(restored_batch)
+        assert restored.secret is False
+        assert restored.required is True
+
+    def test_reads_four_column_batch_as_not_secret(self) -> None:
+        """A peer that predates both columns reads as neither required nor secret."""
+        type_bytes = pa.schema([pa.field("value", pa.string())]).serialize().to_pybytes()
+        legacy = pa.RecordBatch.from_pylist(
+            [{"name": "region", "description": "", "type": type_bytes, "default_value": None}],
+            schema=_SpecBase.ARROW_SCHEMA,
+        )
+        restored = AttachOptionSpec.deserialize(legacy)
+        assert (restored.required, restored.secret) == (False, False)
+
+    def test_null_secret_reads_as_false(self) -> None:
+        """An explicit null in the nullable column means "not secret"."""
+        type_bytes = pa.schema([pa.field("value", pa.string())]).serialize().to_pybytes()
+        batch = pa.RecordBatch.from_pylist(
+            [
+                {
+                    "name": "region",
+                    "description": "",
+                    "type": type_bytes,
+                    "default_value": None,
+                    "required": None,
+                    "secret": None,
+                }
+            ],
+            schema=AttachOptionSpec.ARROW_SCHEMA,
+        )
+        restored, _ = deserialize_record_batch(serialize_record_batch_bytes(batch))
+        assert AttachOptionSpec.deserialize(restored).secret is False
+
+    def test_older_reader_ignores_secret_column(self) -> None:
+        """A four-column reader still reads a six-column batch (forward compatible)."""
+        spec = AttachOptionSpec(name="api_key", desc="", type=pa.string(), default=None, required=True, secret=True)
+        batch, _ = deserialize_record_batch(spec.serialize())
+        assert SettingSpec.deserialize(batch).name == "api_key"
+
+
 class TestValidateRequiredAttachOptions:
     """The shared validation helper."""
 

{vgi_python-0.37.3 → vgi_python-0.38.0}/tests/test_worker_page.py

@@ -23,9 +23,10 @@ from vgi.catalog import (
     Table,
     View,
 )
-from vgi.catalog.attach_option import AttachOption, extract_attach_option_specs
+from vgi.catalog.attach_option import AttachOption, AttachOptionSpec, extract_attach_option_specs
 from vgi.http.worker_page import (
     WorkerPageResource,
+    _build_attach_options_table,
     _display_function_type,
     build_worker_page,
 )
@@ -542,6 +543,20 @@ class TestAttachOptions:
         # Defaults render via repr() and HTML-escape the quotes.
         assert "'hello'" in html
 
+    def test_required_and_secret_flags_render(self) -> None:
+        """Required/secret options are flagged, and a secret's default is never shown."""
+        specs = (
+            AttachOptionSpec(name="api_key", desc="", type=pa.string(), default=None, required=True, secret=True),
+            AttachOptionSpec(name="password", desc="", type=pa.string(), default="hunter2", secret=True),
+            AttachOptionSpec(name="region", desc="", type=pa.string(), default="us-east-1"),
+        )
+        html = _build_attach_options_table(specs)
+        assert ">required</span>" in html
+        assert ">secret</span>" in html
+        assert "hunter2" not in html
+        assert "<em>hidden</em>" in html
+        assert "&#x27;us-east-1&#x27;" in html
+
 
 class TestVersionDisplay:
     """Implementation chip + Data version input."""