vibedata-dlt-redshift-studio 0.1.0__tar.gz

vibedata_dlt_redshift_studio-0.1.0/.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+dist/
+build/
+.venv/
+.env
+.pytest_cache/

vibedata_dlt_redshift_studio-0.1.0/PKG-INFO

@@ -0,0 +1,58 @@
+Metadata-Version: 2.5
+Name: vibedata-dlt-redshift-studio
+Version: 0.1.0
+Summary: VibeData dlt runtime — Amazon Redshift Studio execution unit (intent-local runs targeting an Intent's Ephemeral Redshift database): the brokered database password for dlt's redshift destination.
+Author-email: Vibedata <eng@acceleratedata.ai>
+License: MIT
+Requires-Python: >=3.11
+Requires-Dist: dlt[redshift]>=1.30
+Requires-Dist: vibedata-dlt-core
+Requires-Dist: vibedata-dlt-studio-secrets
+Provides-Extra: dev
+Requires-Dist: pytest-mock>=3.12; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# vibedata-dlt-redshift-studio
+
+The dlt runtime execution unit for an Amazon Redshift Domain, installed in the
+Studio-local intent venv.
+
+```python
+import dlt
+from vibedata.dlt.redshift import setup_environment, finalize
+
+setup_environment()
+pipeline = dlt.pipeline(
+    pipeline_name="my_pipeline", destination="redshift", dataset_name="analytics"
+)
+try:
+    pipeline.run(source)
+except Exception as e:
+    finalize(pipeline, error_message=str(e))
+    raise
+finalize(pipeline)
+```
+
+`setup_environment()` registers the Domain's Secret Store provider and mints the
+Ephemeral database's password into this process, under dlt's own
+`DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD`. Studio publishes the other four
+coordinates — `HOST`, `PORT`, `DATABASE`, `USERNAME` — into the Intent
+environment under the matching names, and dlt reads those natively, so the
+pipeline names no `VD_*` variable itself.
+
+The password is separate because Redshift authenticates with a credential minted
+per invocation, scoped to one database and valid for at most an hour.
+
+Without one, dlt refuses to configure the destination and names
+`DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD` — which no authored pipeline
+sets, so this unit fails at the mint and names the Broker instead. A *blank*
+password is the case worth guarding: dlt resolves it cleanly and the load then
+reaches Redshift as a real login attempt, so an empty Broker answer is refused
+rather than set.
+
+The minted password lives in `os.environ` for the life of the process, as the
+MotherDuck unit's token does — inherited by any child it spawns.
+
+`finalize()` persists the audit tables. A Redshift load writes directly, so
+there is nothing to sync afterwards.

vibedata_dlt_redshift_studio-0.1.0/README.md

@@ -0,0 +1,43 @@
+# vibedata-dlt-redshift-studio
+
+The dlt runtime execution unit for an Amazon Redshift Domain, installed in the
+Studio-local intent venv.
+
+```python
+import dlt
+from vibedata.dlt.redshift import setup_environment, finalize
+
+setup_environment()
+pipeline = dlt.pipeline(
+    pipeline_name="my_pipeline", destination="redshift", dataset_name="analytics"
+)
+try:
+    pipeline.run(source)
+except Exception as e:
+    finalize(pipeline, error_message=str(e))
+    raise
+finalize(pipeline)
+```
+
+`setup_environment()` registers the Domain's Secret Store provider and mints the
+Ephemeral database's password into this process, under dlt's own
+`DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD`. Studio publishes the other four
+coordinates — `HOST`, `PORT`, `DATABASE`, `USERNAME` — into the Intent
+environment under the matching names, and dlt reads those natively, so the
+pipeline names no `VD_*` variable itself.
+
+The password is separate because Redshift authenticates with a credential minted
+per invocation, scoped to one database and valid for at most an hour.
+
+Without one, dlt refuses to configure the destination and names
+`DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD` — which no authored pipeline
+sets, so this unit fails at the mint and names the Broker instead. A *blank*
+password is the case worth guarding: dlt resolves it cleanly and the load then
+reaches Redshift as a real login attempt, so an empty Broker answer is refused
+rather than set.
+
+The minted password lives in `os.environ` for the life of the process, as the
+MotherDuck unit's token does — inherited by any child it spawns.
+
+`finalize()` persists the audit tables. A Redshift load writes directly, so
+there is nothing to sync afterwards.

vibedata_dlt_redshift_studio-0.1.0/pyproject.toml

@@ -0,0 +1,38 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vibedata-dlt-redshift-studio"
+version = "0.1.0"
+description = "VibeData dlt runtime — Amazon Redshift Studio execution unit (intent-local runs targeting an Intent's Ephemeral Redshift database): the brokered database password for dlt's redshift destination."
+readme = "README.md"
+requires-python = ">=3.11"
+license = { text = "MIT" }
+authors = [{ name = "Vibedata", email = "eng@acceleratedata.ai" }]
+
+dependencies = [
+    # `persist_audit`, which `finalize` delegates to unchanged.
+    "vibedata-dlt-core",
+    # The Key Vault provider and the kind-gated registration every Studio unit
+    # performs before its destination work. A Redshift Domain binds a Secret
+    # Store like any other (ADR-0022).
+    "vibedata-dlt-studio-secrets",
+    # The `redshift` extra is required, not convenience: it is what brings
+    # psycopg2, and without it dlt has no usable Redshift destination at all
+    # rather than a degraded one. No upper bound — unlike the Warehouse unit,
+    # this one patches no private seam of the destination, so a minor bump has
+    # nothing here to move. The floor tracks the other units in this workspace
+    # rather than a Redshift capability — that destination long predates 1.30.
+    "dlt[redshift]>=1.30",
+]
+
+[project.optional-dependencies]
+dev = ["pytest>=8.0", "pytest-mock>=3.12"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vibedata"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]

vibedata_dlt_redshift_studio-0.1.0/src/vibedata/dlt/redshift/__init__.py

@@ -0,0 +1,32 @@
+"""VibeData dlt runtime — Amazon Redshift Studio execution unit.
+
+The installed unit for Redshift domains running in the Studio-local intent
+venv. It exposes the stable, unit-agnostic import:
+
+    from vibedata.dlt.redshift import setup_environment, finalize
+
+There is no `vibedata-dlt-redshift-core`: unlike DuckDB and Fabric, Redshift has
+one execution context — the Intent's Ephemeral database — so there is no second
+unit for a core to be shared with. `finalize` therefore delegates straight to
+the workspace-wide audit helper, and `setup_environment` does what the agent
+container needs: the Domain's Secret Store provider, then the brokered database
+password (see `bootstrap`).
+"""
+
+from typing import Any, Optional
+
+from vibedata.dlt.core.audit import persist_audit
+
+from .bootstrap import RedshiftCredentialError, setup_environment
+
+
+def finalize(pipeline: Any, error_message: Optional[str] = None) -> dict[str, Any]:
+    """Persist audit tables. Redshift writes land directly — nothing to sync."""
+    return persist_audit(pipeline, pipeline.last_trace, error_message=error_message)
+
+
+__all__ = [
+    "setup_environment",
+    "finalize",
+    "RedshiftCredentialError",
+]

vibedata_dlt_redshift_studio-0.1.0/src/vibedata/dlt/redshift/bootstrap.py

@@ -0,0 +1,173 @@
+"""
+Bootstrap for STUDIO (intent-local) dlt runs — Redshift platform dialect:
+
+    import dlt
+    from vibedata.dlt.redshift import setup_environment, finalize
+
+    setup_environment()
+    pipeline = dlt.pipeline(pipeline_name="<pipeline_name>",
+                            destination="redshift",
+                            dataset_name="<schema>")
+    try:
+        pipeline.run(source)
+    except Exception as e:
+        finalize(pipeline, error_message=str(e))
+        raise
+    finalize(pipeline)
+
+This unit contributes exactly one thing to dlt's config chain: the database
+password. Studio already publishes the other four coordinates into the Intent
+environment under dlt's own documented names —
+`DESTINATION__REDSHIFT__CREDENTIALS__{HOST,PORT,DATABASE,USERNAME}` — and dlt
+reads those natively, so a generated pipeline is plain `destination="redshift"`
+with no `VD_*` read of its own.
+
+The password is the one field that cannot ride the Intent environment the way a
+host or port does: Redshift authenticates with a credential minted per
+invocation, scoped to exactly one database, and valid for at most an hour. It is
+minted here and set for this process only, which is the same contract the dbt
+shim (`vd-dbt.sh`) holds for `VD_EPHM_REDSHIFT_PASSWORD`.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+from vibedata.dlt.studio_secrets import register_secret_provider
+
+logger = logging.getLogger(__name__)
+
+PLATFORM_ENV = "VD_DOMAIN_DATA_PLATFORM"
+REDSHIFT_PLATFORM = "redshift"
+
+# dlt's documented config name for the destination's password. The sibling four
+# are published by Studio and read by dlt without this unit's help.
+DLT_PASSWORD_ENV = "DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD"
+
+BROKER_URL_ENV = "VD_CREDENTIAL_BROKER_URL"
+CREDENTIAL_FILE_ENV = "VD_CREDENTIAL_FILE"
+BROKER_LANE_ENV = "VD_CREDENTIAL_LANE"
+CONFIGURATION_REVISION_ENV = "VD_CREDENTIAL_CONFIGURATION_REVISION"
+AUDIENCE = "redshift"
+BROKER_TIMEOUT_SECONDS = 20
+
+
+class RedshiftCredentialError(RuntimeError):
+    """Raised when a Redshift run cannot obtain a database password.
+
+    Raised rather than allowed to proceed, for two different reasons either side
+    of the value. Absent, dlt refuses to configure the destination at all and
+    raises `ConfigFieldMissingException` naming
+    `DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD` — loud, but pointed at a
+    variable no authored pipeline sets, so failing at the mint and naming the
+    Broker is the actionable version of the same stop. Empty is the worse case:
+    dlt resolves a blank password cleanly and the load reaches Redshift as a
+    real login attempt, which is why an empty Broker answer is refused here.
+    """
+
+
+def setup_environment() -> None:
+    """Register the bound Secret Store's provider, then bridge the password.
+
+    The Secret Store is a Domain property independent of the platform
+    (ADR-0022): a Redshift Domain bound to Key Vault resolves its source
+    credentials through the same provider a Fabric Domain does, and it must be
+    in dlt's chain before the pipeline's first `dlt.secrets` read. A
+    `local_toml` store registers nothing (VD-5474).
+    """
+    register_secret_provider()
+    apply_redshift_environment()
+    logger.info("VibeData dlt environment ready (unit=redshift)")
+
+
+def apply_redshift_environment() -> None:
+    """Mint the database password and set it under dlt's own config name.
+
+    No-op on any other platform, whose destination is resolved elsewhere.
+    """
+    if os.environ.get(PLATFORM_ENV, "").strip() != REDSHIFT_PLATFORM:
+        logger.info("Domain does not target Redshift; skipping runtime resolution")
+        return
+
+    os.environ[DLT_PASSWORD_ENV] = _mint_password()
+    logger.info("Redshift database password resolved for the dlt destination")
+
+
+def _mint_password() -> str:
+    """One Data Platform credential for the acting user, from Studio's Broker.
+
+    The audience selects the kind; the Broker decides which database it is
+    minted against, from the Intent the capability is scoped to. This unit
+    never names a database — the ephemeral database name is not usable as an
+    input here, and sending one would be a shape the Broker rejects.
+    """
+    broker_url = os.environ.get(BROKER_URL_ENV, "").strip().rstrip("/")
+    credential_file = os.environ.get(CREDENTIAL_FILE_ENV, "").strip()
+    lane = os.environ.get(BROKER_LANE_ENV, "").strip()
+    revision_value = os.environ.get(CONFIGURATION_REVISION_ENV, "").strip()
+    try:
+        configuration_revision = int(revision_value)
+    except ValueError:
+        configuration_revision = -1
+    if (
+        not broker_url
+        or not credential_file
+        or lane not in {"user", "service"}
+        or configuration_revision < 0
+    ):
+        raise RedshiftCredentialError(
+            "The credential broker URL, credential file, lane, and Domain configuration "
+            "revision are required to resolve a Redshift credential. Confirm the "
+            "runtime environment is complete."
+        )
+
+    try:
+        bearer = Path(credential_file).read_text(encoding="utf-8").strip()
+    except OSError as exc:
+        raise RedshiftCredentialError(
+            f"Could not read {credential_file!r}. Is the Studio intent session active?"
+        ) from exc
+    if not bearer:
+        raise RedshiftCredentialError(f"{credential_file!r} is empty.")
+
+    request = urllib.request.Request(
+        f"{broker_url}/token",
+        data=json.dumps(
+            {
+                "audience": AUDIENCE,
+                "scope": "data_platform",
+                "lane": lane,
+                "configurationRevision": configuration_revision,
+            }
+        ).encode("utf-8"),
+        headers={
+            "Authorization": f"Bearer {bearer}",
+            "Content-Type": "application/json",
+        },
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=BROKER_TIMEOUT_SECONDS) as response:
+            payload = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as exc:
+        raise RedshiftCredentialError(
+            f"The credential broker rejected the Redshift request ({exc.code}). "
+            "The intent access token may be expired or revoked, or Studio's AWS "
+            "identity may lack permission to mint for this database."
+        ) from exc
+    except (urllib.error.URLError, TimeoutError) as exc:
+        raise RedshiftCredentialError(
+            f"Could not reach the credential broker at {broker_url}/token. Is Studio running?"
+        ) from exc
+
+    token = payload.get("access_token")
+    if not isinstance(token, str) or not token.strip():
+        raise RedshiftCredentialError(
+            "The credential broker returned no Redshift database password."
+        )
+    return token.strip()

vibedata_dlt_redshift_studio-0.1.0/tests/conftest.py

@@ -0,0 +1,33 @@
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def clean_env(monkeypatch):
+    """Strip all contract env vars so each test declares exactly what it needs."""
+    for var in (
+        "VD_DOMAIN_DATA_PLATFORM",
+        "VD_DOMAIN_SECRET_STORE_KIND",
+        "VD_DOMAIN_SECRET_STORE_LOCATION",
+        "VD_CREDENTIAL_BROKER_URL",
+        "VD_CREDENTIAL_FILE",
+        "VD_CREDENTIAL_LANE",
+        "VD_CREDENTIAL_CONFIGURATION_REVISION",
+        "DESTINATION__REDSHIFT__CREDENTIALS__PASSWORD",
+    ):
+        monkeypatch.delenv(var, raising=False)
+
+
+@pytest.fixture
+def redshift_env(monkeypatch, tmp_path):
+    """Minimal valid Redshift intent environment (mirrors build-agent-server-env.ts)."""
+    monkeypatch.setenv("VD_DOMAIN_SECRET_STORE_KIND", "local_toml")
+    monkeypatch.setenv("VD_DOMAIN_DATA_PLATFORM", "redshift")
+    monkeypatch.setenv(
+        "VD_CREDENTIAL_BROKER_URL", "http://127.0.0.1:3001/api/v1/credential-broker"
+    )
+    cred_file = tmp_path / "credential"
+    cred_file.write_text("vd_iat_test-intent-token")
+    monkeypatch.setenv("VD_CREDENTIAL_FILE", str(cred_file))
+    monkeypatch.setenv("VD_CREDENTIAL_LANE", "user")
+    monkeypatch.setenv("VD_CREDENTIAL_CONFIGURATION_REVISION", "7")
+    return tmp_path

vibedata_dlt_redshift_studio-0.1.0/tests/test_bootstrap.py

@@ -0,0 +1,116 @@
+"""Redshift database-password resolution for intent-local dlt runs.
+
+What this guards: the password is the one destination field that cannot ride the
+Intent environment, because Redshift mints it per invocation against exactly one
+database. Studio publishes the other four coordinates and dlt reads them
+natively, field by field.
+
+An absent password is caught by dlt itself, which refuses to configure the
+destination and names the variable. The case that needs guarding here is a
+*blank* one: dlt resolves it cleanly and the load reaches Redshift as a real
+login attempt, so a Broker answer carrying no password is refused rather than
+set.
+"""
+
+import io
+import json
+import os
+import urllib.error
+
+import pytest
+
+from vibedata.dlt.redshift import bootstrap
+from vibedata.dlt.redshift.bootstrap import (
+    DLT_PASSWORD_ENV,
+    RedshiftCredentialError,
+    apply_redshift_environment,
+)
+
+
+def _broker_response(payload):
+    return io.BytesIO(json.dumps(payload).encode("utf-8"))
+
+
+@pytest.fixture
+def broker(monkeypatch):
+    """Capture the Broker request and answer it with a password."""
+    seen = {}
+
+    def fake_urlopen(request, timeout=None):
+        seen["url"] = request.full_url
+        seen["headers"] = dict(request.headers)
+        seen["body"] = json.loads(request.data.decode("utf-8"))
+        seen["timeout"] = timeout
+        return _FakeCtx(_broker_response({"access_token": "minted-password"}))
+
+    monkeypatch.setattr(bootstrap.urllib.request, "urlopen", fake_urlopen)
+    return seen
+
+
+class _FakeCtx:
+    def __init__(self, body):
+        self._body = body
+
+    def __enter__(self):
+        return self._body
+
+    def __exit__(self, *_):
+        return False
+
+
+class TestRedshiftDomain:
+    def test_puts_the_minted_password_under_dlts_own_config_name(self, redshift_env, broker):
+        apply_redshift_environment()
+        assert os.environ[DLT_PASSWORD_ENV] == "minted-password"
+
+    def test_mints_the_redshift_audience_with_the_intent_bearer(self, redshift_env, broker):
+        apply_redshift_environment()
+        assert broker["url"].endswith("/token")
+        assert broker["headers"]["Authorization"] == "Bearer vd_iat_test-intent-token"
+        # The Broker decides which database this is for; naming one is a shape
+        # it rejects, so the request carries no database.
+        assert broker["body"] == {
+            "audience": "redshift",
+            "scope": "data_platform",
+            "lane": "user",
+            "configurationRevision": 7,
+        }
+
+    def test_requires_the_explicit_lane_and_pinned_revision(self, redshift_env, monkeypatch):
+        monkeypatch.delenv("VD_CREDENTIAL_LANE")
+        with pytest.raises(RedshiftCredentialError):
+            apply_redshift_environment()
+
+    def test_rejects_a_nonsense_revision_rather_than_sending_it(self, redshift_env, monkeypatch):
+        monkeypatch.setenv("VD_CREDENTIAL_CONFIGURATION_REVISION", "not-a-number")
+        with pytest.raises(RedshiftCredentialError):
+            apply_redshift_environment()
+
+    def test_refuses_an_empty_credential_file(self, redshift_env, monkeypatch):
+        (redshift_env / "credential").write_text("")
+        with pytest.raises(RedshiftCredentialError):
+            apply_redshift_environment()
+
+    def test_refuses_a_broker_answer_carrying_no_password(self, redshift_env, monkeypatch):
+        def no_token(request, timeout=None):
+            return _FakeCtx(_broker_response({"access_token": ""}))
+
+        monkeypatch.setattr(bootstrap.urllib.request, "urlopen", no_token)
+        with pytest.raises(RedshiftCredentialError):
+            apply_redshift_environment()
+
+    def test_surfaces_a_broker_rejection_rather_than_proceeding(self, redshift_env, monkeypatch):
+        def rejected(request, timeout=None):
+            raise urllib.error.HTTPError(request.full_url, 403, "Forbidden", {}, None)
+
+        monkeypatch.setattr(bootstrap.urllib.request, "urlopen", rejected)
+        with pytest.raises(RedshiftCredentialError):
+            apply_redshift_environment()
+        assert DLT_PASSWORD_ENV not in os.environ
+
+
+class TestOtherPlatforms:
+    def test_is_a_no_op_off_redshift(self, monkeypatch):
+        monkeypatch.setenv("VD_DOMAIN_DATA_PLATFORM", "motherduck")
+        apply_redshift_environment()
+        assert DLT_PASSWORD_ENV not in os.environ