databricks-tellr 0.3.13.dev4__tar.gz → 0.3.13.dev12__tar.gz

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr
-Version: 0.3.13.dev4
+Version: 0.3.13.dev12
 Summary: Tellr deployment tooling for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/databricks_tellr/_templates/app.yaml.template

@@ -27,6 +27,8 @@ env:
     value: "${LAKEBASE_INSTANCE}"
   - name: LAKEBASE_SCHEMA
     value: "${LAKEBASE_SCHEMA}"
+  - name: GOOGLE_OAUTH_ENCRYPTION_KEY
+    value: "${GOOGLE_OAUTH_ENCRYPTION_KEY}"
   - name: LAKEBASE_TYPE
     value: "${LAKEBASE_TYPE}"
   - name: LAKEBASE_PG_HOST
@@ -37,6 +39,8 @@ env:
     value: "${LAKEBASE_ENDPOINT_NAME}"
   - name: DATABRICKS_HOST
     valueFrom: "system.databricks_host"
+  - name: DATABRICKS_TOKEN
+    valueFrom: "system.databricks_token"
   # Opt-in to trust x-forwarded-email / x-forwarded-user on the MCP endpoint
   # for app-to-app calls. Safe inside Databricks Apps because the proxy sets
   # these headers itself and strips any caller-supplied versions. See

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/databricks_tellr/deploy.py

@@ -30,8 +30,6 @@ from databricks.sdk.service.apps import (
 from databricks.sdk.service.database import DatabaseInstance
 from databricks.sdk.service.workspace import ImportFormat
 
-from databricks_tellr.identifiers import validate_client_id, validate_schema_name
-
 # Autoscaling imports (Lakebase next-gen)
 try:
     from databricks.sdk.service.postgres import (
@@ -197,6 +195,7 @@ def create(
     client: WorkspaceClient | None = None,
     profile: str | None = None,
     config_yaml_path: str | None = None,
+    encryption_key: str | None = None,
     mlflow_tracing: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     """Deploy Tellr to Databricks Apps.
@@ -226,6 +225,7 @@ def create(
         client: External WorkspaceClient (optional)
         profile: Databricks CLI profile name (optional)
         config_yaml_path: Path to deployment config YAML (mutually exclusive with other args)
+        encryption_key: Fernet key for Google OAuth encryption. Auto-generated if not provided.
         mlflow_tracing: Optional overrides for UC tracing env vars in generated ``app.yaml``.
             Keys: ``MLFLOW_TRACING_SQL_WAREHOUSE_ID``, ``TELLR_MLFLOW_UC_CATALOG``,
             ``TELLR_MLFLOW_UC_SCHEMA``, ``TELLR_MLFLOW_UC_TABLE_PREFIX``. With
@@ -258,6 +258,7 @@ def create(
         profile=profile,
         config_yaml_path=config_yaml_path,
         seed_databricks_defaults=False,
+        encryption_key=encryption_key,
         mlflow_tracing=mlflow_tracing,
     )
 
@@ -287,9 +288,8 @@ def update(
         reset_database: If True, drop and recreate the schema (tables recreated on app startup)
         client: External WorkspaceClient (optional)
         profile: Databricks CLI profile name (optional)
-        encryption_key: Override for the legacy Fernet key to relocate into
-            the encryption_keys table. Default: read from the deployed
-            app.yaml. The key is no longer written to app.yaml.
+        encryption_key: Fernet key for Google OAuth encryption. If not provided, the
+            existing key is read from the deployed app.yaml to preserve encrypted data.
         mlflow_tracing: Optional overrides for UC tracing env vars (same keys as ``create``).
             Values from deployment YAML are not loaded on update; use this argument or
             ``TELLR_DEPLOY_MLFLOW_*`` environment variables.
@@ -333,6 +333,7 @@ def _create_databricks(
     profile: str | None = None,
     config_yaml_path: str | None = None,
     seed_databricks_defaults: bool = True,
+    encryption_key: str | None = None,
     mlflow_tracing: dict[str, str] | None = None,
 ) -> dict[str, Any]:
     """Deploy Tellr to Databricks Apps with configurable seeding.
@@ -352,6 +353,7 @@ def _create_databricks(
         profile: Databricks CLI profile name (optional)
         config_yaml_path: Path to deployment config YAML (mutually exclusive with other args)
         seed_databricks_defaults: If True, seed Databricks-specific content on startup
+        encryption_key: Fernet key for Google OAuth encryption. Auto-generated if not provided.
         mlflow_tracing: Optional overrides for UC tracing placeholders in ``app.yaml``.
 
     Returns:
@@ -415,10 +417,11 @@ def _create_databricks(
                 lakebase_name,
                 schema_name,
                 seed_databricks_defaults=seed_databricks_defaults,
+                encryption_key=encryption_key,
                 lakebase_result=lakebase_result,
                 mlflow_tracing=mlflow_subs,
             )
-            print("   Generated app.yaml")
+            print("   Generated app.yaml (with encryption key)")
 
             print(f"Uploading to: {app_file_workspace_path}")
             _upload_files(ws, staging, app_file_workspace_path)
@@ -504,9 +507,8 @@ def _update_databricks(
         client: External WorkspaceClient (optional)
         profile: Databricks CLI profile name (optional)
         seed_databricks_defaults: If True, seed Databricks-specific content on startup
-        encryption_key: Override for the legacy Fernet key to relocate into
-            the encryption_keys table. Default: read from the deployed
-            app.yaml. The key is no longer written to app.yaml.
+        encryption_key: Fernet key for Google OAuth encryption. If not provided, reads
+            the existing key from the deployed app.yaml to preserve encrypted data.
         mlflow_tracing: Optional overrides for UC tracing placeholders in ``app.yaml``.
 
     Returns:
@@ -545,28 +547,6 @@ def _update_databricks(
             print(f"   Schema '{schema_name}' reset (tables will be recreated on app startup)")
             print()
 
-        # CRITICAL-3 migration: relocate the legacy app.yaml key into the
-        # encryption_keys table BEFORE the new (keyless) app.yaml overwrites
-        # it. Runs at most once per install: after it succeeds, the deployed
-        # app.yaml has no key entry and encryption_key is None on re-runs.
-        if encryption_key:
-            print("Relocating encryption key into Lakebase (encryption_keys)...")
-            app_for_grant = ws.apps.get(name=app_name)
-            grant_client_id = _get_app_client_id(app_for_grant)
-            if not grant_client_id:
-                print("   Warning: no app client ID — table grant will be skipped")
-            mig_conn, _ = _get_lakebase_connection(
-                ws, lakebase_name, lakebase_result=lakebase_result
-            )
-            try:
-                with mig_conn.cursor() as cur:
-                    _migrate_encryption_key_to_lakebase(
-                        cur, schema_name, grant_client_id, encryption_key
-                    )
-            finally:
-                mig_conn.close()
-            print("   Key relocated (relocate, not rotate — no re-encryption)")
-
         # Generate and upload updated files
         with _staging_dir() as staging:
             _write_requirements(staging, app_version)
@@ -575,6 +555,7 @@ def _update_databricks(
                 lakebase_name,
                 schema_name,
                 seed_databricks_defaults=seed_databricks_defaults,
+                encryption_key=encryption_key,
                 lakebase_result=lakebase_result,
                 mlflow_tracing=mlflow_subs,
             )
@@ -752,96 +733,32 @@ def _check_breaking_migrations(
 def _read_existing_encryption_key(
     ws: WorkspaceClient, workspace_path: str
 ) -> str | None:
-    """Read GOOGLE_OAUTH_ENCRYPTION_KEY from an existing deployed app.yaml.
+    """Read the GOOGLE_OAUTH_ENCRYPTION_KEY from an existing deployed app.yaml.
 
-    Returns the key string, or None when app.yaml is readable but carries no
-    key entry (already migrated, or a fresh-era install).
+    This preserves the encryption key across updates so that previously encrypted
+    credentials and tokens remain decryptable.
 
-    Raises DeploymentError when app.yaml cannot be downloaded or parsed:
-    silently treating an unreadable app.yaml as "no key" would skip the
-    CRITICAL-3 migration and let the new code generate a fresh key,
-    orphaning all existing ciphertext.
+    Returns:
+        The encryption key string, or None if not found.
     """
     try:
         resp = ws.workspace.download(f"{workspace_path}/app.yaml")
         raw = resp.read() if hasattr(resp, "read") else resp
         content = raw.decode("utf-8") if isinstance(raw, bytes) else str(raw)
+
         existing = yaml.safe_load(content)
+        for env_entry in existing.get("env", []):
+            if env_entry.get("name") == "GOOGLE_OAUTH_ENCRYPTION_KEY":
+                key = env_entry.get("value")
+                if key:
+                    print("   Preserved existing encryption key from deployed app.yaml")
+                    return key
     except Exception as e:
-        raise DeploymentError(
-            f"Could not read the deployed app.yaml at {workspace_path}/app.yaml "
-            f"({e}). Aborting: the update must know whether a legacy encryption "
-            f"key needs relocating before it overwrites app.yaml."
-        ) from e
+        logger.warning("Could not read existing encryption key from app.yaml: %s", e)
 
-    for env_entry in (existing or {}).get("env", []):
-        if env_entry.get("name") == "GOOGLE_OAUTH_ENCRYPTION_KEY":
-            key = env_entry.get("value")
-            if key:
-                print("   Found legacy encryption key in deployed app.yaml")
-                return key
     return None
 
 
-def _migrate_encryption_key_to_lakebase(
-    cur: Any,
-    schema_name: str,
-    client_id: str | None,
-    encryption_key: str,
-) -> None:
-    """Relocate the legacy app.yaml Fernet key into <schema>.encryption_keys.
-
-    SDR-4437 CRITICAL-3, deploy-time migration. Relocate, not rotate: no
-    re-encryption of existing rows. Idempotent: re-running with the same key
-    is a no-op. A *different* pre-existing key is a hard error — silently
-    keeping either key would orphan the ciphertext encrypted under the other.
-
-    The GRANT is unconditional: the original deployer's ALTER DEFAULT
-    PRIVILEGES attaches to the creating role, so when a different identity
-    runs the upgrade this explicit grant is the only one that applies.
-    INSERT is included so the app's boot seed (insert-when-missing branch)
-    can never be privilege-blocked.
-
-    Takes an open cursor so callers control the endpoint (prod branch,
-    devloop fork, ...). Caller owns the connection lifecycle.
-    """
-    validate_schema_name(schema_name)
-    if client_id is not None:
-        validate_client_id(client_id)
-
-    # Matches the SQLAlchemy model (src/database/models/encryption_key.py):
-    # id INTEGER PK (no autoincrement), key_value TEXT, created_at TIMESTAMP.
-    cur.execute(
-        f'CREATE TABLE IF NOT EXISTS "{schema_name}".encryption_keys ('
-        "id INTEGER PRIMARY KEY, "
-        "key_value TEXT NOT NULL, "
-        "created_at TIMESTAMP NOT NULL)"
-    )
-    cur.execute(
-        f'INSERT INTO "{schema_name}".encryption_keys (id, key_value, created_at) '
-        "VALUES (1, %s, CURRENT_TIMESTAMP) ON CONFLICT (id) DO NOTHING",
-        (encryption_key,),
-    )
-    cur.execute(
-        f'SELECT key_value FROM "{schema_name}".encryption_keys WHERE id = 1'
-    )
-    row = cur.fetchone()
-    if not row or row[0] != encryption_key:
-        raise DeploymentError(
-            "encryption_keys already holds a DIFFERENT key than the deployed "
-            "app.yaml. Refusing to continue: proceeding would orphan ciphertext "
-            "encrypted under one of the two keys. Determine which key decrypts "
-            "the existing google_oauth_tokens/google_global_credentials rows, "
-            "fix the encryption_keys row (or the app.yaml) to match, then "
-            "re-run the update. See SDR-4437 PR-3 design."
-        )
-    if client_id:
-        cur.execute(
-            f'GRANT SELECT, INSERT ON "{schema_name}".encryption_keys '
-            f'TO "{client_id}"'
-        )
-
-
 def _load_deployment_config(config_yaml_path: str) -> dict[str, str]:
     """Load deployment settings from config/deployment.yaml-style files.
 
@@ -1360,19 +1277,19 @@ def _write_app_yaml(
     lakebase_name: str,
     schema_name: str,
     seed_databricks_defaults: bool = False,
+    encryption_key: str | None = None,
     lakebase_result: dict[str, Any] | None = None,
     mlflow_tracing: dict[str, str] | None = None,
 ) -> None:
     """Generate app.yaml with environment variables.
 
-    The Fernet encryption key is deliberately NOT written here (SDR-4437
-    CRITICAL-3): the app reads/seeds it from the encryption_keys table.
-
     Args:
         staging_dir: Directory to write the app.yaml file
         lakebase_name: Lakebase instance name
         schema_name: Schema name
         seed_databricks_defaults: If True, include Databricks-specific content seeding
+        encryption_key: Fernet encryption key for Google OAuth credentials/tokens.
+            Auto-generated if not provided.
         lakebase_result: Result dict from _get_or_create_lakebase() with type info.
         mlflow_tracing: Resolved template keys for UC tracing (four entries). If
             omitted, values are taken only from ``TELLR_DEPLOY_MLFLOW_*`` env vars.
@@ -1383,6 +1300,12 @@ def _write_app_yaml(
     else:
         init_call = "init_database()"
 
+    if not encryption_key:
+        from cryptography.fernet import Fernet
+
+        encryption_key = Fernet.generate_key().decode()
+        logger.info("Auto-generated GOOGLE_OAUTH_ENCRYPTION_KEY for deployment")
+
     # Determine lakebase type info for env vars
     lakebase_type = (lakebase_result or {}).get("type", "provisioned")
     lakebase_pg_host = (lakebase_result or {}).get("host", "")
@@ -1397,6 +1320,7 @@ def _write_app_yaml(
         LAKEBASE_INSTANCE=lakebase_name,
         LAKEBASE_SCHEMA=schema_name,
         INIT_DATABASE_CALL=init_call,
+        GOOGLE_OAUTH_ENCRYPTION_KEY=encryption_key,
         LAKEBASE_TYPE=lakebase_type,
         LAKEBASE_PG_HOST=lakebase_pg_host,
         LAKEBASE_PROJECT_ID=lakebase_project_id,
@@ -1643,9 +1567,6 @@ def _setup_database_schema(
         print("   Warning: Could not get app client ID - schema setup skipped")
         return
 
-    validate_schema_name(schema_name)
-    validate_client_id(client_id)
-
     conn, _ = _get_lakebase_connection(ws, lakebase_name, lakebase_result=lakebase_result)
 
     try:
@@ -1676,10 +1597,6 @@ def _reset_schema(
     """
     client_id = _get_app_client_id(app)
 
-    validate_schema_name(schema_name)
-    if client_id:
-        validate_client_id(client_id)
-
     conn, _ = _get_lakebase_connection(ws, lakebase_name, lakebase_result=lakebase_result)
 
     try:
@@ -1707,9 +1624,6 @@ def _grant_schema_permissions(cur: Any, schema_name: str, client_id: str) -> Non
 
     This function only grants schema/table permissions — it does NOT create roles.
     """
-    validate_schema_name(schema_name)
-    validate_client_id(client_id)
-
     # Verify the role exists before granting
     cur.execute(
         "SELECT 1 FROM pg_roles WHERE rolname = %s", (client_id,)

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/databricks_tellr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr
-Version: 0.3.13.dev4
+Version: 0.3.13.dev12
 Summary: Tellr deployment tooling for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/databricks_tellr.egg-info/SOURCES.txt

@@ -2,7 +2,6 @@ README.md
 pyproject.toml
 databricks_tellr/__init__.py
 databricks_tellr/deploy.py
-databricks_tellr/identifiers.py
 databricks_tellr.egg-info/PKG-INFO
 databricks_tellr.egg-info/SOURCES.txt
 databricks_tellr.egg-info/dependency_links.txt

{databricks_tellr-0.3.13.dev4 → databricks_tellr-0.3.13.dev12}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "databricks-tellr"
-version = "0.3.13.dev4"
+version = "0.3.13.dev12"
 description = "Tellr deployment tooling for Databricks Apps"
 readme = "README.md"
 requires-python = ">=3.10"

databricks_tellr-0.3.13.dev4/databricks_tellr/identifiers.py

@@ -1,37 +0,0 @@
-"""Validation for identifiers interpolated into Postgres DDL (SDR-4437 MEDIUM-5).
-
-Both inputs are config/platform-derived today (schema_name from deploy
-config, client_id from the app SP), so this is hardening against future
-user-derived values, not a live injection.
-
-Lives only in this (deploy-tool) distribution: the app distribution has no
-DDL-identifier interpolation site once the dead setup_lakebase_schema is
-removed, so there is no counterpart to keep in sync.
-"""
-
-import re
-
-_SCHEMA_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
-# UUID (App SP client id) OR all-digits (str(service_principal_id) fallback
-# in _get_app_client_id). Both are injection-safe charsets.
-_CLIENT_ID_RE = re.compile(
-    r"^(?:[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}"
-    r"-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|[0-9]+)$"
-)
-
-
-def validate_schema_name(schema: str) -> str:
-    """Return *schema* if it is a safe Postgres schema identifier; else raise."""
-    if not isinstance(schema, str) or not _SCHEMA_RE.match(schema):
-        raise ValueError(f"Invalid Postgres schema name: {schema!r}")
-    return schema
-
-
-def validate_client_id(client_id: str) -> str:
-    """Return *client_id* if it is a UUID or numeric SP id; else raise."""
-    if not isinstance(client_id, str) or not _CLIENT_ID_RE.match(client_id):
-        raise ValueError(
-            f"Invalid service-principal client id (expected UUID or numeric id): "
-            f"{client_id!r}"
-        )
-    return client_id