databricks-tellr 0.3.13.dev17__tar.gz → 0.3.13.dev19__tar.gz

{databricks_tellr-0.3.13.dev17 → databricks_tellr-0.3.13.dev19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr
-Version: 0.3.13.dev17
+Version: 0.3.13.dev19
 Summary: Tellr deployment tooling for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr-0.3.13.dev17 → databricks_tellr-0.3.13.dev19}/databricks_tellr/_templates/app.yaml.template

@@ -67,4 +67,3 @@ env:
   # Set to "0" if you want to force-disable the fast path entirely.
   - name: HUASHU_PIPELINE_ENABLED
     value: "1"
-$ENCRYPTION_KEY_BLOCK

{databricks_tellr-0.3.13.dev17 → databricks_tellr-0.3.13.dev19}/databricks_tellr/deploy.py

@@ -287,9 +287,9 @@ def update(
         reset_database: If True, drop and recreate the schema (tables recreated on app startup)
         client: External WorkspaceClient (optional)
         profile: Databricks CLI profile name (optional)
-        encryption_key: Override for the legacy Fernet key to carry forward
-            into the regenerated app.yaml (boot then migrates it into the
-            encryption_keys table). Default: read from the deployed app.yaml.
+        encryption_key: Override for the legacy Fernet key to relocate into
+            the encryption_keys table. Default: read from the deployed
+            app.yaml. The key is no longer written to app.yaml.
         mlflow_tracing: Optional overrides for UC tracing env vars (same keys as ``create``).
             Values from deployment YAML are not loaded on update; use this argument or
             ``TELLR_DEPLOY_MLFLOW_*`` environment variables.
@@ -504,9 +504,9 @@ def _update_databricks(
         client: External WorkspaceClient (optional)
         profile: Databricks CLI profile name (optional)
         seed_databricks_defaults: If True, seed Databricks-specific content on startup
-        encryption_key: Override for the legacy Fernet key to carry forward
-            into the regenerated app.yaml (boot then migrates it into the
-            encryption_keys table). Default: read from the deployed app.yaml.
+        encryption_key: Override for the legacy Fernet key to relocate into
+            the encryption_keys table. Default: read from the deployed
+            app.yaml. The key is no longer written to app.yaml.
         mlflow_tracing: Optional overrides for UC tracing placeholders in ``app.yaml``.
 
     Returns:
@@ -524,7 +524,7 @@ def _update_databricks(
         overrides=mlflow_tracing,
     )
 
-    # Preserve the existing key so boot can migrate it (carry-forward).
+    # Preserve the existing encryption key so we don't invalidate encrypted data
     if not encryption_key:
         encryption_key = _read_existing_encryption_key(ws, app_file_workspace_path)
 
@@ -545,6 +545,28 @@ def _update_databricks(
             print(f"   Schema '{schema_name}' reset (tables will be recreated on app startup)")
             print()
 
+        # CRITICAL-3 migration: relocate the legacy app.yaml key into the
+        # encryption_keys table BEFORE the new (keyless) app.yaml overwrites
+        # it. Runs at most once per install: after it succeeds, the deployed
+        # app.yaml has no key entry and encryption_key is None on re-runs.
+        if encryption_key:
+            print("Relocating encryption key into Lakebase (encryption_keys)...")
+            app_for_grant = ws.apps.get(name=app_name)
+            grant_client_id = _get_app_client_id(app_for_grant)
+            if not grant_client_id:
+                print("   Warning: no app client ID — table grant will be skipped")
+            mig_conn, _ = _get_lakebase_connection(
+                ws, lakebase_name, lakebase_result=lakebase_result
+            )
+            try:
+                with mig_conn.cursor() as cur:
+                    _migrate_encryption_key_to_lakebase(
+                        cur, schema_name, grant_client_id, encryption_key
+                    )
+            finally:
+                mig_conn.close()
+            print("   Key relocated (relocate, not rotate — no re-encryption)")
+
         # Generate and upload updated files
         with _staging_dir() as staging:
             _write_requirements(staging, app_version)
@@ -555,7 +577,6 @@ def _update_databricks(
                 seed_databricks_defaults=seed_databricks_defaults,
                 lakebase_result=lakebase_result,
                 mlflow_tracing=mlflow_subs,
-                encryption_key=encryption_key,
             )
             _upload_files(ws, staging, app_file_workspace_path)
             print("   Files updated")
@@ -767,6 +788,65 @@ def _read_existing_encryption_key(
     return None
 
 
+def _migrate_encryption_key_to_lakebase(
+    cur: Any,
+    schema_name: str,
+    client_id: str | None,
+    encryption_key: str,
+) -> None:
+    """Relocate the legacy app.yaml Fernet key into <schema>.encryption_keys.
+
+    SDR-4437 CRITICAL-3, deploy-time migration. Relocate, not rotate: no
+    re-encryption of existing rows. Idempotent: re-running with the same key
+    is a no-op. A *different* pre-existing key is a hard error — silently
+    keeping either key would orphan the ciphertext encrypted under the other.
+
+    The GRANT is unconditional: the original deployer's ALTER DEFAULT
+    PRIVILEGES attaches to the creating role, so when a different identity
+    runs the upgrade this explicit grant is the only one that applies.
+    INSERT is included so the app's boot seed (insert-when-missing branch)
+    can never be privilege-blocked.
+
+    Takes an open cursor so callers control the endpoint (prod branch,
+    devloop fork, ...). Caller owns the connection lifecycle.
+    """
+    validate_schema_name(schema_name)
+    if client_id is not None:
+        validate_client_id(client_id)
+
+    # Matches the SQLAlchemy model (src/database/models/encryption_key.py):
+    # id INTEGER PK (no autoincrement), key_value TEXT, created_at TIMESTAMP.
+    cur.execute(
+        f'CREATE TABLE IF NOT EXISTS "{schema_name}".encryption_keys ('
+        "id INTEGER PRIMARY KEY, "
+        "key_value TEXT NOT NULL, "
+        "created_at TIMESTAMP NOT NULL)"
+    )
+    cur.execute(
+        f'INSERT INTO "{schema_name}".encryption_keys (id, key_value, created_at) '
+        "VALUES (1, %s, CURRENT_TIMESTAMP) ON CONFLICT (id) DO NOTHING",
+        (encryption_key,),
+    )
+    cur.execute(
+        f'SELECT key_value FROM "{schema_name}".encryption_keys WHERE id = 1'
+    )
+    row = cur.fetchone()
+    if not row or row[0] != encryption_key:
+        raise DeploymentError(
+            "encryption_keys already holds a DIFFERENT key than the deployed "
+            "app.yaml. Refusing to continue: proceeding would orphan ciphertext "
+            "encrypted under one of the two keys. Determine which key decrypts "
+            "the existing google_oauth_tokens/google_global_credentials rows, "
+            "fix the encryption_keys row (or the app.yaml) to match, then "
+            "re-run the update. See SDR-4437 PR-3 design."
+        )
+    if client_id:
+        cur.execute(
+            f'GRANT SELECT, INSERT ON "{schema_name}".encryption_keys '
+            f'TO "{client_id}"'
+        )
+
+
 def _load_deployment_config(config_yaml_path: str) -> dict[str, str]:
     """Load deployment settings from config/deployment.yaml-style files.
 
@@ -1287,14 +1367,11 @@ def _write_app_yaml(
     seed_databricks_defaults: bool = False,
     lakebase_result: dict[str, Any] | None = None,
     mlflow_tracing: dict[str, str] | None = None,
-    encryption_key: str | None = None,
 ) -> None:
     """Generate app.yaml with environment variables.
 
-    The Fernet encryption key is written ONLY when ``encryption_key`` is
-    provided (the one-time legacy→table carry-forward). Boot seeds the
-    encryption_keys table from it and then scrubs it from app.yaml. When
-    ``encryption_key`` is None the app.yaml is keyless (steady state).
+    The Fernet encryption key is deliberately NOT written here (SDR-4437
+    CRITICAL-3): the app reads/seeds it from the encryption_keys table.
 
     Args:
         staging_dir: Directory to write the app.yaml file
@@ -1320,17 +1397,8 @@ def _write_app_yaml(
     if mlflow_tracing is None:
         mlflow_tracing = _mlflow_substitutions_for_app_yaml()
 
-    if encryption_key:
-        key_block = (
-            "  - name: GOOGLE_OAUTH_ENCRYPTION_KEY\n"
-            f'    value: "{encryption_key}"'
-        )
-    else:
-        key_block = ""
-
     template_content = _load_template("app.yaml.template")
     content = Template(template_content).substitute(
-        ENCRYPTION_KEY_BLOCK=key_block,
         LAKEBASE_INSTANCE=lakebase_name,
         LAKEBASE_SCHEMA=schema_name,
         INIT_DATABASE_CALL=init_call,

{databricks_tellr-0.3.13.dev17 → databricks_tellr-0.3.13.dev19}/databricks_tellr.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: databricks-tellr
-Version: 0.3.13.dev17
+Version: 0.3.13.dev19
 Summary: Tellr deployment tooling for Databricks Apps
 Requires-Python: >=3.10
 Description-Content-Type: text/markdown

{databricks_tellr-0.3.13.dev17 → databricks_tellr-0.3.13.dev19}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "databricks-tellr"
-version = "0.3.13.dev17"
+version = "0.3.13.dev19"
 description = "Tellr deployment tooling for Databricks Apps"
 readme = "README.md"
 requires-python = ">=3.10"