databricks-tellr 0.4.3.dev18__tar.gz → 0.4.3.dev20__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/PKG-INFO +1 -2
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/_templates/app.yaml.template +0 -1
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/deploy.py +6 -160
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/PKG-INFO +1 -2
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/SOURCES.txt +0 -1
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/requires.txt +0 -1
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/pyproject.toml +1 -2
- databricks_tellr-0.4.3.dev18/databricks_tellr/secret_key.py +0 -354
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/README.md +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/__init__.py +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/_templates/requirements.txt.template +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/identifiers.py +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/dependency_links.txt +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/top_level.txt +0 -0
- {databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/setup.cfg +0 -0
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: databricks-tellr
|
|
3
|
-
Version: 0.4.3.
|
|
3
|
+
Version: 0.4.3.dev20
|
|
4
4
|
Summary: Tellr deployment tooling for Databricks Apps
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: databricks-sdk>=0.85.0
|
|
8
8
|
Requires-Dist: psycopg2-binary>=2.9.0
|
|
9
9
|
Requires-Dist: pyyaml>=6.0.0
|
|
10
|
-
Requires-Dist: requests>=2.31.0
|
|
11
10
|
|
|
12
11
|
# databricks-tellr
|
|
13
12
|
|
|
@@ -30,7 +30,6 @@ from databricks.sdk.service.apps import (
|
|
|
30
30
|
from databricks.sdk.service.database import DatabaseInstance
|
|
31
31
|
from databricks.sdk.service.workspace import ImportFormat
|
|
32
32
|
|
|
33
|
-
from databricks_tellr import secret_key
|
|
34
33
|
from databricks_tellr.identifiers import validate_client_id, validate_schema_name
|
|
35
34
|
|
|
36
35
|
# Autoscaling imports (Lakebase next-gen)
|
|
@@ -199,8 +198,6 @@ def create(
|
|
|
199
198
|
profile: str | None = None,
|
|
200
199
|
config_yaml_path: str | None = None,
|
|
201
200
|
mlflow_tracing: dict[str, str] | None = None,
|
|
202
|
-
encryption_secret_scope: str | None = None,
|
|
203
|
-
encryption_secret_key: str = secret_key.DEFAULT_SECRET_KEY,
|
|
204
201
|
) -> dict[str, Any]:
|
|
205
202
|
"""Deploy Tellr to Databricks Apps.
|
|
206
203
|
|
|
@@ -235,11 +232,6 @@ def create(
|
|
|
235
232
|
``config_yaml_path``, YAML ``mlflow_tracing`` applies first; non-empty
|
|
236
233
|
entries here override. Empty slots can be filled from deploy-time env
|
|
237
234
|
vars ``TELLR_DEPLOY_MLFLOW_*``.
|
|
238
|
-
encryption_secret_scope: Opt in to the secret-backed Fernet key. When set,
|
|
239
|
-
the key is stored in this Databricks secret scope and attached to the
|
|
240
|
-
app as a secret resource instead of living in the encryption_keys
|
|
241
|
-
Lakebase table. When omitted, the Lakebase-backed path is used.
|
|
242
|
-
encryption_secret_key: Secret key name within that scope.
|
|
243
235
|
|
|
244
236
|
Returns:
|
|
245
237
|
Dictionary with deployment info:
|
|
@@ -267,8 +259,6 @@ def create(
|
|
|
267
259
|
config_yaml_path=config_yaml_path,
|
|
268
260
|
seed_databricks_defaults=False,
|
|
269
261
|
mlflow_tracing=mlflow_tracing,
|
|
270
|
-
encryption_secret_scope=encryption_secret_scope,
|
|
271
|
-
encryption_secret_key=encryption_secret_key,
|
|
272
262
|
)
|
|
273
263
|
|
|
274
264
|
|
|
@@ -283,8 +273,6 @@ def update(
|
|
|
283
273
|
profile: str | None = None,
|
|
284
274
|
encryption_key: str | None = None,
|
|
285
275
|
mlflow_tracing: dict[str, str] | None = None,
|
|
286
|
-
encryption_secret_scope: str | None = None,
|
|
287
|
-
encryption_secret_key: str = secret_key.DEFAULT_SECRET_KEY,
|
|
288
276
|
) -> dict[str, Any]:
|
|
289
277
|
"""Deploy a new version of an existing Tellr app.
|
|
290
278
|
|
|
@@ -305,11 +293,6 @@ def update(
|
|
|
305
293
|
mlflow_tracing: Optional overrides for UC tracing env vars (same keys as ``create``).
|
|
306
294
|
Values from deployment YAML are not loaded on update; use this argument or
|
|
307
295
|
``TELLR_DEPLOY_MLFLOW_*`` environment variables.
|
|
308
|
-
encryption_secret_scope: Databricks secret scope that holds the Fernet
|
|
309
|
-
master key (opt-in secret mode). When set, the key is stored in and
|
|
310
|
-
read from this scope rather than Lakebase.
|
|
311
|
-
encryption_secret_key: Name of the secret within *encryption_secret_scope*.
|
|
312
|
-
Defaults to ``DEFAULT_SECRET_KEY`` ("tellr-encryption-key").
|
|
313
296
|
|
|
314
297
|
Returns:
|
|
315
298
|
Dictionary with deployment info
|
|
@@ -329,8 +312,6 @@ def update(
|
|
|
329
312
|
seed_databricks_defaults=False,
|
|
330
313
|
encryption_key=encryption_key,
|
|
331
314
|
mlflow_tracing=mlflow_tracing,
|
|
332
|
-
encryption_secret_scope=encryption_secret_scope,
|
|
333
|
-
encryption_secret_key=encryption_secret_key,
|
|
334
315
|
)
|
|
335
316
|
|
|
336
317
|
|
|
@@ -353,8 +334,6 @@ def _create_databricks(
|
|
|
353
334
|
config_yaml_path: str | None = None,
|
|
354
335
|
seed_databricks_defaults: bool = True,
|
|
355
336
|
mlflow_tracing: dict[str, str] | None = None,
|
|
356
|
-
encryption_secret_scope: str | None = None,
|
|
357
|
-
encryption_secret_key: str = secret_key.DEFAULT_SECRET_KEY,
|
|
358
337
|
) -> dict[str, Any]:
|
|
359
338
|
"""Deploy Tellr to Databricks Apps with configurable seeding.
|
|
360
339
|
|
|
@@ -398,25 +377,6 @@ def _create_databricks(
|
|
|
398
377
|
app_file_workspace_path = config.get("app_file_workspace_path")
|
|
399
378
|
lakebase_compute = config.get("lakebase_compute", lakebase_compute)
|
|
400
379
|
app_compute = config.get("app_compute", app_compute)
|
|
401
|
-
encryption_secret_scope = encryption_secret_scope or config.get(
|
|
402
|
-
"encryption_secret_scope"
|
|
403
|
-
)
|
|
404
|
-
encryption_secret_key = (
|
|
405
|
-
config.get("encryption_secret_key") or encryption_secret_key
|
|
406
|
-
)
|
|
407
|
-
|
|
408
|
-
# Preflight before anything is created, so a failure leaves nothing behind.
|
|
409
|
-
resolved_key: str | None = None
|
|
410
|
-
if encryption_secret_scope:
|
|
411
|
-
print(f"Secret-backed encryption key: {encryption_secret_scope}/{encryption_secret_key}")
|
|
412
|
-
secret_key.preflight_scope(ws, encryption_secret_scope)
|
|
413
|
-
resolved_key = secret_key.resolve_key_for_create(
|
|
414
|
-
ws, encryption_secret_scope, encryption_secret_key
|
|
415
|
-
)
|
|
416
|
-
|
|
417
|
-
encryption_secret_resource_key = (
|
|
418
|
-
secret_key.RESOURCE_KEY if encryption_secret_scope else None
|
|
419
|
-
)
|
|
420
380
|
|
|
421
381
|
mlflow_subs = _mlflow_substitutions_for_app_yaml(
|
|
422
382
|
deployment_flat=deployment_flat_for_mlflow,
|
|
@@ -457,7 +417,6 @@ def _create_databricks(
|
|
|
457
417
|
seed_databricks_defaults=seed_databricks_defaults,
|
|
458
418
|
lakebase_result=lakebase_result,
|
|
459
419
|
mlflow_tracing=mlflow_subs,
|
|
460
|
-
encryption_secret_resource_key=encryption_secret_resource_key,
|
|
461
420
|
)
|
|
462
421
|
print(" Generated app.yaml")
|
|
463
422
|
|
|
@@ -476,8 +435,6 @@ def _create_databricks(
|
|
|
476
435
|
compute_size=app_compute,
|
|
477
436
|
lakebase_name=lakebase_name,
|
|
478
437
|
lakebase_type=lakebase_type,
|
|
479
|
-
encryption_secret_scope=encryption_secret_scope,
|
|
480
|
-
encryption_secret_key=encryption_secret_key,
|
|
481
438
|
)
|
|
482
439
|
print(" App registered")
|
|
483
440
|
print()
|
|
@@ -532,11 +489,9 @@ def _update_databricks(
|
|
|
532
489
|
seed_databricks_defaults: bool = True,
|
|
533
490
|
encryption_key: str | None = None,
|
|
534
491
|
mlflow_tracing: dict[str, str] | None = None,
|
|
535
|
-
encryption_secret_scope: str | None = None,
|
|
536
|
-
encryption_secret_key: str = secret_key.DEFAULT_SECRET_KEY,
|
|
537
492
|
) -> dict[str, Any]:
|
|
538
493
|
"""Deploy a new version of an existing Tellr app with configurable seeding.
|
|
539
|
-
|
|
494
|
+
|
|
540
495
|
Internal function with full control over seeding behavior.
|
|
541
496
|
|
|
542
497
|
Args:
|
|
@@ -553,11 +508,6 @@ def _update_databricks(
|
|
|
553
508
|
the encryption_keys table. Default: read from the deployed
|
|
554
509
|
app.yaml. The key is no longer written to app.yaml.
|
|
555
510
|
mlflow_tracing: Optional overrides for UC tracing placeholders in ``app.yaml``.
|
|
556
|
-
encryption_secret_scope: Databricks secret scope that holds the Fernet
|
|
557
|
-
master key (opt-in secret mode). When set, the key is stored in and
|
|
558
|
-
read from this scope rather than Lakebase.
|
|
559
|
-
encryption_secret_key: Name of the secret within *encryption_secret_scope*.
|
|
560
|
-
Defaults to ``DEFAULT_SECRET_KEY`` ("tellr-encryption-key").
|
|
561
511
|
|
|
562
512
|
Returns:
|
|
563
513
|
Dictionary with deployment info
|
|
@@ -569,24 +519,6 @@ def _update_databricks(
|
|
|
569
519
|
|
|
570
520
|
ws = _get_workspace_client(client, profile)
|
|
571
521
|
|
|
572
|
-
app_already_secret = secret_key.app_is_secret_mode(ws, app_name)
|
|
573
|
-
if not encryption_secret_scope and app_already_secret:
|
|
574
|
-
if encryption_key:
|
|
575
|
-
raise DeploymentError(
|
|
576
|
-
f"App {app_name} uses a secret-backed encryption key, so passing "
|
|
577
|
-
f"encryption_key would recreate the Lakebase key row this app was "
|
|
578
|
-
f"migrated off — and a mismatched value would silently orphan "
|
|
579
|
-
f"stored credentials. Re-run with "
|
|
580
|
-
f"encryption_secret_scope=... instead."
|
|
581
|
-
)
|
|
582
|
-
print(" Secret-backed encryption key retained (app resource unchanged)")
|
|
583
|
-
|
|
584
|
-
encryption_secret_resource_key = (
|
|
585
|
-
secret_key.RESOURCE_KEY
|
|
586
|
-
if (encryption_secret_scope or app_already_secret)
|
|
587
|
-
else None
|
|
588
|
-
)
|
|
589
|
-
|
|
590
522
|
mlflow_subs = _mlflow_substitutions_for_app_yaml(
|
|
591
523
|
deployment_flat={},
|
|
592
524
|
overrides=mlflow_tracing,
|
|
@@ -613,32 +545,11 @@ def _update_databricks(
|
|
|
613
545
|
print(f" Schema '{schema_name}' reset (tables will be recreated on app startup)")
|
|
614
546
|
print()
|
|
615
547
|
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
ws, lakebase_name, lakebase_result=lakebase_result
|
|
622
|
-
)
|
|
623
|
-
try:
|
|
624
|
-
with mig_conn.cursor() as cur:
|
|
625
|
-
secret_key.preflight_lakebase_privileges(cur, schema_name)
|
|
626
|
-
lakebase_key = secret_key.read_lakebase_key(cur, schema_name)
|
|
627
|
-
finally:
|
|
628
|
-
mig_conn.close()
|
|
629
|
-
|
|
630
|
-
_, wrote = secret_key.resolve_key_for_update(
|
|
631
|
-
ws, encryption_secret_scope, encryption_secret_key,
|
|
632
|
-
lakebase_key, encryption_key,
|
|
633
|
-
)
|
|
634
|
-
if wrote:
|
|
635
|
-
print(" Key written to the secret and verified")
|
|
636
|
-
secret_key.attach_secret_resource(
|
|
637
|
-
ws, app_name, encryption_secret_scope, encryption_secret_key
|
|
638
|
-
)
|
|
639
|
-
print(" Secret resource attached")
|
|
640
|
-
elif encryption_key:
|
|
641
|
-
# Legacy CRITICAL-3 relocation into Lakebase — unchanged.
|
|
548
|
+
# CRITICAL-3 migration: relocate the legacy app.yaml key into the
|
|
549
|
+
# encryption_keys table BEFORE the new (keyless) app.yaml overwrites
|
|
550
|
+
# it. Runs at most once per install: after it succeeds, the deployed
|
|
551
|
+
# app.yaml has no key entry and encryption_key is None on re-runs.
|
|
552
|
+
if encryption_key:
|
|
642
553
|
print("Relocating encryption key into Lakebase (encryption_keys)...")
|
|
643
554
|
app_for_grant = ws.apps.get(name=app_name)
|
|
644
555
|
grant_client_id = _get_app_client_id(app_for_grant)
|
|
@@ -666,7 +577,6 @@ def _update_databricks(
|
|
|
666
577
|
seed_databricks_defaults=seed_databricks_defaults,
|
|
667
578
|
lakebase_result=lakebase_result,
|
|
668
579
|
mlflow_tracing=mlflow_subs,
|
|
669
|
-
encryption_secret_resource_key=encryption_secret_resource_key,
|
|
670
580
|
)
|
|
671
581
|
_upload_files(ws, staging, app_file_workspace_path)
|
|
672
582
|
print(" Files updated")
|
|
@@ -681,35 +591,6 @@ def _update_databricks(
|
|
|
681
591
|
if app.url:
|
|
682
592
|
print(f" URL: {app.url}")
|
|
683
593
|
|
|
684
|
-
if encryption_secret_scope:
|
|
685
|
-
if not app.url:
|
|
686
|
-
print(" WARNING: no app URL — cannot confirm the key source; "
|
|
687
|
-
"leaving the Lakebase key row in place")
|
|
688
|
-
elif secret_key.app_reports_secret_source(ws, app.url):
|
|
689
|
-
del_conn = None
|
|
690
|
-
try:
|
|
691
|
-
del_conn, _ = _get_lakebase_connection(
|
|
692
|
-
ws, lakebase_name, lakebase_result=lakebase_result
|
|
693
|
-
)
|
|
694
|
-
with del_conn.cursor() as cur:
|
|
695
|
-
if secret_key.read_lakebase_key(cur, schema_name):
|
|
696
|
-
secret_key.delete_lakebase_key_row(cur, schema_name)
|
|
697
|
-
print(" Lakebase key row deleted — the secret is now "
|
|
698
|
-
"the only copy")
|
|
699
|
-
else:
|
|
700
|
-
print(" No Lakebase key row to remove")
|
|
701
|
-
except Exception as exc: # noqa: BLE001 — deploy already succeeded
|
|
702
|
-
print(f" WARNING: could not delete the Lakebase key row: {exc}")
|
|
703
|
-
print(f' Run manually: DELETE FROM "{schema_name}".'
|
|
704
|
-
f"encryption_keys WHERE id = 1;")
|
|
705
|
-
finally:
|
|
706
|
-
if del_conn is not None:
|
|
707
|
-
del_conn.close()
|
|
708
|
-
else:
|
|
709
|
-
print(" WARNING: the deployed app does not report the secret as "
|
|
710
|
-
"its key source. Leaving the Lakebase key row in place. This "
|
|
711
|
-
"is expected if the app version predates secret-mode support.")
|
|
712
|
-
|
|
713
594
|
return {
|
|
714
595
|
"url": app.url,
|
|
715
596
|
"app_name": app_name,
|
|
@@ -735,12 +616,6 @@ def delete(
|
|
|
735
616
|
|
|
736
617
|
Note: This does not delete the Lakebase instance by default.
|
|
737
618
|
|
|
738
|
-
Deliberately does NOT touch secrets or secret scopes. A devloop fork shares
|
|
739
|
-
the production app's secret scope and key by design (so it can decrypt
|
|
740
|
-
inherited ciphertext), which means deleting the secret here would destroy
|
|
741
|
-
production's Fernet master key during a fork teardown. Only the app and its
|
|
742
|
-
Lakebase branch are removed. Do not "clean up" the secret here.
|
|
743
|
-
|
|
744
619
|
Args:
|
|
745
620
|
app_name: Name of the app to delete
|
|
746
621
|
lakebase_name: Lakebase instance name (required if reset_database=True)
|
|
@@ -1013,8 +888,6 @@ def _load_deployment_config(config_yaml_path: str) -> dict[str, str]:
|
|
|
1013
888
|
"lakebase_name": lakebase_config.get("database_name"),
|
|
1014
889
|
"schema_name": lakebase_config.get("schema"),
|
|
1015
890
|
"lakebase_compute": lakebase_config.get("capacity"),
|
|
1016
|
-
"encryption_secret_scope": env_config.get("encryption_secret_scope"),
|
|
1017
|
-
"encryption_secret_key": env_config.get("encryption_secret_key"),
|
|
1018
891
|
**ml_flat,
|
|
1019
892
|
}
|
|
1020
893
|
|
|
@@ -1494,7 +1367,6 @@ def _write_app_yaml(
|
|
|
1494
1367
|
seed_databricks_defaults: bool = False,
|
|
1495
1368
|
lakebase_result: dict[str, Any] | None = None,
|
|
1496
1369
|
mlflow_tracing: dict[str, str] | None = None,
|
|
1497
|
-
encryption_secret_resource_key: str | None = None,
|
|
1498
1370
|
) -> None:
|
|
1499
1371
|
"""Generate app.yaml with environment variables.
|
|
1500
1372
|
|
|
@@ -1509,11 +1381,6 @@ def _write_app_yaml(
|
|
|
1509
1381
|
lakebase_result: Result dict from _get_or_create_lakebase() with type info.
|
|
1510
1382
|
mlflow_tracing: Resolved template keys for UC tracing (four entries). If
|
|
1511
1383
|
omitted, values are taken only from ``TELLR_DEPLOY_MLFLOW_*`` env vars.
|
|
1512
|
-
encryption_secret_resource_key: When set, add an ``env:`` entry mapping
|
|
1513
|
-
this Apps secret resource key into the environment. The resource
|
|
1514
|
-
declaration alone does not inject anything — the ``valueFrom`` entry
|
|
1515
|
-
is required (verified live). No key material is written; this is a
|
|
1516
|
-
resource reference only. Leave None for the Lakebase-backed path.
|
|
1517
1384
|
"""
|
|
1518
1385
|
# Build init_database call - only show seed_databricks_defaults when True
|
|
1519
1386
|
if seed_databricks_defaults:
|
|
@@ -1530,15 +1397,6 @@ def _write_app_yaml(
|
|
|
1530
1397
|
if mlflow_tracing is None:
|
|
1531
1398
|
mlflow_tracing = _mlflow_substitutions_for_app_yaml()
|
|
1532
1399
|
|
|
1533
|
-
if encryption_secret_resource_key:
|
|
1534
|
-
# The Apps secret resource supplies the value; this only names it.
|
|
1535
|
-
secret_env_block = (
|
|
1536
|
-
f" - name: {encryption_secret_resource_key}\n"
|
|
1537
|
-
f' valueFrom: "{encryption_secret_resource_key}"\n'
|
|
1538
|
-
)
|
|
1539
|
-
else:
|
|
1540
|
-
secret_env_block = ""
|
|
1541
|
-
|
|
1542
1400
|
template_content = _load_template("app.yaml.template")
|
|
1543
1401
|
content = Template(template_content).substitute(
|
|
1544
1402
|
LAKEBASE_INSTANCE=lakebase_name,
|
|
@@ -1556,7 +1414,6 @@ def _write_app_yaml(
|
|
|
1556
1414
|
TELLR_MLFLOW_UC_TABLE_PREFIX=mlflow_tracing.get(
|
|
1557
1415
|
"TELLR_MLFLOW_UC_TABLE_PREFIX", ""
|
|
1558
1416
|
),
|
|
1559
|
-
ENCRYPTION_SECRET_ENV_BLOCK=secret_env_block,
|
|
1560
1417
|
)
|
|
1561
1418
|
(staging_dir / "app.yaml").write_text(content)
|
|
1562
1419
|
|
|
@@ -1628,8 +1485,6 @@ def _create_app(
|
|
|
1628
1485
|
compute_size: str,
|
|
1629
1486
|
lakebase_name: str,
|
|
1630
1487
|
lakebase_type: str = "provisioned",
|
|
1631
|
-
encryption_secret_scope: str | None = None,
|
|
1632
|
-
encryption_secret_key: str | None = None,
|
|
1633
1488
|
) -> App:
|
|
1634
1489
|
"""Create Databricks App with database resource (without deploying).
|
|
1635
1490
|
|
|
@@ -1659,15 +1514,6 @@ def _create_app(
|
|
|
1659
1514
|
app_resources = []
|
|
1660
1515
|
logger.info("Autoscaling mode: skipping AppResourceDatabase (using env vars)")
|
|
1661
1516
|
|
|
1662
|
-
# Both branches: autoscaling builds an empty resource list, so appending
|
|
1663
|
-
# after the branch covers provisioned and autoscaling alike.
|
|
1664
|
-
if encryption_secret_scope:
|
|
1665
|
-
app_resources.append(
|
|
1666
|
-
secret_key.build_secret_resource(
|
|
1667
|
-
encryption_secret_scope, encryption_secret_key
|
|
1668
|
-
)
|
|
1669
|
-
)
|
|
1670
|
-
|
|
1671
1517
|
app = App(
|
|
1672
1518
|
name=app_name,
|
|
1673
1519
|
description=description,
|
{databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr.egg-info/PKG-INFO
RENAMED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: databricks-tellr
|
|
3
|
-
Version: 0.4.3.
|
|
3
|
+
Version: 0.4.3.dev20
|
|
4
4
|
Summary: Tellr deployment tooling for Databricks Apps
|
|
5
5
|
Requires-Python: >=3.10
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
7
7
|
Requires-Dist: databricks-sdk>=0.85.0
|
|
8
8
|
Requires-Dist: psycopg2-binary>=2.9.0
|
|
9
9
|
Requires-Dist: pyyaml>=6.0.0
|
|
10
|
-
Requires-Dist: requests>=2.31.0
|
|
11
10
|
|
|
12
11
|
# databricks-tellr
|
|
13
12
|
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "databricks-tellr"
|
|
7
|
-
version = "0.4.3.
|
|
7
|
+
version = "0.4.3.dev20"
|
|
8
8
|
description = "Tellr deployment tooling for Databricks Apps"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.10"
|
|
@@ -12,7 +12,6 @@ dependencies = [
|
|
|
12
12
|
"databricks-sdk>=0.85.0",
|
|
13
13
|
"psycopg2-binary>=2.9.0",
|
|
14
14
|
"pyyaml>=6.0.0",
|
|
15
|
-
"requests>=2.31.0",
|
|
16
15
|
]
|
|
17
16
|
|
|
18
17
|
[tool.setuptools.packages.find]
|
|
@@ -1,354 +0,0 @@
|
|
|
1
|
-
"""Secret-backed Fernet master key helpers for the Tellr deploy tool.
|
|
2
|
-
|
|
3
|
-
The Lakebase-backed path (SDR-4437 CRITICAL-3) remains the default; everything
|
|
4
|
-
here serves the opt-in path that stores the key in a Databricks secret and
|
|
5
|
-
attaches it to the app as a secret resource. Design and the live-verified
|
|
6
|
-
platform behaviour this relies on:
|
|
7
|
-
docs/superpowers/specs/2026-08-20-secret-backed-fernet-key-design.md
|
|
8
|
-
|
|
9
|
-
Deliberately a separate module: deploy.py is already very large, and keeping
|
|
10
|
-
these helpers here makes them unit-testable without mocking a whole deployment.
|
|
11
|
-
"""
|
|
12
|
-
|
|
13
|
-
from __future__ import annotations
|
|
14
|
-
|
|
15
|
-
import base64
|
|
16
|
-
import logging
|
|
17
|
-
import time
|
|
18
|
-
from typing import Any
|
|
19
|
-
|
|
20
|
-
import requests
|
|
21
|
-
from cryptography.fernet import Fernet
|
|
22
|
-
from databricks.sdk.errors import NotFound
|
|
23
|
-
from databricks.sdk.service.apps import (
|
|
24
|
-
App,
|
|
25
|
-
AppResource,
|
|
26
|
-
AppResourceSecret,
|
|
27
|
-
AppResourceSecretSecretPermission,
|
|
28
|
-
)
|
|
29
|
-
from databricks.sdk.service.workspace import ScopeBackendType
|
|
30
|
-
|
|
31
|
-
logger = logging.getLogger(__name__)
|
|
32
|
-
|
|
33
|
-
#: Apps secret-resource key, and therefore the injected env var name. Verified
|
|
34
|
-
#: that uppercase-underscore resource keys are accepted.
|
|
35
|
-
RESOURCE_KEY = "TELLR_ENCRYPTION_KEY"
|
|
36
|
-
|
|
37
|
-
#: Default secret key name. Spaces are actually accepted by the API, but a
|
|
38
|
-
#: hyphenated name keeps it consistent with the resource key.
|
|
39
|
-
DEFAULT_SECRET_KEY = "tellr-encryption-key"
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
class SecretKeyError(Exception):
|
|
43
|
-
"""Raised for any recoverable secret-mode setup failure."""
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
def preflight_scope(ws: Any, scope: str) -> None:
|
|
47
|
-
"""Ensure *scope* exists and is usable, before any key material is written.
|
|
48
|
-
|
|
49
|
-
Runs before the app is created or updated so a failure leaves nothing
|
|
50
|
-
half-done. Creates the scope when absent, with ``initial_manage_principal``
|
|
51
|
-
omitted so the creator is sole manager — explicitly NOT ``"users"``, which
|
|
52
|
-
would grant every workspace user MANAGE on the scope holding the key.
|
|
53
|
-
"""
|
|
54
|
-
existing = {s.name: s for s in ws.secrets.list_scopes()}
|
|
55
|
-
if scope in existing:
|
|
56
|
-
backend = existing[scope].backend_type
|
|
57
|
-
if backend is not None and backend != ScopeBackendType.DATABRICKS:
|
|
58
|
-
raise SecretKeyError(
|
|
59
|
-
f"Secret scope '{scope}' is {backend} backed. Secret mode writes "
|
|
60
|
-
f"the key and reads it back to verify; Key Vault backed scopes are "
|
|
61
|
-
f"written and read through Key Vault, not this API. Use a "
|
|
62
|
-
f"Databricks-backed scope."
|
|
63
|
-
)
|
|
64
|
-
return
|
|
65
|
-
|
|
66
|
-
try:
|
|
67
|
-
ws.secrets.create_scope(scope=scope)
|
|
68
|
-
except Exception as exc: # noqa: BLE001 — mapped to actionable guidance below
|
|
69
|
-
text = str(exc).upper()
|
|
70
|
-
manual = f"databricks secrets create-scope {scope}"
|
|
71
|
-
if "LIMIT" in text:
|
|
72
|
-
raise SecretKeyError(
|
|
73
|
-
f"Cannot create secret scope '{scope}': this workspace is at its "
|
|
74
|
-
f"secret-scope limit (1,000 by default, raisable on request). Pass "
|
|
75
|
-
f"an existing scope via encryption_secret_scope instead."
|
|
76
|
-
) from exc
|
|
77
|
-
if "UNAUTHORIZED" in text or "PERMISSION" in text or "BAD_REQUEST" in text:
|
|
78
|
-
raise SecretKeyError(
|
|
79
|
-
f"Not permitted to create secret scope '{scope}'. Ask a workspace "
|
|
80
|
-
f"admin to run `{manual}` and grant you MANAGE on it, then re-run."
|
|
81
|
-
) from exc
|
|
82
|
-
raise SecretKeyError(
|
|
83
|
-
f"Could not create secret scope '{scope}': {exc}. Create it manually "
|
|
84
|
-
f"with `{manual}` and re-run."
|
|
85
|
-
) from exc
|
|
86
|
-
logger.info("Created secret scope %s (creator is sole manager)", scope)
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
def preflight_lakebase_privileges(cur: Any, schema_name: str) -> None:
|
|
90
|
-
"""Verify the deploying human can read and delete the encryption_keys row.
|
|
91
|
-
|
|
92
|
-
On installs where the app SP created the table and ``REASSIGN OWNED`` rehomed
|
|
93
|
-
it to ``tellr_app_owners``, a deployer outside ``databricks_superuser`` has
|
|
94
|
-
neither privilege. Without this probe a denied SELECT would be indistinguishable
|
|
95
|
-
from "no row", and the relocate ladder would mint a fresh key over live
|
|
96
|
-
ciphertext.
|
|
97
|
-
"""
|
|
98
|
-
table = f"{schema_name}.encryption_keys"
|
|
99
|
-
missing = []
|
|
100
|
-
for priv in ("SELECT", "DELETE"):
|
|
101
|
-
cur.execute(
|
|
102
|
-
"SELECT has_table_privilege(current_user, %s, %s)", (table, priv)
|
|
103
|
-
)
|
|
104
|
-
row = cur.fetchone()
|
|
105
|
-
if not row or not row[0]:
|
|
106
|
-
missing.append(priv)
|
|
107
|
-
if missing:
|
|
108
|
-
raise SecretKeyError(
|
|
109
|
-
f"The deploying identity lacks {' and '.join(missing)} on {table}, so "
|
|
110
|
-
f"the key cannot be safely relocated out of Lakebase. Grant those "
|
|
111
|
-
f"privileges (or run the deploy as an identity in databricks_superuser) "
|
|
112
|
-
f"and re-run. Nothing has been changed."
|
|
113
|
-
)
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
def _looks_absent(exc: Exception) -> bool:
|
|
117
|
-
"""True when *exc* means the secret/scope genuinely does not exist.
|
|
118
|
-
|
|
119
|
-
The SDK raises a typed ``NotFound`` (subclass ``ResourceDoesNotExist``,
|
|
120
|
-
``error_code == "RESOURCE_DOES_NOT_EXIST"``) whose ``str()`` is a human
|
|
121
|
-
message like ``"Failed to get secret X for scope Y"`` that does NOT contain
|
|
122
|
-
the code — so match on the exception TYPE (and ``error_code``), never the
|
|
123
|
-
message text. A permission denial is a distinct ``PermissionDenied`` (403)
|
|
124
|
-
that is not a ``NotFound``, so it is correctly NOT treated as absent — the
|
|
125
|
-
guard against overwriting a live key still holds.
|
|
126
|
-
"""
|
|
127
|
-
if isinstance(exc, NotFound):
|
|
128
|
-
return True
|
|
129
|
-
if getattr(exc, "error_code", None) == "RESOURCE_DOES_NOT_EXIST":
|
|
130
|
-
return True
|
|
131
|
-
# Fallback for un-typed errors that still name the code in their text.
|
|
132
|
-
return "RESOURCE_DOES_NOT_EXIST" in str(exc).upper()
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
def read_secret_key(ws: Any, scope: str, key: str) -> str | None:
|
|
136
|
-
"""Return the Fernet key stored at *(scope, key)*, or None if truly absent.
|
|
137
|
-
|
|
138
|
-
``GetSecretResponse.value`` is base64 (the API returns the value "in its byte
|
|
139
|
-
representation"), so it must be decoded. Any error other than a genuine
|
|
140
|
-
not-found is raised: treating a permission denial as absence would let the
|
|
141
|
-
caller overwrite a secret that is already protecting live ciphertext.
|
|
142
|
-
"""
|
|
143
|
-
try:
|
|
144
|
-
resp = ws.secrets.get_secret(scope=scope, key=key)
|
|
145
|
-
except Exception as exc: # noqa: BLE001
|
|
146
|
-
if _looks_absent(exc):
|
|
147
|
-
return None
|
|
148
|
-
_code = getattr(exc, "error_code", None)
|
|
149
|
-
raise SecretKeyError(
|
|
150
|
-
f"Could not read secret {scope}/{key}: "
|
|
151
|
-
f"[{type(exc).__name__}{'/' + _code if _code else ''}] {exc}. "
|
|
152
|
-
f"This is not a recognized 'not found' error (most likely a permission "
|
|
153
|
-
f"problem) — refusing to continue rather than risk overwriting a key "
|
|
154
|
-
f"that may already protect stored credentials."
|
|
155
|
-
) from exc
|
|
156
|
-
|
|
157
|
-
if resp is None or not resp.value:
|
|
158
|
-
return None
|
|
159
|
-
value = base64.b64decode(resp.value).decode()
|
|
160
|
-
try:
|
|
161
|
-
Fernet(value.encode())
|
|
162
|
-
except (ValueError, TypeError) as exc:
|
|
163
|
-
raise SecretKeyError(
|
|
164
|
-
f"Secret {scope}/{key} exists but is not a valid Fernet key. Refusing "
|
|
165
|
-
f"to use or overwrite it — inspect it manually and resolve."
|
|
166
|
-
) from exc
|
|
167
|
-
return value
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def write_and_verify_secret_key(ws: Any, scope: str, key: str, value: str) -> None:
|
|
171
|
-
"""Write *value* to *(scope, key)* and confirm it reads back identically.
|
|
172
|
-
|
|
173
|
-
The read-back is the gate that later permits deleting the Lakebase row: no
|
|
174
|
-
key material is removed from Lakebase until the secret has been proven
|
|
175
|
-
present and correct.
|
|
176
|
-
"""
|
|
177
|
-
ws.secrets.put_secret(scope=scope, key=key, string_value=value)
|
|
178
|
-
stored = read_secret_key(ws, scope, key)
|
|
179
|
-
if stored != value:
|
|
180
|
-
raise SecretKeyError(
|
|
181
|
-
f"Secret {scope}/{key} failed read-back verification after write. "
|
|
182
|
-
f"The Lakebase key row has NOT been touched. Resolve the secret "
|
|
183
|
-
f"store problem and re-run."
|
|
184
|
-
)
|
|
185
|
-
logger.info("Secret %s/%s written and verified", scope, key)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
def build_secret_resource(scope: str, key: str) -> AppResource:
|
|
189
|
-
"""Build the Apps secret resource that injects the key.
|
|
190
|
-
|
|
191
|
-
No ``put_acl`` is needed anywhere: attaching this resource auto-grants the
|
|
192
|
-
app's service principal READ on the scope (verified live). Adding one would
|
|
193
|
-
require MANAGE on the scope, which is what made fork creation unworkable.
|
|
194
|
-
"""
|
|
195
|
-
return AppResource(
|
|
196
|
-
name=RESOURCE_KEY,
|
|
197
|
-
secret=AppResourceSecret(
|
|
198
|
-
scope=scope, key=key,
|
|
199
|
-
permission=AppResourceSecretSecretPermission.READ,
|
|
200
|
-
),
|
|
201
|
-
)
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
def resolve_key_for_create(ws: Any, scope: str, key: str) -> str:
|
|
205
|
-
"""Return the key a fresh install should use, writing one if none exists."""
|
|
206
|
-
existing = read_secret_key(ws, scope, key)
|
|
207
|
-
if existing:
|
|
208
|
-
logger.info("Reusing the existing Fernet key at %s/%s", scope, key)
|
|
209
|
-
return existing
|
|
210
|
-
generated = Fernet.generate_key().decode()
|
|
211
|
-
write_and_verify_secret_key(ws, scope, key, generated)
|
|
212
|
-
return generated
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def read_lakebase_key(cur: Any, schema_name: str) -> str | None:
|
|
216
|
-
"""Read the existing Fernet key row, failing closed on anything but absence.
|
|
217
|
-
|
|
218
|
-
Only a genuinely missing table or row yields None. A permission error must
|
|
219
|
-
abort: on installs where the app SP owns the table, a deployer outside
|
|
220
|
-
``databricks_superuser`` gets denied, and treating that as "no row" would walk
|
|
221
|
-
the ladder to case 4 and mint a fresh key over live ciphertext.
|
|
222
|
-
"""
|
|
223
|
-
try:
|
|
224
|
-
cur.execute(
|
|
225
|
-
f'SELECT key_value FROM "{schema_name}".encryption_keys WHERE id = 1'
|
|
226
|
-
)
|
|
227
|
-
except Exception as exc: # noqa: BLE001
|
|
228
|
-
text = str(exc).lower()
|
|
229
|
-
if "does not exist" in text and "relation" in text:
|
|
230
|
-
return None
|
|
231
|
-
raise SecretKeyError(
|
|
232
|
-
f"Could not read the existing key from {schema_name}.encryption_keys: "
|
|
233
|
-
f"{exc}. This is not a missing-table error, so it is most likely a "
|
|
234
|
-
f"permission problem. Refusing to continue — proceeding could generate "
|
|
235
|
-
f"a fresh key and orphan every stored Google credential."
|
|
236
|
-
) from exc
|
|
237
|
-
row = cur.fetchone()
|
|
238
|
-
return row[0] if row and row[0] else None
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
def resolve_key_for_update(
|
|
242
|
-
ws: Any,
|
|
243
|
-
scope: str,
|
|
244
|
-
key: str,
|
|
245
|
-
lakebase_key: str | None,
|
|
246
|
-
app_yaml_key: str | None,
|
|
247
|
-
) -> tuple[str, bool]:
|
|
248
|
-
"""Decide which key secret mode should hold. Returns (value, wrote_secret).
|
|
249
|
-
|
|
250
|
-
First hit wins:
|
|
251
|
-
1. A valid key already in the secret — authoritative, and hard-fails if it
|
|
252
|
-
disagrees with an existing Lakebase row (mirrors the guard in
|
|
253
|
-
``_migrate_encryption_key_to_lakebase``).
|
|
254
|
-
2. The existing Lakebase row — the relocate case.
|
|
255
|
-
3. A legacy ``GOOGLE_OAUTH_ENCRYPTION_KEY`` still in the deployed app.yaml.
|
|
256
|
-
4. Nothing anywhere — generate fresh.
|
|
257
|
-
"""
|
|
258
|
-
existing = read_secret_key(ws, scope, key)
|
|
259
|
-
if existing:
|
|
260
|
-
if lakebase_key and lakebase_key != existing:
|
|
261
|
-
raise SecretKeyError(
|
|
262
|
-
f"Secret {scope}/{key} holds a different key from "
|
|
263
|
-
f"encryption_keys. Refusing to continue: attaching the secret and "
|
|
264
|
-
f"deleting the row would orphan every credential encrypted under "
|
|
265
|
-
f"the other key. Resolve which key is correct, then re-run."
|
|
266
|
-
)
|
|
267
|
-
logger.info("Secret %s/%s already holds the key — reusing", scope, key)
|
|
268
|
-
return existing, False
|
|
269
|
-
|
|
270
|
-
value = lakebase_key or app_yaml_key or Fernet.generate_key().decode()
|
|
271
|
-
write_and_verify_secret_key(ws, scope, key, value)
|
|
272
|
-
return value, True
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
def app_reports_secret_source(
|
|
276
|
-
ws: Any, app_url: str, attempts: int = 10, delay: float = 6.0
|
|
277
|
-
) -> bool:
|
|
278
|
-
"""Return True only if the live app reports it is reading from the secret.
|
|
279
|
-
|
|
280
|
-
This is the gate on deleting the Lakebase key row. It **fails closed**: a
|
|
281
|
-
missing field, an unparseable body, a non-200, or a network error all return
|
|
282
|
-
False, because every one of those is indistinguishable from "the deployed code
|
|
283
|
-
does not know about secret mode". An app pinned to a pre-feature version has
|
|
284
|
-
no ``encryption_key_source`` field at all, which is exactly the case that must
|
|
285
|
-
not delete the row.
|
|
286
|
-
|
|
287
|
-
Databricks Apps sit behind the workspace proxy, so the request carries
|
|
288
|
-
workspace credentials from ``ws.config.authenticate()``.
|
|
289
|
-
"""
|
|
290
|
-
url = f"{app_url.rstrip('/')}/api/health"
|
|
291
|
-
for attempt in range(1, attempts + 1):
|
|
292
|
-
try:
|
|
293
|
-
resp = requests.get(url, headers=ws.config.authenticate(), timeout=15)
|
|
294
|
-
if resp.status_code == 200:
|
|
295
|
-
source = resp.json().get("encryption_key_source")
|
|
296
|
-
if source == "secret":
|
|
297
|
-
return True
|
|
298
|
-
logger.info(
|
|
299
|
-
"Health gate attempt %s/%s: key source is %r, not 'secret'",
|
|
300
|
-
attempt, attempts, source,
|
|
301
|
-
)
|
|
302
|
-
else:
|
|
303
|
-
logger.info(
|
|
304
|
-
"Health gate attempt %s/%s: HTTP %s", attempt, attempts,
|
|
305
|
-
resp.status_code,
|
|
306
|
-
)
|
|
307
|
-
except Exception as exc: # noqa: BLE001 — any failure means "not confirmed"
|
|
308
|
-
logger.info("Health gate attempt %s/%s failed: %s", attempt, attempts, exc)
|
|
309
|
-
if attempt < attempts and delay:
|
|
310
|
-
time.sleep(delay)
|
|
311
|
-
return False
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
def attach_secret_resource(ws: Any, app_name: str, scope: str, key: str) -> None:
|
|
315
|
-
"""Add or replace the secret resource on an existing app.
|
|
316
|
-
|
|
317
|
-
A GET-fetched ``App`` cannot be passed back: ``apps.update`` rejects it with
|
|
318
|
-
"Compute size updates are not supported in this update API" (verified). It is
|
|
319
|
-
also a full replace on the fields it does accept — omitting ``description``
|
|
320
|
-
blanks it and omitting ``user_api_scopes`` nulls it. So build a fresh App,
|
|
321
|
-
carry every mutable field explicitly, and omit ``compute_size``.
|
|
322
|
-
"""
|
|
323
|
-
cur = ws.apps.get(name=app_name)
|
|
324
|
-
# Filter by name only: an app may legitimately carry other secret
|
|
325
|
-
# resources, and dropping those would silently break it.
|
|
326
|
-
kept = [r for r in (cur.resources or []) if r.name != RESOURCE_KEY]
|
|
327
|
-
kept.append(build_secret_resource(scope, key))
|
|
328
|
-
ws.apps.update(
|
|
329
|
-
name=app_name,
|
|
330
|
-
app=App(
|
|
331
|
-
name=app_name,
|
|
332
|
-
description=cur.description,
|
|
333
|
-
default_source_code_path=cur.default_source_code_path,
|
|
334
|
-
user_api_scopes=cur.user_api_scopes,
|
|
335
|
-
resources=kept,
|
|
336
|
-
# compute_size deliberately omitted — passing it is rejected.
|
|
337
|
-
),
|
|
338
|
-
)
|
|
339
|
-
logger.info("Attached %s resource to app %s", RESOURCE_KEY, app_name)
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
def delete_lakebase_key_row(cur: Any, schema_name: str) -> None:
|
|
343
|
-
"""Remove the relocated key row. Only ever called behind the health gate."""
|
|
344
|
-
cur.execute(f'DELETE FROM "{schema_name}".encryption_keys WHERE id = 1')
|
|
345
|
-
logger.info("Deleted the relocated key row from %s.encryption_keys", schema_name)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
def app_is_secret_mode(ws: Any, app_name: str) -> bool:
|
|
349
|
-
"""True when the app already carries the encryption-key secret resource."""
|
|
350
|
-
try:
|
|
351
|
-
app = ws.apps.get(name=app_name)
|
|
352
|
-
except Exception: # noqa: BLE001 — absence is not secret mode
|
|
353
|
-
return False
|
|
354
|
-
return any(r.name == RESOURCE_KEY for r in (app.resources or []))
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{databricks_tellr-0.4.3.dev18 → databricks_tellr-0.4.3.dev20}/databricks_tellr/identifiers.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|