dataplat 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. dataplat/__init__.py +3 -0
  2. dataplat/cli/__init__.py +1 -0
  3. dataplat/cli/_missing.py +108 -0
  4. dataplat/cli/bi/__init__.py +1 -0
  5. dataplat/cli/bi/app.py +14 -0
  6. dataplat/cli/bi/superset.py +558 -0
  7. dataplat/cli/ci/__init__.py +1 -0
  8. dataplat/cli/ci/app.py +14 -0
  9. dataplat/cli/ci/github/__init__.py +1 -0
  10. dataplat/cli/ci/github/app.py +10 -0
  11. dataplat/cli/ci/github/runner.py +399 -0
  12. dataplat/cli/cloud/__init__.py +1 -0
  13. dataplat/cli/cloud/app.py +14 -0
  14. dataplat/cli/cloud/aws/__init__.py +1 -0
  15. dataplat/cli/cloud/aws/_common.py +77 -0
  16. dataplat/cli/cloud/aws/app.py +12 -0
  17. dataplat/cli/cloud/aws/rds.py +686 -0
  18. dataplat/cli/cloud/aws/redshift.py +312 -0
  19. dataplat/cli/cloud/aws/secrets.py +875 -0
  20. dataplat/cli/config.py +492 -0
  21. dataplat/cli/db/__init__.py +342 -0
  22. dataplat/cli/db/_common.py +130 -0
  23. dataplat/cli/db/_report.py +180 -0
  24. dataplat/cli/db/dbt_orphans.py +842 -0
  25. dataplat/cli/db/describe.py +939 -0
  26. dataplat/cli/db/long_queries.py +299 -0
  27. dataplat/cli/db/role.py +434 -0
  28. dataplat/cli/db/role_create.py +452 -0
  29. dataplat/cli/db/role_drop.py +290 -0
  30. dataplat/cli/db/role_list.py +147 -0
  31. dataplat/cli/db/top_tables.py +337 -0
  32. dataplat/cli/ingest/__init__.py +1 -0
  33. dataplat/cli/ingest/airbyte/__init__.py +5 -0
  34. dataplat/cli/ingest/airbyte/_common.py +36 -0
  35. dataplat/cli/ingest/airbyte/_cursor.py +268 -0
  36. dataplat/cli/ingest/airbyte/_resource.py +192 -0
  37. dataplat/cli/ingest/airbyte/app.py +31 -0
  38. dataplat/cli/ingest/airbyte/connections.py +1234 -0
  39. dataplat/cli/ingest/airbyte/definitions.py +122 -0
  40. dataplat/cli/ingest/airbyte/destinations.py +26 -0
  41. dataplat/cli/ingest/airbyte/enums.py +45 -0
  42. dataplat/cli/ingest/airbyte/jobs.py +112 -0
  43. dataplat/cli/ingest/airbyte/sources.py +26 -0
  44. dataplat/cli/ingest/airbyte/tags.py +57 -0
  45. dataplat/cli/ingest/airbyte/templates.py +144 -0
  46. dataplat/cli/ingest/airbyte/tui.py +123 -0
  47. dataplat/cli/ingest/airbyte/workspaces.py +80 -0
  48. dataplat/cli/ingest/app.py +14 -0
  49. dataplat/cli/open.py +117 -0
  50. dataplat/cli/status.py +308 -0
  51. dataplat/core/__init__.py +1 -0
  52. dataplat/core/deps.py +138 -0
  53. dataplat/core/envrc.py +125 -0
  54. dataplat/core/errors.py +23 -0
  55. dataplat/main.py +87 -0
  56. dataplat/services/__init__.py +1 -0
  57. dataplat/services/airbyte/__init__.py +1 -0
  58. dataplat/services/airbyte/_resource.py +113 -0
  59. dataplat/services/airbyte/client.py +280 -0
  60. dataplat/services/airbyte/connections.py +306 -0
  61. dataplat/services/airbyte/definitions.py +64 -0
  62. dataplat/services/airbyte/destinations.py +68 -0
  63. dataplat/services/airbyte/jobs.py +72 -0
  64. dataplat/services/airbyte/sources.py +58 -0
  65. dataplat/services/airbyte/tags.py +120 -0
  66. dataplat/services/airbyte/workspaces.py +47 -0
  67. dataplat/services/aws/__init__.py +1 -0
  68. dataplat/services/aws/auth.py +80 -0
  69. dataplat/services/db/__init__.py +1 -0
  70. dataplat/services/db/connection.py +169 -0
  71. dataplat/services/db/describe.py +1034 -0
  72. dataplat/services/db/long_queries.py +378 -0
  73. dataplat/services/db/orphans.py +429 -0
  74. dataplat/services/db/role.py +812 -0
  75. dataplat/services/db/role_admin.py +458 -0
  76. dataplat/services/db/role_dialects.py +521 -0
  77. dataplat/services/db/targets.py +128 -0
  78. dataplat/services/db/top_tables.py +193 -0
  79. dataplat/services/superset/__init__.py +1 -0
  80. dataplat/services/superset/client.py +319 -0
  81. dataplat-0.1.0.dist-info/METADATA +326 -0
  82. dataplat-0.1.0.dist-info/RECORD +85 -0
  83. dataplat-0.1.0.dist-info/WHEEL +4 -0
  84. dataplat-0.1.0.dist-info/entry_points.txt +2 -0
  85. dataplat-0.1.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,842 @@
1
+ """`dp db dbt-orphans` — discover and rename orphan dbt tables."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import glob
6
+ import json
7
+ import os
8
+ from datetime import UTC, datetime, timedelta
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ import typer
13
+ from rich.console import Console
14
+
15
+ from dataplat.core.errors import ConfigError, ServiceError, ValidationError
16
+ from dataplat.services.db.connection import SqlEngine
17
+ from dataplat.services.db.orphans import (
18
+ DEPRECATED_SUFFIX,
19
+ LIVE_STATUSES,
20
+ DropEntry,
21
+ ObjectKind,
22
+ RenameEntry,
23
+ classify_object,
24
+ diff_orphans,
25
+ drop_object,
26
+ excluded_schemas,
27
+ fetch_deprecated_objects,
28
+ fetch_existing_relations,
29
+ fetch_live_model_relations,
30
+ invocation_command,
31
+ node_prefix,
32
+ open_transactional_connection,
33
+ rename_object,
34
+ resolve_orphans_connection_params,
35
+ )
36
+ from dataplat.services.db.targets import resolve_targets
37
+
38
+ DEFAULT_WINDOW_DAYS = 7
39
+
40
+ app = typer.Typer(
41
+ name="dbt-orphans",
42
+ help=(
43
+ "Discover orphan dbt tables (objects in the warehouse that the "
44
+ "latest dbt build no longer produces) and rename them with a "
45
+ "_deprecated suffix."
46
+ ),
47
+ invoke_without_command=True,
48
+ no_args_is_help=False,
49
+ )
50
+
51
+ console = Console()
52
+
53
+ LOG_DIR = Path.home() / ".config" / "dataplat" / "logs" / "dbt-orphans"
54
+ # Older releases wrote logs into ./local — keep reading them for revert/purge.
55
+ LEGACY_LOG_DIR = Path("local")
56
+ APPLY_LOG_PREFIX = "dbt_orphans"
57
+ PURGE_LOG_PREFIX = "dbt_orphans_purge"
58
+
59
+ # Log entries keep the historical engine labels so old logs stay revertable.
60
+ _ENGINE_LABELS: dict[SqlEngine, str] = {
61
+ SqlEngine.postgresql: "postgres",
62
+ SqlEngine.redshift: "redshift",
63
+ }
64
+
65
+ TargetOption = typer.Option(
66
+ "all",
67
+ "--target",
68
+ "-t",
69
+ help="Named DB target from DP_TARGETS, or all.",
70
+ )
71
+ YesOption = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt.")
72
+
73
+
74
+ def _timestamped_log_path(prefix: str) -> str:
75
+ """Build a unique-per-run log path like ``<log dir>/<prefix>-<UTC ISO>.log.json``."""
76
+ LOG_DIR.mkdir(parents=True, exist_ok=True)
77
+ stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
78
+ return str(LOG_DIR / f"{prefix}-{stamp}.log.json")
79
+
80
+
81
+ def _matching_logs(prefix: str) -> list[str]:
82
+ """All timestamped logs for ``prefix``, oldest first (by timestamped name)."""
83
+ matches = glob.glob(str(LOG_DIR / f"{prefix}-*.log.json"))
84
+ matches += glob.glob(str(LEGACY_LOG_DIR / f"{prefix}-*.log.json"))
85
+ return sorted(matches, key=os.path.basename)
86
+
87
+
88
+ def _find_latest_log(prefix: str) -> str | None:
89
+ """Return the newest timestamped log for ``prefix`` or ``None`` if absent."""
90
+ matches = _matching_logs(prefix)
91
+ return matches[-1] if matches else None
92
+
93
+
94
+ def _engines_for_target(name: str) -> list[tuple[str, SqlEngine, str]]:
95
+ """Return ``(label, engine, env_prefix)`` per target; label is the log key."""
96
+ try:
97
+ targets = resolve_targets(name)
98
+ except ValidationError as exc:
99
+ console.print(f"[red]Error: {exc}[/red]")
100
+ raise typer.Exit(code=1)
101
+ return [(_ENGINE_LABELS[t.engine], t.engine, t.env_prefix) for t in targets]
102
+
103
+
104
+ def _confirm_or_abort(prompt: str, yes: bool) -> None:
105
+ if yes:
106
+ return
107
+ if not typer.confirm(prompt, default=False):
108
+ console.print("[yellow]Aborted.[/yellow]")
109
+ raise typer.Exit(code=1)
110
+
111
+
112
+ def _parse_exclusions(
113
+ tokens: list[str], file_path: str | None
114
+ ) -> tuple[frozenset[str], frozenset[tuple[str, str]]]:
115
+ """Parse --exclude flags and an optional --exclude-file into two sets.
116
+
117
+ Returns ``(excluded_schemas, excluded_relations)``. A token with no dot
118
+ excludes a whole schema; a token with exactly one dot excludes a single
119
+ ``schema.name``. Tokens are whitespace-trimmed and matched case-sensitively
120
+ against warehouse objects. Each ``--exclude`` argument may contain one or
121
+ more tokens separated by commas (e.g. ``--exclude public,analytics``).
122
+ Blank lines and ``#`` comments in the file are ignored.
123
+ """
124
+ raw_tokens: list[str] = list(tokens)
125
+
126
+ if file_path is not None:
127
+ if not os.path.exists(file_path):
128
+ raise ValidationError(f"--exclude-file not found: {file_path}")
129
+ with open(file_path) as f:
130
+ for line in f:
131
+ line = line.strip()
132
+ if not line or line.startswith("#"):
133
+ continue
134
+ raw_tokens.append(line)
135
+
136
+ schemas: set[str] = set()
137
+ relations: set[tuple[str, str]] = set()
138
+ for raw in raw_tokens:
139
+ for piece in raw.split(","):
140
+ token = piece.strip()
141
+ if not token:
142
+ raise ValidationError(
143
+ f"Empty exclusion token in {raw!r}"
144
+ )
145
+ parts = token.split(".")
146
+ if len(parts) == 1:
147
+ schemas.add(parts[0])
148
+ elif len(parts) == 2:
149
+ relations.add((parts[0], parts[1]))
150
+ else:
151
+ raise ValidationError(
152
+ f"Invalid exclusion {token!r}: expected 'schema' or "
153
+ f"'schema.name'"
154
+ )
155
+ return frozenset(schemas), frozenset(relations)
156
+
157
+
158
+ @app.callback(invoke_without_command=True)
159
+ def main(
160
+ ctx: typer.Context,
161
+ log: str | None = typer.Option(
162
+ None,
163
+ "--log",
164
+ help=(
165
+ "Audit log output path. Defaults to "
166
+ "~/.config/dataplat/logs/dbt-orphans/"
167
+ "dbt_orphans-<UTC timestamp>.log.json (unique per run)."
168
+ ),
169
+ ),
170
+ target: str = TargetOption,
171
+ dry_run: bool = typer.Option(
172
+ True,
173
+ "--dry-run/--no-dry-run",
174
+ help="Preview without changes (default). Pass --no-dry-run to apply.",
175
+ ),
176
+ yes: bool = YesOption,
177
+ exclude: list[str] = typer.Option(
178
+ [],
179
+ "--exclude",
180
+ help="Schema or schema.name to skip (repeatable).",
181
+ ),
182
+ exclude_file: str | None = typer.Option(
183
+ None,
184
+ "--exclude-file",
185
+ help="Path to a file with one exclusion token per line.",
186
+ ),
187
+ window_days: int = typer.Option(
188
+ DEFAULT_WINDOW_DAYS,
189
+ "--window-days",
190
+ help=(
191
+ "Consider a model 'live' if any matching dbt build in the last "
192
+ "N days produced it. Larger windows are more conservative "
193
+ "(fewer false-positive renames)."
194
+ ),
195
+ ),
196
+ ) -> None:
197
+ """Discover orphan dbt tables and rename them with _deprecated."""
198
+ if ctx.invoked_subcommand is not None:
199
+ return
200
+
201
+ if log is None:
202
+ log = _timestamped_log_path(APPLY_LOG_PREFIX)
203
+
204
+ if window_days < 1:
205
+ console.print("[red]Error: --window-days must be >= 1[/red]")
206
+ raise typer.Exit(code=1)
207
+
208
+ try:
209
+ excluded_user_schemas, excluded_user_relations = _parse_exclusions(
210
+ exclude, exclude_file
211
+ )
212
+ except ValidationError as exc:
213
+ console.print(f"[red]Error: {exc}[/red]")
214
+ raise typer.Exit(code=1)
215
+
216
+ engines = _engines_for_target(target)
217
+ if not dry_run:
218
+ _confirm_or_abort(
219
+ "Rename every orphaned dbt object with a _deprecated suffix?", yes
220
+ )
221
+
222
+ since = datetime.now(UTC) - timedelta(days=window_days)
223
+
224
+ all_entries: list[RenameEntry] = []
225
+ try:
226
+ for label, engine, env_prefix in engines:
227
+ all_entries.extend(
228
+ _run_for_engine(
229
+ label,
230
+ engine,
231
+ env_prefix=env_prefix,
232
+ excluded_user_schemas=excluded_user_schemas,
233
+ excluded_user_relations=excluded_user_relations,
234
+ window_days=window_days,
235
+ since=since,
236
+ dry_run=dry_run,
237
+ )
238
+ )
239
+ except ServiceError as exc:
240
+ _write_audit_log(log, all_entries, dry_run=dry_run)
241
+ console.print(f"[red]{exc}[/red]")
242
+ console.print(
243
+ f"[yellow]Partial audit log written to {log} "
244
+ f"({len(all_entries)} entries).[/yellow]"
245
+ )
246
+ raise typer.Exit(code=1)
247
+
248
+ _write_audit_log(log, all_entries, dry_run=dry_run)
249
+
250
+ prefix = "[DRY-RUN] " if dry_run else ""
251
+ console.print(
252
+ f"[green]{prefix}Processed {len(all_entries)} object(s). "
253
+ f"Log written to {log}.[/green]"
254
+ )
255
+ if dry_run and all_entries:
256
+ console.print(
257
+ "[dim]Re-run with --no-dry-run to apply these renames.[/dim]"
258
+ )
259
+
260
+
261
+ def _write_audit_log(
262
+ log_path: str, entries: list[RenameEntry], *, dry_run: bool
263
+ ) -> None:
264
+ payload = {
265
+ "generated_at": datetime.now(UTC).isoformat(),
266
+ "dry_run": dry_run,
267
+ "source": "dbt-orphans",
268
+ "renames": entries,
269
+ }
270
+ with open(log_path, "w") as f:
271
+ json.dump(payload, f, indent=4)
272
+
273
+
274
+ def _run_for_engine(
275
+ label: str,
276
+ engine: SqlEngine,
277
+ *,
278
+ env_prefix: str,
279
+ excluded_user_schemas: frozenset[str],
280
+ excluded_user_relations: frozenset[tuple[str, str]],
281
+ window_days: int,
282
+ since: datetime,
283
+ dry_run: bool,
284
+ ) -> list[RenameEntry]:
285
+ try:
286
+ params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
287
+ dbt_node_prefix = node_prefix()
288
+ except ConfigError as exc:
289
+ raise ServiceError(f"[{label}] {exc}") from exc
290
+ if params is None:
291
+ console.print(
292
+ f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
293
+ )
294
+ return []
295
+
296
+ is_redshift = engine is SqlEngine.redshift
297
+ entries: list[RenameEntry] = []
298
+
299
+ with (
300
+ open_transactional_connection(params, dry_run=dry_run) as conn,
301
+ conn.cursor() as cur,
302
+ ):
303
+ live = fetch_live_model_relations(
304
+ cur,
305
+ invocation_command=invocation_command(),
306
+ node_prefix=dbt_node_prefix,
307
+ statuses=LIVE_STATUSES,
308
+ since=since,
309
+ )
310
+ if not live:
311
+ console.print(
312
+ f"[yellow][{label}] No matching dbt builds in the last "
313
+ f"{window_days} day(s); skipping to avoid diffing "
314
+ f"against empty set.[/yellow]"
315
+ )
316
+ return []
317
+
318
+ excluded = excluded_schemas()
319
+ schemas_to_scan = sorted(s for s in live if s not in excluded)
320
+ existing = fetch_existing_relations(
321
+ cur, schemas_to_scan, is_redshift=is_redshift
322
+ )
323
+ orphans = diff_orphans(
324
+ live=live,
325
+ existing=existing,
326
+ excluded_schemas=excluded,
327
+ excluded_user_schemas=excluded_user_schemas,
328
+ excluded_user_relations=excluded_user_relations,
329
+ )
330
+
331
+ _print_summary(label, live, existing, orphans)
332
+
333
+ for schema, names in orphans.items():
334
+ for name in names:
335
+ entry = _rename_orphan(
336
+ cur,
337
+ label,
338
+ schema,
339
+ name,
340
+ is_redshift=is_redshift,
341
+ dry_run=dry_run,
342
+ )
343
+ if entry is not None:
344
+ entries.append(entry)
345
+
346
+ return entries
347
+
348
+
349
+ def _print_summary(
350
+ label: str,
351
+ live: dict[str, set[str]],
352
+ existing: dict[str, set[str]],
353
+ orphans: dict[str, list[str]],
354
+ ) -> None:
355
+ live_count = sum(len(v) for v in live.values())
356
+ existing_count = sum(len(v) for v in existing.values())
357
+ orphan_count = sum(len(v) for v in orphans.values())
358
+ per_schema = ", ".join(
359
+ f"{len(v)} in {s}" for s, v in sorted(orphans.items())
360
+ )
361
+ suffix = f" ({per_schema})" if per_schema else ""
362
+ console.print(
363
+ f"[cyan][{label}] {live_count} live dbt models; {existing_count} "
364
+ f"existing in live schemas; {orphan_count} orphans after "
365
+ f"exclusions{suffix}[/cyan]"
366
+ )
367
+
368
+
369
+ def _rename_orphan(
370
+ cur: Any,
371
+ label: str,
372
+ schema: str,
373
+ name: str,
374
+ *,
375
+ is_redshift: bool,
376
+ dry_run: bool,
377
+ ) -> RenameEntry | None:
378
+ kind = classify_object(cur, schema, name, is_redshift=is_redshift)
379
+ if kind is None:
380
+ console.print(
381
+ f"[yellow][{label}] {schema}.{name} no longer present, skipping.[/yellow]"
382
+ )
383
+ return None
384
+
385
+ new_name = f"{name}{DEPRECATED_SUFFIX}"
386
+ if classify_object(cur, schema, new_name, is_redshift=is_redshift) is not None:
387
+ console.print(
388
+ f"[yellow][{label}] {schema}.{new_name} already exists, "
389
+ f"skipping rename of {schema}.{name}.[/yellow]"
390
+ )
391
+ return None
392
+
393
+ action = "[DRY-RUN] Would rename" if dry_run else "Renaming"
394
+ console.print(
395
+ f"[blue][{label}] {action} {kind} "
396
+ f"{schema}.{name} -> {schema}.{new_name}[/blue]"
397
+ )
398
+
399
+ if not dry_run:
400
+ rename_object(cur, schema, name, new_name, kind, is_redshift=is_redshift)
401
+
402
+ return RenameEntry(
403
+ database=label,
404
+ schema=schema,
405
+ old_name=name,
406
+ new_name=new_name,
407
+ kind=kind,
408
+ )
409
+
410
+
411
+ @app.command("revert")
412
+ def revert_cmd(
413
+ log: str | None = typer.Option(
414
+ None,
415
+ "--log",
416
+ help=(
417
+ "Audit log input path. Defaults to the newest "
418
+ "dbt_orphans-*.log.json in the log directory."
419
+ ),
420
+ ),
421
+ target: str = TargetOption,
422
+ dry_run: bool = typer.Option(
423
+ True,
424
+ "--dry-run/--no-dry-run",
425
+ help="Preview without changes (default). Pass --no-dry-run to revert.",
426
+ ),
427
+ ) -> None:
428
+ """Undo a previous dbt-orphans run using the audit log."""
429
+ if log is None:
430
+ log = _find_latest_log(APPLY_LOG_PREFIX)
431
+ if log is None:
432
+ console.print(
433
+ f"[red]Error: no {APPLY_LOG_PREFIX} log found in "
434
+ f"{LOG_DIR}/[/red]"
435
+ )
436
+ console.print(
437
+ "[dim]Run `dp db dbt-orphans` first or pass --log explicitly.[/dim]"
438
+ )
439
+ raise typer.Exit(code=1)
440
+ console.print(f"[dim]Using latest log: {log}[/dim]")
441
+ if not os.path.exists(log):
442
+ console.print(f"[red]Error: log file not found: {log}[/red]")
443
+ console.print(
444
+ "[dim]Run `dp db dbt-orphans` first to generate it.[/dim]"
445
+ )
446
+ raise typer.Exit(code=1)
447
+
448
+ try:
449
+ with open(log) as f:
450
+ payload = json.load(f)
451
+ except (OSError, json.JSONDecodeError) as exc:
452
+ console.print(f"[red]Error: could not read log {log}: {exc}[/red]")
453
+ raise typer.Exit(code=1)
454
+ if not isinstance(payload, dict):
455
+ console.print(f"[red]Error: malformed log {log}: expected an object[/red]")
456
+ raise typer.Exit(code=1)
457
+
458
+ if payload.get("dry_run"):
459
+ console.print(
460
+ "[yellow]Log was generated in dry-run mode; "
461
+ "no renames were actually applied.[/yellow]"
462
+ )
463
+
464
+ renames = payload.get("renames") or []
465
+ if not renames:
466
+ console.print(
467
+ "[dim]No renames recorded in the log; nothing to revert.[/dim]"
468
+ )
469
+ return
470
+
471
+ total = 0
472
+ try:
473
+ for label, engine, env_prefix in _engines_for_target(target):
474
+ entries = [
475
+ r for r in renames
476
+ if isinstance(r, dict) and r.get("database") == label
477
+ ]
478
+ if not entries:
479
+ console.print(f"[dim][{label}] No entries in log.[/dim]")
480
+ continue
481
+ total += _revert_for_engine(
482
+ label, engine, entries, env_prefix=env_prefix, dry_run=dry_run
483
+ )
484
+ except ServiceError as exc:
485
+ console.print(f"[red]{exc}[/red]")
486
+ console.print(
487
+ f"[yellow]Reverted {total} object(s) before the failure.[/yellow]"
488
+ )
489
+ raise typer.Exit(code=1)
490
+
491
+ prefix = "[DRY-RUN] " if dry_run else ""
492
+ console.print(f"[green]{prefix}Reverted {total} object(s).[/green]")
493
+ if dry_run and total:
494
+ console.print(
495
+ "[dim]Re-run with --no-dry-run to revert these renames.[/dim]"
496
+ )
497
+
498
+
499
+ def _revert_for_engine(
500
+ label: str,
501
+ engine: SqlEngine,
502
+ entries: list[dict],
503
+ *,
504
+ env_prefix: str,
505
+ dry_run: bool,
506
+ ) -> int:
507
+ try:
508
+ params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
509
+ except ConfigError as exc:
510
+ raise ServiceError(f"[{label}] {exc}") from exc
511
+ if params is None:
512
+ console.print(
513
+ f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
514
+ )
515
+ return 0
516
+
517
+ is_redshift = engine is SqlEngine.redshift
518
+ reverted = 0
519
+
520
+ with (
521
+ open_transactional_connection(params, dry_run=dry_run) as conn,
522
+ conn.cursor() as cur,
523
+ ):
524
+ for entry in entries:
525
+ if _revert_one(
526
+ cur, label, entry, is_redshift=is_redshift, dry_run=dry_run
527
+ ):
528
+ reverted += 1
529
+
530
+ return reverted
531
+
532
+
533
+ def _revert_one(
534
+ cur: Any,
535
+ label: str,
536
+ entry: dict,
537
+ *,
538
+ is_redshift: bool,
539
+ dry_run: bool,
540
+ ) -> bool:
541
+ schema = entry.get("schema")
542
+ current_name = entry.get("new_name")
543
+ original_name = entry.get("old_name")
544
+ if not (schema and current_name and original_name):
545
+ console.print(
546
+ f"[yellow][{label}] Skipping malformed log entry: {entry!r}[/yellow]"
547
+ )
548
+ return False
549
+ kind: ObjectKind = entry.get("kind", "table")
550
+
551
+ if classify_object(cur, schema, current_name, is_redshift=is_redshift) is None:
552
+ console.print(
553
+ f"[dim][{label}] {schema}.{current_name} not found, "
554
+ f"skipping revert.[/dim]"
555
+ )
556
+ return False
557
+
558
+ if classify_object(cur, schema, original_name, is_redshift=is_redshift) is not None:
559
+ console.print(
560
+ f"[yellow][{label}] {schema}.{original_name} already exists, "
561
+ f"cannot revert {schema}.{current_name}.[/yellow]"
562
+ )
563
+ return False
564
+
565
+ action = "[DRY-RUN] Would revert" if dry_run else "Reverting"
566
+ console.print(
567
+ f"[blue][{label}] {action} {kind} "
568
+ f"{schema}.{current_name} -> {schema}.{original_name}[/blue]"
569
+ )
570
+
571
+ if not dry_run:
572
+ rename_object(
573
+ cur, schema, current_name, original_name, kind, is_redshift=is_redshift
574
+ )
575
+ return True
576
+
577
+
578
+ def _renamed_at_index() -> dict[tuple[str, str, str], datetime]:
579
+ """Map ``(database, schema, new_name)`` -> newest rename timestamp.
580
+
581
+ Built from the applied (non-dry-run) apply logs so purge can enforce a
582
+ grace period even though the warehouse doesn't track rename times.
583
+ """
584
+ index: dict[tuple[str, str, str], datetime] = {}
585
+ for path in _matching_logs(APPLY_LOG_PREFIX):
586
+ try:
587
+ with open(path) as f:
588
+ payload = json.load(f)
589
+ except (OSError, json.JSONDecodeError):
590
+ continue
591
+ if not isinstance(payload, dict) or payload.get("dry_run"):
592
+ continue
593
+ try:
594
+ when = datetime.fromisoformat(payload.get("generated_at", ""))
595
+ except ValueError:
596
+ continue
597
+ for r in payload.get("renames") or []:
598
+ if not isinstance(r, dict):
599
+ continue
600
+ database = r.get("database")
601
+ schema = r.get("schema")
602
+ new_name = r.get("new_name")
603
+ if not (database and schema and new_name):
604
+ continue
605
+ key = (str(database), str(schema), str(new_name))
606
+ if key not in index or when > index[key]:
607
+ index[key] = when
608
+ return index
609
+
610
+
611
+ @app.command("purge")
612
+ def purge_cmd(
613
+ log: str | None = typer.Option(
614
+ None,
615
+ "--log",
616
+ help=(
617
+ "Purge audit log output path. Defaults to "
618
+ "~/.config/dataplat/logs/dbt-orphans/"
619
+ "dbt_orphans_purge-<UTC timestamp>.log.json (unique per run)."
620
+ ),
621
+ ),
622
+ target: str = TargetOption,
623
+ dry_run: bool = typer.Option(
624
+ True,
625
+ "--dry-run/--no-dry-run",
626
+ help="Preview without changes (default). Pass --no-dry-run to drop.",
627
+ ),
628
+ yes: bool = YesOption,
629
+ older_than: int | None = typer.Option(
630
+ None,
631
+ "--older-than",
632
+ min=0,
633
+ help=(
634
+ "Only drop objects renamed at least N days ago (per the audit "
635
+ "logs). Objects with no recorded rename are skipped unless "
636
+ "--include-unknown."
637
+ ),
638
+ ),
639
+ include_unknown: bool = typer.Option(
640
+ False,
641
+ "--include-unknown",
642
+ help="With --older-than: also drop objects that have no recorded rename.",
643
+ ),
644
+ exclude: list[str] = typer.Option(
645
+ [],
646
+ "--exclude",
647
+ help="Schema or schema.name to skip (repeatable).",
648
+ ),
649
+ exclude_file: str | None = typer.Option(
650
+ None,
651
+ "--exclude-file",
652
+ help="Path to a file with one exclusion token per line.",
653
+ ),
654
+ ) -> None:
655
+ """Permanently drop every object ending in _deprecated (irreversible)."""
656
+ if log is None:
657
+ log = _timestamped_log_path(PURGE_LOG_PREFIX)
658
+
659
+ try:
660
+ excluded_user_schemas, excluded_user_relations = _parse_exclusions(
661
+ exclude, exclude_file
662
+ )
663
+ except ValidationError as exc:
664
+ console.print(f"[red]Error: {exc}[/red]")
665
+ raise typer.Exit(code=1)
666
+
667
+ engines = _engines_for_target(target)
668
+ if not dry_run:
669
+ _confirm_or_abort(
670
+ "Permanently DROP every *_deprecated object? This cannot be undone.",
671
+ yes,
672
+ )
673
+
674
+ renamed_at = _renamed_at_index() if older_than is not None else None
675
+ cutoff = (
676
+ datetime.now(UTC) - timedelta(days=older_than)
677
+ if older_than is not None
678
+ else None
679
+ )
680
+
681
+ all_drops: list[DropEntry] = []
682
+ try:
683
+ for label, engine, env_prefix in engines:
684
+ all_drops.extend(
685
+ _purge_for_engine(
686
+ label,
687
+ engine,
688
+ env_prefix=env_prefix,
689
+ excluded_user_schemas=excluded_user_schemas,
690
+ excluded_user_relations=excluded_user_relations,
691
+ dry_run=dry_run,
692
+ renamed_at=renamed_at,
693
+ cutoff=cutoff,
694
+ include_unknown=include_unknown,
695
+ )
696
+ )
697
+ except ServiceError as exc:
698
+ _write_purge_log(log, all_drops, dry_run=dry_run)
699
+ console.print(f"[red]{exc}[/red]")
700
+ console.print(
701
+ f"[yellow]Partial purge log written to {log} "
702
+ f"({len(all_drops)} entries).[/yellow]"
703
+ )
704
+ raise typer.Exit(code=1)
705
+
706
+ _write_purge_log(log, all_drops, dry_run=dry_run)
707
+
708
+ prefix = "[DRY-RUN] " if dry_run else ""
709
+ console.print(
710
+ f"[green]{prefix}Dropped {len(all_drops)} object(s). "
711
+ f"Log written to {log}.[/green]"
712
+ )
713
+ if dry_run and all_drops:
714
+ console.print(
715
+ "[dim]Re-run with --no-dry-run to apply these drops "
716
+ "(irreversible).[/dim]"
717
+ )
718
+
719
+
720
+ def _purge_for_engine(
721
+ label: str,
722
+ engine: SqlEngine,
723
+ *,
724
+ env_prefix: str,
725
+ excluded_user_schemas: frozenset[str],
726
+ excluded_user_relations: frozenset[tuple[str, str]],
727
+ dry_run: bool,
728
+ renamed_at: dict[tuple[str, str, str], datetime] | None = None,
729
+ cutoff: datetime | None = None,
730
+ include_unknown: bool = False,
731
+ ) -> list[DropEntry]:
732
+ try:
733
+ params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
734
+ except ConfigError as exc:
735
+ raise ServiceError(f"[{label}] {exc}") from exc
736
+ if params is None:
737
+ console.print(
738
+ f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
739
+ )
740
+ return []
741
+
742
+ is_redshift = engine is SqlEngine.redshift
743
+ effective_excluded_schemas = excluded_schemas() | excluded_user_schemas
744
+ drops: list[DropEntry] = []
745
+
746
+ with (
747
+ open_transactional_connection(params, dry_run=dry_run) as conn,
748
+ conn.cursor() as cur,
749
+ ):
750
+ deprecated = fetch_deprecated_objects(
751
+ cur,
752
+ is_redshift=is_redshift,
753
+ excluded_schemas=effective_excluded_schemas,
754
+ )
755
+ deprecated = [
756
+ (s, n, k)
757
+ for s, n, k in deprecated
758
+ if (s, n) not in excluded_user_relations
759
+ ]
760
+
761
+ if renamed_at is not None and cutoff is not None:
762
+ deprecated = _apply_age_filter(
763
+ deprecated,
764
+ label=label,
765
+ renamed_at=renamed_at,
766
+ cutoff=cutoff,
767
+ include_unknown=include_unknown,
768
+ )
769
+
770
+ console.print(
771
+ f"[cyan][{label}] {len(deprecated)} deprecated "
772
+ f"object(s) after exclusions.[/cyan]"
773
+ )
774
+
775
+ for schema, name, kind in deprecated:
776
+ entry = _drop_one(
777
+ cur, label, schema, name, kind, dry_run=dry_run
778
+ )
779
+ if entry is not None:
780
+ drops.append(entry)
781
+
782
+ return drops
783
+
784
+
785
+ def _apply_age_filter(
786
+ deprecated: list[tuple[str, str, Any]],
787
+ *,
788
+ label: str,
789
+ renamed_at: dict[tuple[str, str, str], datetime],
790
+ cutoff: datetime,
791
+ include_unknown: bool,
792
+ ) -> list[tuple[str, str, Any]]:
793
+ """Keep only objects renamed before ``cutoff`` per the audit logs."""
794
+ kept: list[tuple[str, str, Any]] = []
795
+ for schema, name, kind in deprecated:
796
+ when = renamed_at.get((label, schema, name))
797
+ if when is None:
798
+ if include_unknown:
799
+ kept.append((schema, name, kind))
800
+ else:
801
+ console.print(
802
+ f"[yellow][{label}] {schema}.{name}: no recorded rename; "
803
+ f"skipping (pass --include-unknown to drop anyway).[/yellow]"
804
+ )
805
+ continue
806
+ if when <= cutoff:
807
+ kept.append((schema, name, kind))
808
+ else:
809
+ console.print(
810
+ f"[dim][{label}] {schema}.{name}: renamed "
811
+ f"{when.date()}, inside the grace period; skipping.[/dim]"
812
+ )
813
+ return kept
814
+
815
+
816
+ def _drop_one(
817
+ cur: Any,
818
+ label: str,
819
+ schema: str,
820
+ name: str,
821
+ kind: ObjectKind,
822
+ *,
823
+ dry_run: bool,
824
+ ) -> DropEntry | None:
825
+ action = "[DRY-RUN] Would drop" if dry_run else "Dropping"
826
+ console.print(f"[blue][{label}] {action} {kind} {schema}.{name}[/blue]")
827
+ if not dry_run:
828
+ drop_object(cur, schema, name, kind)
829
+ return DropEntry(database=label, schema=schema, name=name, kind=kind)
830
+
831
+
832
+ def _write_purge_log(
833
+ log_path: str, entries: list[DropEntry], *, dry_run: bool
834
+ ) -> None:
835
+ payload = {
836
+ "generated_at": datetime.now(UTC).isoformat(),
837
+ "dry_run": dry_run,
838
+ "source": "dbt-orphans-purge",
839
+ "drops": entries,
840
+ }
841
+ with open(log_path, "w") as f:
842
+ json.dump(payload, f, indent=4)