dataplat 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataplat/__init__.py +3 -0
- dataplat/cli/__init__.py +1 -0
- dataplat/cli/_missing.py +108 -0
- dataplat/cli/bi/__init__.py +1 -0
- dataplat/cli/bi/app.py +14 -0
- dataplat/cli/bi/superset.py +558 -0
- dataplat/cli/ci/__init__.py +1 -0
- dataplat/cli/ci/app.py +14 -0
- dataplat/cli/ci/github/__init__.py +1 -0
- dataplat/cli/ci/github/app.py +10 -0
- dataplat/cli/ci/github/runner.py +399 -0
- dataplat/cli/cloud/__init__.py +1 -0
- dataplat/cli/cloud/app.py +14 -0
- dataplat/cli/cloud/aws/__init__.py +1 -0
- dataplat/cli/cloud/aws/_common.py +77 -0
- dataplat/cli/cloud/aws/app.py +12 -0
- dataplat/cli/cloud/aws/rds.py +686 -0
- dataplat/cli/cloud/aws/redshift.py +312 -0
- dataplat/cli/cloud/aws/secrets.py +875 -0
- dataplat/cli/config.py +492 -0
- dataplat/cli/db/__init__.py +342 -0
- dataplat/cli/db/_common.py +130 -0
- dataplat/cli/db/_report.py +180 -0
- dataplat/cli/db/dbt_orphans.py +842 -0
- dataplat/cli/db/describe.py +939 -0
- dataplat/cli/db/long_queries.py +299 -0
- dataplat/cli/db/role.py +434 -0
- dataplat/cli/db/role_create.py +452 -0
- dataplat/cli/db/role_drop.py +290 -0
- dataplat/cli/db/role_list.py +147 -0
- dataplat/cli/db/top_tables.py +337 -0
- dataplat/cli/ingest/__init__.py +1 -0
- dataplat/cli/ingest/airbyte/__init__.py +5 -0
- dataplat/cli/ingest/airbyte/_common.py +36 -0
- dataplat/cli/ingest/airbyte/_cursor.py +268 -0
- dataplat/cli/ingest/airbyte/_resource.py +192 -0
- dataplat/cli/ingest/airbyte/app.py +31 -0
- dataplat/cli/ingest/airbyte/connections.py +1234 -0
- dataplat/cli/ingest/airbyte/definitions.py +122 -0
- dataplat/cli/ingest/airbyte/destinations.py +26 -0
- dataplat/cli/ingest/airbyte/enums.py +45 -0
- dataplat/cli/ingest/airbyte/jobs.py +112 -0
- dataplat/cli/ingest/airbyte/sources.py +26 -0
- dataplat/cli/ingest/airbyte/tags.py +57 -0
- dataplat/cli/ingest/airbyte/templates.py +144 -0
- dataplat/cli/ingest/airbyte/tui.py +123 -0
- dataplat/cli/ingest/airbyte/workspaces.py +80 -0
- dataplat/cli/ingest/app.py +14 -0
- dataplat/cli/open.py +117 -0
- dataplat/cli/status.py +308 -0
- dataplat/core/__init__.py +1 -0
- dataplat/core/deps.py +138 -0
- dataplat/core/envrc.py +125 -0
- dataplat/core/errors.py +23 -0
- dataplat/main.py +87 -0
- dataplat/services/__init__.py +1 -0
- dataplat/services/airbyte/__init__.py +1 -0
- dataplat/services/airbyte/_resource.py +113 -0
- dataplat/services/airbyte/client.py +280 -0
- dataplat/services/airbyte/connections.py +306 -0
- dataplat/services/airbyte/definitions.py +64 -0
- dataplat/services/airbyte/destinations.py +68 -0
- dataplat/services/airbyte/jobs.py +72 -0
- dataplat/services/airbyte/sources.py +58 -0
- dataplat/services/airbyte/tags.py +120 -0
- dataplat/services/airbyte/workspaces.py +47 -0
- dataplat/services/aws/__init__.py +1 -0
- dataplat/services/aws/auth.py +80 -0
- dataplat/services/db/__init__.py +1 -0
- dataplat/services/db/connection.py +169 -0
- dataplat/services/db/describe.py +1034 -0
- dataplat/services/db/long_queries.py +378 -0
- dataplat/services/db/orphans.py +429 -0
- dataplat/services/db/role.py +812 -0
- dataplat/services/db/role_admin.py +458 -0
- dataplat/services/db/role_dialects.py +521 -0
- dataplat/services/db/targets.py +128 -0
- dataplat/services/db/top_tables.py +193 -0
- dataplat/services/superset/__init__.py +1 -0
- dataplat/services/superset/client.py +319 -0
- dataplat-0.1.0.dist-info/METADATA +326 -0
- dataplat-0.1.0.dist-info/RECORD +85 -0
- dataplat-0.1.0.dist-info/WHEEL +4 -0
- dataplat-0.1.0.dist-info/entry_points.txt +2 -0
- dataplat-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,842 @@
|
|
|
1
|
+
"""`dp db dbt-orphans` — discover and rename orphan dbt tables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import glob
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
from datetime import UTC, datetime, timedelta
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
import typer
|
|
13
|
+
from rich.console import Console
|
|
14
|
+
|
|
15
|
+
from dataplat.core.errors import ConfigError, ServiceError, ValidationError
|
|
16
|
+
from dataplat.services.db.connection import SqlEngine
|
|
17
|
+
from dataplat.services.db.orphans import (
|
|
18
|
+
DEPRECATED_SUFFIX,
|
|
19
|
+
LIVE_STATUSES,
|
|
20
|
+
DropEntry,
|
|
21
|
+
ObjectKind,
|
|
22
|
+
RenameEntry,
|
|
23
|
+
classify_object,
|
|
24
|
+
diff_orphans,
|
|
25
|
+
drop_object,
|
|
26
|
+
excluded_schemas,
|
|
27
|
+
fetch_deprecated_objects,
|
|
28
|
+
fetch_existing_relations,
|
|
29
|
+
fetch_live_model_relations,
|
|
30
|
+
invocation_command,
|
|
31
|
+
node_prefix,
|
|
32
|
+
open_transactional_connection,
|
|
33
|
+
rename_object,
|
|
34
|
+
resolve_orphans_connection_params,
|
|
35
|
+
)
|
|
36
|
+
from dataplat.services.db.targets import resolve_targets
|
|
37
|
+
|
|
38
|
+
DEFAULT_WINDOW_DAYS = 7
|
|
39
|
+
|
|
40
|
+
app = typer.Typer(
|
|
41
|
+
name="dbt-orphans",
|
|
42
|
+
help=(
|
|
43
|
+
"Discover orphan dbt tables (objects in the warehouse that the "
|
|
44
|
+
"latest dbt build no longer produces) and rename them with a "
|
|
45
|
+
"_deprecated suffix."
|
|
46
|
+
),
|
|
47
|
+
invoke_without_command=True,
|
|
48
|
+
no_args_is_help=False,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
console = Console()
|
|
52
|
+
|
|
53
|
+
LOG_DIR = Path.home() / ".config" / "dataplat" / "logs" / "dbt-orphans"
|
|
54
|
+
# Older releases wrote logs into ./local — keep reading them for revert/purge.
|
|
55
|
+
LEGACY_LOG_DIR = Path("local")
|
|
56
|
+
APPLY_LOG_PREFIX = "dbt_orphans"
|
|
57
|
+
PURGE_LOG_PREFIX = "dbt_orphans_purge"
|
|
58
|
+
|
|
59
|
+
# Log entries keep the historical engine labels so old logs stay revertable.
|
|
60
|
+
_ENGINE_LABELS: dict[SqlEngine, str] = {
|
|
61
|
+
SqlEngine.postgresql: "postgres",
|
|
62
|
+
SqlEngine.redshift: "redshift",
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
TargetOption = typer.Option(
|
|
66
|
+
"all",
|
|
67
|
+
"--target",
|
|
68
|
+
"-t",
|
|
69
|
+
help="Named DB target from DP_TARGETS, or all.",
|
|
70
|
+
)
|
|
71
|
+
YesOption = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt.")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _timestamped_log_path(prefix: str) -> str:
|
|
75
|
+
"""Build a unique-per-run log path like ``<log dir>/<prefix>-<UTC ISO>.log.json``."""
|
|
76
|
+
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
|
|
78
|
+
return str(LOG_DIR / f"{prefix}-{stamp}.log.json")
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _matching_logs(prefix: str) -> list[str]:
|
|
82
|
+
"""All timestamped logs for ``prefix``, oldest first (by timestamped name)."""
|
|
83
|
+
matches = glob.glob(str(LOG_DIR / f"{prefix}-*.log.json"))
|
|
84
|
+
matches += glob.glob(str(LEGACY_LOG_DIR / f"{prefix}-*.log.json"))
|
|
85
|
+
return sorted(matches, key=os.path.basename)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _find_latest_log(prefix: str) -> str | None:
|
|
89
|
+
"""Return the newest timestamped log for ``prefix`` or ``None`` if absent."""
|
|
90
|
+
matches = _matching_logs(prefix)
|
|
91
|
+
return matches[-1] if matches else None
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _engines_for_target(name: str) -> list[tuple[str, SqlEngine, str]]:
|
|
95
|
+
"""Return ``(label, engine, env_prefix)`` per target; label is the log key."""
|
|
96
|
+
try:
|
|
97
|
+
targets = resolve_targets(name)
|
|
98
|
+
except ValidationError as exc:
|
|
99
|
+
console.print(f"[red]Error: {exc}[/red]")
|
|
100
|
+
raise typer.Exit(code=1)
|
|
101
|
+
return [(_ENGINE_LABELS[t.engine], t.engine, t.env_prefix) for t in targets]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _confirm_or_abort(prompt: str, yes: bool) -> None:
|
|
105
|
+
if yes:
|
|
106
|
+
return
|
|
107
|
+
if not typer.confirm(prompt, default=False):
|
|
108
|
+
console.print("[yellow]Aborted.[/yellow]")
|
|
109
|
+
raise typer.Exit(code=1)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _parse_exclusions(
|
|
113
|
+
tokens: list[str], file_path: str | None
|
|
114
|
+
) -> tuple[frozenset[str], frozenset[tuple[str, str]]]:
|
|
115
|
+
"""Parse --exclude flags and an optional --exclude-file into two sets.
|
|
116
|
+
|
|
117
|
+
Returns ``(excluded_schemas, excluded_relations)``. A token with no dot
|
|
118
|
+
excludes a whole schema; a token with exactly one dot excludes a single
|
|
119
|
+
``schema.name``. Tokens are whitespace-trimmed and matched case-sensitively
|
|
120
|
+
against warehouse objects. Each ``--exclude`` argument may contain one or
|
|
121
|
+
more tokens separated by commas (e.g. ``--exclude public,analytics``).
|
|
122
|
+
Blank lines and ``#`` comments in the file are ignored.
|
|
123
|
+
"""
|
|
124
|
+
raw_tokens: list[str] = list(tokens)
|
|
125
|
+
|
|
126
|
+
if file_path is not None:
|
|
127
|
+
if not os.path.exists(file_path):
|
|
128
|
+
raise ValidationError(f"--exclude-file not found: {file_path}")
|
|
129
|
+
with open(file_path) as f:
|
|
130
|
+
for line in f:
|
|
131
|
+
line = line.strip()
|
|
132
|
+
if not line or line.startswith("#"):
|
|
133
|
+
continue
|
|
134
|
+
raw_tokens.append(line)
|
|
135
|
+
|
|
136
|
+
schemas: set[str] = set()
|
|
137
|
+
relations: set[tuple[str, str]] = set()
|
|
138
|
+
for raw in raw_tokens:
|
|
139
|
+
for piece in raw.split(","):
|
|
140
|
+
token = piece.strip()
|
|
141
|
+
if not token:
|
|
142
|
+
raise ValidationError(
|
|
143
|
+
f"Empty exclusion token in {raw!r}"
|
|
144
|
+
)
|
|
145
|
+
parts = token.split(".")
|
|
146
|
+
if len(parts) == 1:
|
|
147
|
+
schemas.add(parts[0])
|
|
148
|
+
elif len(parts) == 2:
|
|
149
|
+
relations.add((parts[0], parts[1]))
|
|
150
|
+
else:
|
|
151
|
+
raise ValidationError(
|
|
152
|
+
f"Invalid exclusion {token!r}: expected 'schema' or "
|
|
153
|
+
f"'schema.name'"
|
|
154
|
+
)
|
|
155
|
+
return frozenset(schemas), frozenset(relations)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
@app.callback(invoke_without_command=True)
|
|
159
|
+
def main(
|
|
160
|
+
ctx: typer.Context,
|
|
161
|
+
log: str | None = typer.Option(
|
|
162
|
+
None,
|
|
163
|
+
"--log",
|
|
164
|
+
help=(
|
|
165
|
+
"Audit log output path. Defaults to "
|
|
166
|
+
"~/.config/dataplat/logs/dbt-orphans/"
|
|
167
|
+
"dbt_orphans-<UTC timestamp>.log.json (unique per run)."
|
|
168
|
+
),
|
|
169
|
+
),
|
|
170
|
+
target: str = TargetOption,
|
|
171
|
+
dry_run: bool = typer.Option(
|
|
172
|
+
True,
|
|
173
|
+
"--dry-run/--no-dry-run",
|
|
174
|
+
help="Preview without changes (default). Pass --no-dry-run to apply.",
|
|
175
|
+
),
|
|
176
|
+
yes: bool = YesOption,
|
|
177
|
+
exclude: list[str] = typer.Option(
|
|
178
|
+
[],
|
|
179
|
+
"--exclude",
|
|
180
|
+
help="Schema or schema.name to skip (repeatable).",
|
|
181
|
+
),
|
|
182
|
+
exclude_file: str | None = typer.Option(
|
|
183
|
+
None,
|
|
184
|
+
"--exclude-file",
|
|
185
|
+
help="Path to a file with one exclusion token per line.",
|
|
186
|
+
),
|
|
187
|
+
window_days: int = typer.Option(
|
|
188
|
+
DEFAULT_WINDOW_DAYS,
|
|
189
|
+
"--window-days",
|
|
190
|
+
help=(
|
|
191
|
+
"Consider a model 'live' if any matching dbt build in the last "
|
|
192
|
+
"N days produced it. Larger windows are more conservative "
|
|
193
|
+
"(fewer false-positive renames)."
|
|
194
|
+
),
|
|
195
|
+
),
|
|
196
|
+
) -> None:
|
|
197
|
+
"""Discover orphan dbt tables and rename them with _deprecated."""
|
|
198
|
+
if ctx.invoked_subcommand is not None:
|
|
199
|
+
return
|
|
200
|
+
|
|
201
|
+
if log is None:
|
|
202
|
+
log = _timestamped_log_path(APPLY_LOG_PREFIX)
|
|
203
|
+
|
|
204
|
+
if window_days < 1:
|
|
205
|
+
console.print("[red]Error: --window-days must be >= 1[/red]")
|
|
206
|
+
raise typer.Exit(code=1)
|
|
207
|
+
|
|
208
|
+
try:
|
|
209
|
+
excluded_user_schemas, excluded_user_relations = _parse_exclusions(
|
|
210
|
+
exclude, exclude_file
|
|
211
|
+
)
|
|
212
|
+
except ValidationError as exc:
|
|
213
|
+
console.print(f"[red]Error: {exc}[/red]")
|
|
214
|
+
raise typer.Exit(code=1)
|
|
215
|
+
|
|
216
|
+
engines = _engines_for_target(target)
|
|
217
|
+
if not dry_run:
|
|
218
|
+
_confirm_or_abort(
|
|
219
|
+
"Rename every orphaned dbt object with a _deprecated suffix?", yes
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
since = datetime.now(UTC) - timedelta(days=window_days)
|
|
223
|
+
|
|
224
|
+
all_entries: list[RenameEntry] = []
|
|
225
|
+
try:
|
|
226
|
+
for label, engine, env_prefix in engines:
|
|
227
|
+
all_entries.extend(
|
|
228
|
+
_run_for_engine(
|
|
229
|
+
label,
|
|
230
|
+
engine,
|
|
231
|
+
env_prefix=env_prefix,
|
|
232
|
+
excluded_user_schemas=excluded_user_schemas,
|
|
233
|
+
excluded_user_relations=excluded_user_relations,
|
|
234
|
+
window_days=window_days,
|
|
235
|
+
since=since,
|
|
236
|
+
dry_run=dry_run,
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
except ServiceError as exc:
|
|
240
|
+
_write_audit_log(log, all_entries, dry_run=dry_run)
|
|
241
|
+
console.print(f"[red]{exc}[/red]")
|
|
242
|
+
console.print(
|
|
243
|
+
f"[yellow]Partial audit log written to {log} "
|
|
244
|
+
f"({len(all_entries)} entries).[/yellow]"
|
|
245
|
+
)
|
|
246
|
+
raise typer.Exit(code=1)
|
|
247
|
+
|
|
248
|
+
_write_audit_log(log, all_entries, dry_run=dry_run)
|
|
249
|
+
|
|
250
|
+
prefix = "[DRY-RUN] " if dry_run else ""
|
|
251
|
+
console.print(
|
|
252
|
+
f"[green]{prefix}Processed {len(all_entries)} object(s). "
|
|
253
|
+
f"Log written to {log}.[/green]"
|
|
254
|
+
)
|
|
255
|
+
if dry_run and all_entries:
|
|
256
|
+
console.print(
|
|
257
|
+
"[dim]Re-run with --no-dry-run to apply these renames.[/dim]"
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _write_audit_log(
|
|
262
|
+
log_path: str, entries: list[RenameEntry], *, dry_run: bool
|
|
263
|
+
) -> None:
|
|
264
|
+
payload = {
|
|
265
|
+
"generated_at": datetime.now(UTC).isoformat(),
|
|
266
|
+
"dry_run": dry_run,
|
|
267
|
+
"source": "dbt-orphans",
|
|
268
|
+
"renames": entries,
|
|
269
|
+
}
|
|
270
|
+
with open(log_path, "w") as f:
|
|
271
|
+
json.dump(payload, f, indent=4)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _run_for_engine(
|
|
275
|
+
label: str,
|
|
276
|
+
engine: SqlEngine,
|
|
277
|
+
*,
|
|
278
|
+
env_prefix: str,
|
|
279
|
+
excluded_user_schemas: frozenset[str],
|
|
280
|
+
excluded_user_relations: frozenset[tuple[str, str]],
|
|
281
|
+
window_days: int,
|
|
282
|
+
since: datetime,
|
|
283
|
+
dry_run: bool,
|
|
284
|
+
) -> list[RenameEntry]:
|
|
285
|
+
try:
|
|
286
|
+
params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
|
|
287
|
+
dbt_node_prefix = node_prefix()
|
|
288
|
+
except ConfigError as exc:
|
|
289
|
+
raise ServiceError(f"[{label}] {exc}") from exc
|
|
290
|
+
if params is None:
|
|
291
|
+
console.print(
|
|
292
|
+
f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
|
|
293
|
+
)
|
|
294
|
+
return []
|
|
295
|
+
|
|
296
|
+
is_redshift = engine is SqlEngine.redshift
|
|
297
|
+
entries: list[RenameEntry] = []
|
|
298
|
+
|
|
299
|
+
with (
|
|
300
|
+
open_transactional_connection(params, dry_run=dry_run) as conn,
|
|
301
|
+
conn.cursor() as cur,
|
|
302
|
+
):
|
|
303
|
+
live = fetch_live_model_relations(
|
|
304
|
+
cur,
|
|
305
|
+
invocation_command=invocation_command(),
|
|
306
|
+
node_prefix=dbt_node_prefix,
|
|
307
|
+
statuses=LIVE_STATUSES,
|
|
308
|
+
since=since,
|
|
309
|
+
)
|
|
310
|
+
if not live:
|
|
311
|
+
console.print(
|
|
312
|
+
f"[yellow][{label}] No matching dbt builds in the last "
|
|
313
|
+
f"{window_days} day(s); skipping to avoid diffing "
|
|
314
|
+
f"against empty set.[/yellow]"
|
|
315
|
+
)
|
|
316
|
+
return []
|
|
317
|
+
|
|
318
|
+
excluded = excluded_schemas()
|
|
319
|
+
schemas_to_scan = sorted(s for s in live if s not in excluded)
|
|
320
|
+
existing = fetch_existing_relations(
|
|
321
|
+
cur, schemas_to_scan, is_redshift=is_redshift
|
|
322
|
+
)
|
|
323
|
+
orphans = diff_orphans(
|
|
324
|
+
live=live,
|
|
325
|
+
existing=existing,
|
|
326
|
+
excluded_schemas=excluded,
|
|
327
|
+
excluded_user_schemas=excluded_user_schemas,
|
|
328
|
+
excluded_user_relations=excluded_user_relations,
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
_print_summary(label, live, existing, orphans)
|
|
332
|
+
|
|
333
|
+
for schema, names in orphans.items():
|
|
334
|
+
for name in names:
|
|
335
|
+
entry = _rename_orphan(
|
|
336
|
+
cur,
|
|
337
|
+
label,
|
|
338
|
+
schema,
|
|
339
|
+
name,
|
|
340
|
+
is_redshift=is_redshift,
|
|
341
|
+
dry_run=dry_run,
|
|
342
|
+
)
|
|
343
|
+
if entry is not None:
|
|
344
|
+
entries.append(entry)
|
|
345
|
+
|
|
346
|
+
return entries
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _print_summary(
|
|
350
|
+
label: str,
|
|
351
|
+
live: dict[str, set[str]],
|
|
352
|
+
existing: dict[str, set[str]],
|
|
353
|
+
orphans: dict[str, list[str]],
|
|
354
|
+
) -> None:
|
|
355
|
+
live_count = sum(len(v) for v in live.values())
|
|
356
|
+
existing_count = sum(len(v) for v in existing.values())
|
|
357
|
+
orphan_count = sum(len(v) for v in orphans.values())
|
|
358
|
+
per_schema = ", ".join(
|
|
359
|
+
f"{len(v)} in {s}" for s, v in sorted(orphans.items())
|
|
360
|
+
)
|
|
361
|
+
suffix = f" ({per_schema})" if per_schema else ""
|
|
362
|
+
console.print(
|
|
363
|
+
f"[cyan][{label}] {live_count} live dbt models; {existing_count} "
|
|
364
|
+
f"existing in live schemas; {orphan_count} orphans after "
|
|
365
|
+
f"exclusions{suffix}[/cyan]"
|
|
366
|
+
)
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
def _rename_orphan(
|
|
370
|
+
cur: Any,
|
|
371
|
+
label: str,
|
|
372
|
+
schema: str,
|
|
373
|
+
name: str,
|
|
374
|
+
*,
|
|
375
|
+
is_redshift: bool,
|
|
376
|
+
dry_run: bool,
|
|
377
|
+
) -> RenameEntry | None:
|
|
378
|
+
kind = classify_object(cur, schema, name, is_redshift=is_redshift)
|
|
379
|
+
if kind is None:
|
|
380
|
+
console.print(
|
|
381
|
+
f"[yellow][{label}] {schema}.{name} no longer present, skipping.[/yellow]"
|
|
382
|
+
)
|
|
383
|
+
return None
|
|
384
|
+
|
|
385
|
+
new_name = f"{name}{DEPRECATED_SUFFIX}"
|
|
386
|
+
if classify_object(cur, schema, new_name, is_redshift=is_redshift) is not None:
|
|
387
|
+
console.print(
|
|
388
|
+
f"[yellow][{label}] {schema}.{new_name} already exists, "
|
|
389
|
+
f"skipping rename of {schema}.{name}.[/yellow]"
|
|
390
|
+
)
|
|
391
|
+
return None
|
|
392
|
+
|
|
393
|
+
action = "[DRY-RUN] Would rename" if dry_run else "Renaming"
|
|
394
|
+
console.print(
|
|
395
|
+
f"[blue][{label}] {action} {kind} "
|
|
396
|
+
f"{schema}.{name} -> {schema}.{new_name}[/blue]"
|
|
397
|
+
)
|
|
398
|
+
|
|
399
|
+
if not dry_run:
|
|
400
|
+
rename_object(cur, schema, name, new_name, kind, is_redshift=is_redshift)
|
|
401
|
+
|
|
402
|
+
return RenameEntry(
|
|
403
|
+
database=label,
|
|
404
|
+
schema=schema,
|
|
405
|
+
old_name=name,
|
|
406
|
+
new_name=new_name,
|
|
407
|
+
kind=kind,
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
@app.command("revert")
|
|
412
|
+
def revert_cmd(
|
|
413
|
+
log: str | None = typer.Option(
|
|
414
|
+
None,
|
|
415
|
+
"--log",
|
|
416
|
+
help=(
|
|
417
|
+
"Audit log input path. Defaults to the newest "
|
|
418
|
+
"dbt_orphans-*.log.json in the log directory."
|
|
419
|
+
),
|
|
420
|
+
),
|
|
421
|
+
target: str = TargetOption,
|
|
422
|
+
dry_run: bool = typer.Option(
|
|
423
|
+
True,
|
|
424
|
+
"--dry-run/--no-dry-run",
|
|
425
|
+
help="Preview without changes (default). Pass --no-dry-run to revert.",
|
|
426
|
+
),
|
|
427
|
+
) -> None:
|
|
428
|
+
"""Undo a previous dbt-orphans run using the audit log."""
|
|
429
|
+
if log is None:
|
|
430
|
+
log = _find_latest_log(APPLY_LOG_PREFIX)
|
|
431
|
+
if log is None:
|
|
432
|
+
console.print(
|
|
433
|
+
f"[red]Error: no {APPLY_LOG_PREFIX} log found in "
|
|
434
|
+
f"{LOG_DIR}/[/red]"
|
|
435
|
+
)
|
|
436
|
+
console.print(
|
|
437
|
+
"[dim]Run `dp db dbt-orphans` first or pass --log explicitly.[/dim]"
|
|
438
|
+
)
|
|
439
|
+
raise typer.Exit(code=1)
|
|
440
|
+
console.print(f"[dim]Using latest log: {log}[/dim]")
|
|
441
|
+
if not os.path.exists(log):
|
|
442
|
+
console.print(f"[red]Error: log file not found: {log}[/red]")
|
|
443
|
+
console.print(
|
|
444
|
+
"[dim]Run `dp db dbt-orphans` first to generate it.[/dim]"
|
|
445
|
+
)
|
|
446
|
+
raise typer.Exit(code=1)
|
|
447
|
+
|
|
448
|
+
try:
|
|
449
|
+
with open(log) as f:
|
|
450
|
+
payload = json.load(f)
|
|
451
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
452
|
+
console.print(f"[red]Error: could not read log {log}: {exc}[/red]")
|
|
453
|
+
raise typer.Exit(code=1)
|
|
454
|
+
if not isinstance(payload, dict):
|
|
455
|
+
console.print(f"[red]Error: malformed log {log}: expected an object[/red]")
|
|
456
|
+
raise typer.Exit(code=1)
|
|
457
|
+
|
|
458
|
+
if payload.get("dry_run"):
|
|
459
|
+
console.print(
|
|
460
|
+
"[yellow]Log was generated in dry-run mode; "
|
|
461
|
+
"no renames were actually applied.[/yellow]"
|
|
462
|
+
)
|
|
463
|
+
|
|
464
|
+
renames = payload.get("renames") or []
|
|
465
|
+
if not renames:
|
|
466
|
+
console.print(
|
|
467
|
+
"[dim]No renames recorded in the log; nothing to revert.[/dim]"
|
|
468
|
+
)
|
|
469
|
+
return
|
|
470
|
+
|
|
471
|
+
total = 0
|
|
472
|
+
try:
|
|
473
|
+
for label, engine, env_prefix in _engines_for_target(target):
|
|
474
|
+
entries = [
|
|
475
|
+
r for r in renames
|
|
476
|
+
if isinstance(r, dict) and r.get("database") == label
|
|
477
|
+
]
|
|
478
|
+
if not entries:
|
|
479
|
+
console.print(f"[dim][{label}] No entries in log.[/dim]")
|
|
480
|
+
continue
|
|
481
|
+
total += _revert_for_engine(
|
|
482
|
+
label, engine, entries, env_prefix=env_prefix, dry_run=dry_run
|
|
483
|
+
)
|
|
484
|
+
except ServiceError as exc:
|
|
485
|
+
console.print(f"[red]{exc}[/red]")
|
|
486
|
+
console.print(
|
|
487
|
+
f"[yellow]Reverted {total} object(s) before the failure.[/yellow]"
|
|
488
|
+
)
|
|
489
|
+
raise typer.Exit(code=1)
|
|
490
|
+
|
|
491
|
+
prefix = "[DRY-RUN] " if dry_run else ""
|
|
492
|
+
console.print(f"[green]{prefix}Reverted {total} object(s).[/green]")
|
|
493
|
+
if dry_run and total:
|
|
494
|
+
console.print(
|
|
495
|
+
"[dim]Re-run with --no-dry-run to revert these renames.[/dim]"
|
|
496
|
+
)
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
def _revert_for_engine(
|
|
500
|
+
label: str,
|
|
501
|
+
engine: SqlEngine,
|
|
502
|
+
entries: list[dict],
|
|
503
|
+
*,
|
|
504
|
+
env_prefix: str,
|
|
505
|
+
dry_run: bool,
|
|
506
|
+
) -> int:
|
|
507
|
+
try:
|
|
508
|
+
params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
|
|
509
|
+
except ConfigError as exc:
|
|
510
|
+
raise ServiceError(f"[{label}] {exc}") from exc
|
|
511
|
+
if params is None:
|
|
512
|
+
console.print(
|
|
513
|
+
f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
|
|
514
|
+
)
|
|
515
|
+
return 0
|
|
516
|
+
|
|
517
|
+
is_redshift = engine is SqlEngine.redshift
|
|
518
|
+
reverted = 0
|
|
519
|
+
|
|
520
|
+
with (
|
|
521
|
+
open_transactional_connection(params, dry_run=dry_run) as conn,
|
|
522
|
+
conn.cursor() as cur,
|
|
523
|
+
):
|
|
524
|
+
for entry in entries:
|
|
525
|
+
if _revert_one(
|
|
526
|
+
cur, label, entry, is_redshift=is_redshift, dry_run=dry_run
|
|
527
|
+
):
|
|
528
|
+
reverted += 1
|
|
529
|
+
|
|
530
|
+
return reverted
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def _revert_one(
|
|
534
|
+
cur: Any,
|
|
535
|
+
label: str,
|
|
536
|
+
entry: dict,
|
|
537
|
+
*,
|
|
538
|
+
is_redshift: bool,
|
|
539
|
+
dry_run: bool,
|
|
540
|
+
) -> bool:
|
|
541
|
+
schema = entry.get("schema")
|
|
542
|
+
current_name = entry.get("new_name")
|
|
543
|
+
original_name = entry.get("old_name")
|
|
544
|
+
if not (schema and current_name and original_name):
|
|
545
|
+
console.print(
|
|
546
|
+
f"[yellow][{label}] Skipping malformed log entry: {entry!r}[/yellow]"
|
|
547
|
+
)
|
|
548
|
+
return False
|
|
549
|
+
kind: ObjectKind = entry.get("kind", "table")
|
|
550
|
+
|
|
551
|
+
if classify_object(cur, schema, current_name, is_redshift=is_redshift) is None:
|
|
552
|
+
console.print(
|
|
553
|
+
f"[dim][{label}] {schema}.{current_name} not found, "
|
|
554
|
+
f"skipping revert.[/dim]"
|
|
555
|
+
)
|
|
556
|
+
return False
|
|
557
|
+
|
|
558
|
+
if classify_object(cur, schema, original_name, is_redshift=is_redshift) is not None:
|
|
559
|
+
console.print(
|
|
560
|
+
f"[yellow][{label}] {schema}.{original_name} already exists, "
|
|
561
|
+
f"cannot revert {schema}.{current_name}.[/yellow]"
|
|
562
|
+
)
|
|
563
|
+
return False
|
|
564
|
+
|
|
565
|
+
action = "[DRY-RUN] Would revert" if dry_run else "Reverting"
|
|
566
|
+
console.print(
|
|
567
|
+
f"[blue][{label}] {action} {kind} "
|
|
568
|
+
f"{schema}.{current_name} -> {schema}.{original_name}[/blue]"
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
if not dry_run:
|
|
572
|
+
rename_object(
|
|
573
|
+
cur, schema, current_name, original_name, kind, is_redshift=is_redshift
|
|
574
|
+
)
|
|
575
|
+
return True
|
|
576
|
+
|
|
577
|
+
|
|
578
|
+
def _renamed_at_index() -> dict[tuple[str, str, str], datetime]:
|
|
579
|
+
"""Map ``(database, schema, new_name)`` -> newest rename timestamp.
|
|
580
|
+
|
|
581
|
+
Built from the applied (non-dry-run) apply logs so purge can enforce a
|
|
582
|
+
grace period even though the warehouse doesn't track rename times.
|
|
583
|
+
"""
|
|
584
|
+
index: dict[tuple[str, str, str], datetime] = {}
|
|
585
|
+
for path in _matching_logs(APPLY_LOG_PREFIX):
|
|
586
|
+
try:
|
|
587
|
+
with open(path) as f:
|
|
588
|
+
payload = json.load(f)
|
|
589
|
+
except (OSError, json.JSONDecodeError):
|
|
590
|
+
continue
|
|
591
|
+
if not isinstance(payload, dict) or payload.get("dry_run"):
|
|
592
|
+
continue
|
|
593
|
+
try:
|
|
594
|
+
when = datetime.fromisoformat(payload.get("generated_at", ""))
|
|
595
|
+
except ValueError:
|
|
596
|
+
continue
|
|
597
|
+
for r in payload.get("renames") or []:
|
|
598
|
+
if not isinstance(r, dict):
|
|
599
|
+
continue
|
|
600
|
+
database = r.get("database")
|
|
601
|
+
schema = r.get("schema")
|
|
602
|
+
new_name = r.get("new_name")
|
|
603
|
+
if not (database and schema and new_name):
|
|
604
|
+
continue
|
|
605
|
+
key = (str(database), str(schema), str(new_name))
|
|
606
|
+
if key not in index or when > index[key]:
|
|
607
|
+
index[key] = when
|
|
608
|
+
return index
|
|
609
|
+
|
|
610
|
+
|
|
611
|
+
@app.command("purge")
|
|
612
|
+
def purge_cmd(
|
|
613
|
+
log: str | None = typer.Option(
|
|
614
|
+
None,
|
|
615
|
+
"--log",
|
|
616
|
+
help=(
|
|
617
|
+
"Purge audit log output path. Defaults to "
|
|
618
|
+
"~/.config/dataplat/logs/dbt-orphans/"
|
|
619
|
+
"dbt_orphans_purge-<UTC timestamp>.log.json (unique per run)."
|
|
620
|
+
),
|
|
621
|
+
),
|
|
622
|
+
target: str = TargetOption,
|
|
623
|
+
dry_run: bool = typer.Option(
|
|
624
|
+
True,
|
|
625
|
+
"--dry-run/--no-dry-run",
|
|
626
|
+
help="Preview without changes (default). Pass --no-dry-run to drop.",
|
|
627
|
+
),
|
|
628
|
+
yes: bool = YesOption,
|
|
629
|
+
older_than: int | None = typer.Option(
|
|
630
|
+
None,
|
|
631
|
+
"--older-than",
|
|
632
|
+
min=0,
|
|
633
|
+
help=(
|
|
634
|
+
"Only drop objects renamed at least N days ago (per the audit "
|
|
635
|
+
"logs). Objects with no recorded rename are skipped unless "
|
|
636
|
+
"--include-unknown."
|
|
637
|
+
),
|
|
638
|
+
),
|
|
639
|
+
include_unknown: bool = typer.Option(
|
|
640
|
+
False,
|
|
641
|
+
"--include-unknown",
|
|
642
|
+
help="With --older-than: also drop objects that have no recorded rename.",
|
|
643
|
+
),
|
|
644
|
+
exclude: list[str] = typer.Option(
|
|
645
|
+
[],
|
|
646
|
+
"--exclude",
|
|
647
|
+
help="Schema or schema.name to skip (repeatable).",
|
|
648
|
+
),
|
|
649
|
+
exclude_file: str | None = typer.Option(
|
|
650
|
+
None,
|
|
651
|
+
"--exclude-file",
|
|
652
|
+
help="Path to a file with one exclusion token per line.",
|
|
653
|
+
),
|
|
654
|
+
) -> None:
|
|
655
|
+
"""Permanently drop every object ending in _deprecated (irreversible)."""
|
|
656
|
+
if log is None:
|
|
657
|
+
log = _timestamped_log_path(PURGE_LOG_PREFIX)
|
|
658
|
+
|
|
659
|
+
try:
|
|
660
|
+
excluded_user_schemas, excluded_user_relations = _parse_exclusions(
|
|
661
|
+
exclude, exclude_file
|
|
662
|
+
)
|
|
663
|
+
except ValidationError as exc:
|
|
664
|
+
console.print(f"[red]Error: {exc}[/red]")
|
|
665
|
+
raise typer.Exit(code=1)
|
|
666
|
+
|
|
667
|
+
engines = _engines_for_target(target)
|
|
668
|
+
if not dry_run:
|
|
669
|
+
_confirm_or_abort(
|
|
670
|
+
"Permanently DROP every *_deprecated object? This cannot be undone.",
|
|
671
|
+
yes,
|
|
672
|
+
)
|
|
673
|
+
|
|
674
|
+
renamed_at = _renamed_at_index() if older_than is not None else None
|
|
675
|
+
cutoff = (
|
|
676
|
+
datetime.now(UTC) - timedelta(days=older_than)
|
|
677
|
+
if older_than is not None
|
|
678
|
+
else None
|
|
679
|
+
)
|
|
680
|
+
|
|
681
|
+
all_drops: list[DropEntry] = []
|
|
682
|
+
try:
|
|
683
|
+
for label, engine, env_prefix in engines:
|
|
684
|
+
all_drops.extend(
|
|
685
|
+
_purge_for_engine(
|
|
686
|
+
label,
|
|
687
|
+
engine,
|
|
688
|
+
env_prefix=env_prefix,
|
|
689
|
+
excluded_user_schemas=excluded_user_schemas,
|
|
690
|
+
excluded_user_relations=excluded_user_relations,
|
|
691
|
+
dry_run=dry_run,
|
|
692
|
+
renamed_at=renamed_at,
|
|
693
|
+
cutoff=cutoff,
|
|
694
|
+
include_unknown=include_unknown,
|
|
695
|
+
)
|
|
696
|
+
)
|
|
697
|
+
except ServiceError as exc:
|
|
698
|
+
_write_purge_log(log, all_drops, dry_run=dry_run)
|
|
699
|
+
console.print(f"[red]{exc}[/red]")
|
|
700
|
+
console.print(
|
|
701
|
+
f"[yellow]Partial purge log written to {log} "
|
|
702
|
+
f"({len(all_drops)} entries).[/yellow]"
|
|
703
|
+
)
|
|
704
|
+
raise typer.Exit(code=1)
|
|
705
|
+
|
|
706
|
+
_write_purge_log(log, all_drops, dry_run=dry_run)
|
|
707
|
+
|
|
708
|
+
prefix = "[DRY-RUN] " if dry_run else ""
|
|
709
|
+
console.print(
|
|
710
|
+
f"[green]{prefix}Dropped {len(all_drops)} object(s). "
|
|
711
|
+
f"Log written to {log}.[/green]"
|
|
712
|
+
)
|
|
713
|
+
if dry_run and all_drops:
|
|
714
|
+
console.print(
|
|
715
|
+
"[dim]Re-run with --no-dry-run to apply these drops "
|
|
716
|
+
"(irreversible).[/dim]"
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _purge_for_engine(
|
|
721
|
+
label: str,
|
|
722
|
+
engine: SqlEngine,
|
|
723
|
+
*,
|
|
724
|
+
env_prefix: str,
|
|
725
|
+
excluded_user_schemas: frozenset[str],
|
|
726
|
+
excluded_user_relations: frozenset[tuple[str, str]],
|
|
727
|
+
dry_run: bool,
|
|
728
|
+
renamed_at: dict[tuple[str, str, str], datetime] | None = None,
|
|
729
|
+
cutoff: datetime | None = None,
|
|
730
|
+
include_unknown: bool = False,
|
|
731
|
+
) -> list[DropEntry]:
|
|
732
|
+
try:
|
|
733
|
+
params = resolve_orphans_connection_params(engine, env_prefix=env_prefix)
|
|
734
|
+
except ConfigError as exc:
|
|
735
|
+
raise ServiceError(f"[{label}] {exc}") from exc
|
|
736
|
+
if params is None:
|
|
737
|
+
console.print(
|
|
738
|
+
f"[yellow][{label}] Missing connection parameters, skipping.[/yellow]"
|
|
739
|
+
)
|
|
740
|
+
return []
|
|
741
|
+
|
|
742
|
+
is_redshift = engine is SqlEngine.redshift
|
|
743
|
+
effective_excluded_schemas = excluded_schemas() | excluded_user_schemas
|
|
744
|
+
drops: list[DropEntry] = []
|
|
745
|
+
|
|
746
|
+
with (
|
|
747
|
+
open_transactional_connection(params, dry_run=dry_run) as conn,
|
|
748
|
+
conn.cursor() as cur,
|
|
749
|
+
):
|
|
750
|
+
deprecated = fetch_deprecated_objects(
|
|
751
|
+
cur,
|
|
752
|
+
is_redshift=is_redshift,
|
|
753
|
+
excluded_schemas=effective_excluded_schemas,
|
|
754
|
+
)
|
|
755
|
+
deprecated = [
|
|
756
|
+
(s, n, k)
|
|
757
|
+
for s, n, k in deprecated
|
|
758
|
+
if (s, n) not in excluded_user_relations
|
|
759
|
+
]
|
|
760
|
+
|
|
761
|
+
if renamed_at is not None and cutoff is not None:
|
|
762
|
+
deprecated = _apply_age_filter(
|
|
763
|
+
deprecated,
|
|
764
|
+
label=label,
|
|
765
|
+
renamed_at=renamed_at,
|
|
766
|
+
cutoff=cutoff,
|
|
767
|
+
include_unknown=include_unknown,
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
console.print(
|
|
771
|
+
f"[cyan][{label}] {len(deprecated)} deprecated "
|
|
772
|
+
f"object(s) after exclusions.[/cyan]"
|
|
773
|
+
)
|
|
774
|
+
|
|
775
|
+
for schema, name, kind in deprecated:
|
|
776
|
+
entry = _drop_one(
|
|
777
|
+
cur, label, schema, name, kind, dry_run=dry_run
|
|
778
|
+
)
|
|
779
|
+
if entry is not None:
|
|
780
|
+
drops.append(entry)
|
|
781
|
+
|
|
782
|
+
return drops
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def _apply_age_filter(
|
|
786
|
+
deprecated: list[tuple[str, str, Any]],
|
|
787
|
+
*,
|
|
788
|
+
label: str,
|
|
789
|
+
renamed_at: dict[tuple[str, str, str], datetime],
|
|
790
|
+
cutoff: datetime,
|
|
791
|
+
include_unknown: bool,
|
|
792
|
+
) -> list[tuple[str, str, Any]]:
|
|
793
|
+
"""Keep only objects renamed before ``cutoff`` per the audit logs."""
|
|
794
|
+
kept: list[tuple[str, str, Any]] = []
|
|
795
|
+
for schema, name, kind in deprecated:
|
|
796
|
+
when = renamed_at.get((label, schema, name))
|
|
797
|
+
if when is None:
|
|
798
|
+
if include_unknown:
|
|
799
|
+
kept.append((schema, name, kind))
|
|
800
|
+
else:
|
|
801
|
+
console.print(
|
|
802
|
+
f"[yellow][{label}] {schema}.{name}: no recorded rename; "
|
|
803
|
+
f"skipping (pass --include-unknown to drop anyway).[/yellow]"
|
|
804
|
+
)
|
|
805
|
+
continue
|
|
806
|
+
if when <= cutoff:
|
|
807
|
+
kept.append((schema, name, kind))
|
|
808
|
+
else:
|
|
809
|
+
console.print(
|
|
810
|
+
f"[dim][{label}] {schema}.{name}: renamed "
|
|
811
|
+
f"{when.date()}, inside the grace period; skipping.[/dim]"
|
|
812
|
+
)
|
|
813
|
+
return kept
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _drop_one(
|
|
817
|
+
cur: Any,
|
|
818
|
+
label: str,
|
|
819
|
+
schema: str,
|
|
820
|
+
name: str,
|
|
821
|
+
kind: ObjectKind,
|
|
822
|
+
*,
|
|
823
|
+
dry_run: bool,
|
|
824
|
+
) -> DropEntry | None:
|
|
825
|
+
action = "[DRY-RUN] Would drop" if dry_run else "Dropping"
|
|
826
|
+
console.print(f"[blue][{label}] {action} {kind} {schema}.{name}[/blue]")
|
|
827
|
+
if not dry_run:
|
|
828
|
+
drop_object(cur, schema, name, kind)
|
|
829
|
+
return DropEntry(database=label, schema=schema, name=name, kind=kind)
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _write_purge_log(
|
|
833
|
+
log_path: str, entries: list[DropEntry], *, dry_run: bool
|
|
834
|
+
) -> None:
|
|
835
|
+
payload = {
|
|
836
|
+
"generated_at": datetime.now(UTC).isoformat(),
|
|
837
|
+
"dry_run": dry_run,
|
|
838
|
+
"source": "dbt-orphans-purge",
|
|
839
|
+
"drops": entries,
|
|
840
|
+
}
|
|
841
|
+
with open(log_path, "w") as f:
|
|
842
|
+
json.dump(payload, f, indent=4)
|