metaxy 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of metaxy might be problematic. Click here for more details.
- metaxy/__init__.py +61 -0
- metaxy/_testing.py +542 -0
- metaxy/_utils.py +16 -0
- metaxy/_version.py +1 -0
- metaxy/cli/app.py +76 -0
- metaxy/cli/context.py +71 -0
- metaxy/cli/graph.py +576 -0
- metaxy/cli/graph_diff.py +290 -0
- metaxy/cli/list.py +42 -0
- metaxy/cli/metadata.py +271 -0
- metaxy/cli/migrations.py +862 -0
- metaxy/cli/push.py +55 -0
- metaxy/config.py +450 -0
- metaxy/data_versioning/__init__.py +24 -0
- metaxy/data_versioning/calculators/__init__.py +13 -0
- metaxy/data_versioning/calculators/base.py +97 -0
- metaxy/data_versioning/calculators/duckdb.py +186 -0
- metaxy/data_versioning/calculators/ibis.py +225 -0
- metaxy/data_versioning/calculators/polars.py +135 -0
- metaxy/data_versioning/diff/__init__.py +15 -0
- metaxy/data_versioning/diff/base.py +150 -0
- metaxy/data_versioning/diff/narwhals.py +108 -0
- metaxy/data_versioning/hash_algorithms.py +19 -0
- metaxy/data_versioning/joiners/__init__.py +9 -0
- metaxy/data_versioning/joiners/base.py +70 -0
- metaxy/data_versioning/joiners/narwhals.py +235 -0
- metaxy/entrypoints.py +309 -0
- metaxy/ext/__init__.py +1 -0
- metaxy/ext/alembic.py +326 -0
- metaxy/ext/sqlmodel.py +172 -0
- metaxy/ext/sqlmodel_system_tables.py +139 -0
- metaxy/graph/__init__.py +21 -0
- metaxy/graph/diff/__init__.py +21 -0
- metaxy/graph/diff/diff_models.py +399 -0
- metaxy/graph/diff/differ.py +740 -0
- metaxy/graph/diff/models.py +418 -0
- metaxy/graph/diff/rendering/__init__.py +18 -0
- metaxy/graph/diff/rendering/base.py +274 -0
- metaxy/graph/diff/rendering/cards.py +188 -0
- metaxy/graph/diff/rendering/formatter.py +805 -0
- metaxy/graph/diff/rendering/graphviz.py +246 -0
- metaxy/graph/diff/rendering/mermaid.py +320 -0
- metaxy/graph/diff/rendering/rich.py +165 -0
- metaxy/graph/diff/rendering/theme.py +48 -0
- metaxy/graph/diff/traversal.py +247 -0
- metaxy/graph/utils.py +58 -0
- metaxy/metadata_store/__init__.py +31 -0
- metaxy/metadata_store/_protocols.py +38 -0
- metaxy/metadata_store/base.py +1676 -0
- metaxy/metadata_store/clickhouse.py +161 -0
- metaxy/metadata_store/duckdb.py +167 -0
- metaxy/metadata_store/exceptions.py +43 -0
- metaxy/metadata_store/ibis.py +451 -0
- metaxy/metadata_store/memory.py +228 -0
- metaxy/metadata_store/sqlite.py +187 -0
- metaxy/metadata_store/system_tables.py +257 -0
- metaxy/migrations/__init__.py +34 -0
- metaxy/migrations/detector.py +153 -0
- metaxy/migrations/executor.py +208 -0
- metaxy/migrations/loader.py +260 -0
- metaxy/migrations/models.py +718 -0
- metaxy/migrations/ops.py +390 -0
- metaxy/models/__init__.py +0 -0
- metaxy/models/bases.py +6 -0
- metaxy/models/constants.py +24 -0
- metaxy/models/feature.py +665 -0
- metaxy/models/feature_spec.py +105 -0
- metaxy/models/field.py +25 -0
- metaxy/models/plan.py +155 -0
- metaxy/models/types.py +157 -0
- metaxy/py.typed +0 -0
- metaxy-0.0.0.dist-info/METADATA +247 -0
- metaxy-0.0.0.dist-info/RECORD +75 -0
- metaxy-0.0.0.dist-info/WHEEL +4 -0
- metaxy-0.0.0.dist-info/entry_points.txt +3 -0
metaxy/cli/migrations.py
ADDED
|
@@ -0,0 +1,862 @@
|
|
|
1
|
+
"""New Migration CLI commands using event-based system."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Annotated
|
|
6
|
+
|
|
7
|
+
import cyclopts
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
|
|
10
|
+
# Rich console for formatted output
|
|
11
|
+
console = Console()
|
|
12
|
+
|
|
13
|
+
# Migrations subcommand app
|
|
14
|
+
app = cyclopts.App(
|
|
15
|
+
name="migrations", # pyrefly: ignore[unexpected-keyword]
|
|
16
|
+
help="Metadata migration commands", # pyrefly: ignore[unexpected-keyword]
|
|
17
|
+
console=console, # pyrefly: ignore[unexpected-keyword]
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@app.command
|
|
22
|
+
def generate(
|
|
23
|
+
*,
|
|
24
|
+
name: Annotated[
|
|
25
|
+
str | None,
|
|
26
|
+
cyclopts.Parameter(help="Migration name (creates {timestamp}_{name} ID)"),
|
|
27
|
+
] = None,
|
|
28
|
+
store: Annotated[
|
|
29
|
+
str | None,
|
|
30
|
+
cyclopts.Parameter(help="Store name (defaults to default)"),
|
|
31
|
+
] = None,
|
|
32
|
+
from_snapshot: Annotated[
|
|
33
|
+
str | None,
|
|
34
|
+
cyclopts.Parameter(
|
|
35
|
+
help="Compare from this historical snapshot version (defaults to latest)"
|
|
36
|
+
),
|
|
37
|
+
] = None,
|
|
38
|
+
op: Annotated[
|
|
39
|
+
list[str],
|
|
40
|
+
cyclopts.Parameter(
|
|
41
|
+
help="Operation class path to use (can be repeated). Example: metaxy.migrations.ops.DataVersionReconciliation"
|
|
42
|
+
),
|
|
43
|
+
],
|
|
44
|
+
):
|
|
45
|
+
"""Generate migration from detected feature changes.
|
|
46
|
+
|
|
47
|
+
Compares the latest snapshot in the store (or specified from_snapshot)
|
|
48
|
+
with the current active graph to detect changes.
|
|
49
|
+
|
|
50
|
+
The migration is recorded in the system tables (not a YAML file).
|
|
51
|
+
|
|
52
|
+
Examples:
|
|
53
|
+
# Generate with DataVersionReconciliation operation
|
|
54
|
+
$ metaxy migrations generate --op metaxy.migrations.ops.DataVersionReconciliation
|
|
55
|
+
|
|
56
|
+
# Custom operation type
|
|
57
|
+
$ metaxy migrations generate --op myproject.ops.CustomReconciliation
|
|
58
|
+
|
|
59
|
+
# Multiple operations (future use)
|
|
60
|
+
$ metaxy migrations generate \
|
|
61
|
+
--op metaxy.migrations.ops.DataVersionReconciliation \
|
|
62
|
+
--op myproject.ops.CustomBackfill
|
|
63
|
+
"""
|
|
64
|
+
from pathlib import Path
|
|
65
|
+
|
|
66
|
+
from metaxy.cli.context import get_config
|
|
67
|
+
from metaxy.entrypoints import load_features
|
|
68
|
+
from metaxy.migrations.detector import detect_migration
|
|
69
|
+
|
|
70
|
+
# Load features from entrypoints
|
|
71
|
+
load_features()
|
|
72
|
+
|
|
73
|
+
# Convert op_type list to ops format
|
|
74
|
+
if len(op) == 0:
|
|
75
|
+
app.console.print(
|
|
76
|
+
"[red]✗[/red] --op is required. "
|
|
77
|
+
"Example: --op metaxy.migrations.ops.DataVersionReconciliation"
|
|
78
|
+
)
|
|
79
|
+
raise SystemExit(1)
|
|
80
|
+
|
|
81
|
+
ops = [{"type": op} for op in op]
|
|
82
|
+
|
|
83
|
+
# Get store
|
|
84
|
+
config = get_config()
|
|
85
|
+
metadata_store = config.get_store(store)
|
|
86
|
+
migrations_dir = Path(config.migrations_dir)
|
|
87
|
+
|
|
88
|
+
with metadata_store:
|
|
89
|
+
# Detect migration and write YAML
|
|
90
|
+
migration = detect_migration(
|
|
91
|
+
metadata_store,
|
|
92
|
+
from_snapshot_version=from_snapshot,
|
|
93
|
+
ops=ops,
|
|
94
|
+
migrations_dir=migrations_dir,
|
|
95
|
+
name=name,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
if migration is None:
|
|
99
|
+
app.console.print("[yellow]No changes detected[/yellow]")
|
|
100
|
+
app.console.print(" Current graph matches latest snapshot")
|
|
101
|
+
return
|
|
102
|
+
|
|
103
|
+
# Print summary
|
|
104
|
+
yaml_path = migrations_dir / f"{migration.migration_id}.yaml"
|
|
105
|
+
app.console.print("\n[green]✓[/green] Migration generated")
|
|
106
|
+
app.console.print(f" Migration ID: {migration.migration_id}")
|
|
107
|
+
app.console.print(f" YAML file: {yaml_path}")
|
|
108
|
+
app.console.print(f" From snapshot: {migration.from_snapshot_version}")
|
|
109
|
+
app.console.print(f" To snapshot: {migration.to_snapshot_version}")
|
|
110
|
+
|
|
111
|
+
# Show description
|
|
112
|
+
description = migration.get_description(metadata_store)
|
|
113
|
+
app.console.print(f" Description: {description}")
|
|
114
|
+
|
|
115
|
+
# Get affected features (computed on-demand)
|
|
116
|
+
affected_features = migration.get_affected_features(metadata_store)
|
|
117
|
+
app.console.print(f"\n Affected features ({len(affected_features)}):")
|
|
118
|
+
for feature_key in affected_features[:5]:
|
|
119
|
+
app.console.print(f" ✓ {feature_key}")
|
|
120
|
+
if len(affected_features) > 5:
|
|
121
|
+
app.console.print(f" ... and {len(affected_features) - 5} more")
|
|
122
|
+
|
|
123
|
+
app.console.print("\n[bold]NEXT STEPS:[/bold]")
|
|
124
|
+
app.console.print(f"1. Review migration YAML: {yaml_path}")
|
|
125
|
+
app.console.print(
|
|
126
|
+
f"2. Run 'metaxy migrations apply {migration.migration_id}' to execute. Use CD for production!"
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@app.command
|
|
131
|
+
def apply(
|
|
132
|
+
migration_id: Annotated[
|
|
133
|
+
str | None,
|
|
134
|
+
cyclopts.Parameter(
|
|
135
|
+
help="Migration ID to apply (applies all unapplied if not specified)"
|
|
136
|
+
),
|
|
137
|
+
] = None,
|
|
138
|
+
*,
|
|
139
|
+
dry_run: Annotated[
|
|
140
|
+
bool,
|
|
141
|
+
cyclopts.Parameter(help="Preview changes without executing"),
|
|
142
|
+
] = False,
|
|
143
|
+
):
|
|
144
|
+
"""Apply migration(s) from YAML files.
|
|
145
|
+
|
|
146
|
+
Reads migration definitions from .metaxy/migrations/ directory (git).
|
|
147
|
+
Follows parent chain to ensure correct order.
|
|
148
|
+
Tracks execution state in database (events).
|
|
149
|
+
|
|
150
|
+
Examples:
|
|
151
|
+
# Apply all unapplied migrations in chain order
|
|
152
|
+
$ metaxy migrations apply
|
|
153
|
+
|
|
154
|
+
# Apply specific migration (and all its unapplied predecessors)
|
|
155
|
+
$ metaxy migrations apply 20250113_103000
|
|
156
|
+
|
|
157
|
+
# Dry run
|
|
158
|
+
$ metaxy migrations apply --dry-run
|
|
159
|
+
"""
|
|
160
|
+
from pathlib import Path
|
|
161
|
+
|
|
162
|
+
from metaxy.cli.context import get_store
|
|
163
|
+
from metaxy.entrypoints import load_features
|
|
164
|
+
from metaxy.metadata_store.system_tables import SystemTableStorage
|
|
165
|
+
from metaxy.migrations.executor import MigrationExecutor
|
|
166
|
+
from metaxy.migrations.loader import build_migration_chain
|
|
167
|
+
|
|
168
|
+
# Load features from entrypoints
|
|
169
|
+
load_features()
|
|
170
|
+
|
|
171
|
+
metadata_store = get_store()
|
|
172
|
+
migrations_dir = Path(".metaxy/migrations")
|
|
173
|
+
|
|
174
|
+
with metadata_store:
|
|
175
|
+
storage = SystemTableStorage(metadata_store)
|
|
176
|
+
|
|
177
|
+
# Build migration chain
|
|
178
|
+
try:
|
|
179
|
+
chain = build_migration_chain(migrations_dir)
|
|
180
|
+
except ValueError as e:
|
|
181
|
+
app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
|
|
182
|
+
raise SystemExit(1)
|
|
183
|
+
|
|
184
|
+
if not chain:
|
|
185
|
+
app.console.print("[yellow]No migrations found[/yellow]")
|
|
186
|
+
app.console.print("Run 'metaxy migrations generate' first")
|
|
187
|
+
return
|
|
188
|
+
|
|
189
|
+
# Get completed migrations from events
|
|
190
|
+
completed_ids = set()
|
|
191
|
+
for mid in [m.migration_id for m in chain]:
|
|
192
|
+
if storage.get_migration_status(mid) == "completed":
|
|
193
|
+
completed_ids.add(mid)
|
|
194
|
+
|
|
195
|
+
# Filter to unapplied migrations
|
|
196
|
+
if migration_id is None:
|
|
197
|
+
# Apply all unapplied
|
|
198
|
+
to_apply = [m for m in chain if m.migration_id not in completed_ids]
|
|
199
|
+
else:
|
|
200
|
+
# Apply specific migration and its unapplied predecessors
|
|
201
|
+
target_index = None
|
|
202
|
+
for i, m in enumerate(chain):
|
|
203
|
+
if m.migration_id == migration_id:
|
|
204
|
+
target_index = i
|
|
205
|
+
break
|
|
206
|
+
|
|
207
|
+
if target_index is None:
|
|
208
|
+
app.console.print(
|
|
209
|
+
f"[red]✗[/red] Migration '{migration_id}' not found in chain"
|
|
210
|
+
)
|
|
211
|
+
raise SystemExit(1)
|
|
212
|
+
|
|
213
|
+
# Include all unapplied migrations up to and including target
|
|
214
|
+
to_apply = [
|
|
215
|
+
m
|
|
216
|
+
for m in chain[: target_index + 1]
|
|
217
|
+
if m.migration_id not in completed_ids
|
|
218
|
+
]
|
|
219
|
+
|
|
220
|
+
if not to_apply:
|
|
221
|
+
app.console.print("[blue]ℹ[/blue] All migrations already completed")
|
|
222
|
+
return
|
|
223
|
+
|
|
224
|
+
# Execute migrations in order
|
|
225
|
+
if dry_run:
|
|
226
|
+
app.console.print("[yellow]=== DRY RUN MODE ===[/yellow]\n")
|
|
227
|
+
|
|
228
|
+
app.console.print(f"Applying {len(to_apply)} migration(s) in chain order:")
|
|
229
|
+
for m in to_apply:
|
|
230
|
+
app.console.print(f" • {m.migration_id}")
|
|
231
|
+
app.console.print()
|
|
232
|
+
|
|
233
|
+
executor = MigrationExecutor(storage)
|
|
234
|
+
|
|
235
|
+
for migration in to_apply:
|
|
236
|
+
app.console.print(f"[bold]Applying: {migration.migration_id}[/bold]")
|
|
237
|
+
app.console.print(
|
|
238
|
+
f" Affecting {len(migration.get_affected_features(metadata_store))} feature(s)"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
result = executor.execute(migration, metadata_store, dry_run=dry_run)
|
|
242
|
+
|
|
243
|
+
# Print result
|
|
244
|
+
if result.status == "completed":
|
|
245
|
+
app.console.print("[green]✓[/green] Migration completed")
|
|
246
|
+
elif result.status == "skipped":
|
|
247
|
+
app.console.print("[yellow]⊘[/yellow] Migration skipped (dry run)")
|
|
248
|
+
else:
|
|
249
|
+
app.console.print("[red]✗[/red] Migration failed")
|
|
250
|
+
|
|
251
|
+
app.console.print(f" Features completed: {result.features_completed}")
|
|
252
|
+
app.console.print(f" Features failed: {result.features_failed}")
|
|
253
|
+
app.console.print(f" Rows affected: {result.rows_affected}")
|
|
254
|
+
app.console.print(f" Duration: {result.duration_seconds:.2f}s")
|
|
255
|
+
|
|
256
|
+
if result.errors:
|
|
257
|
+
app.console.print("\n[red]Errors:[/red]")
|
|
258
|
+
for feature_key, error in result.errors.items():
|
|
259
|
+
app.console.print(f" ✗ {feature_key}: {error}")
|
|
260
|
+
|
|
261
|
+
if result.status == "failed":
|
|
262
|
+
app.console.print(
|
|
263
|
+
f"\n[red]Migration {migration.migration_id} failed. "
|
|
264
|
+
"Stopping chain execution.[/red]"
|
|
265
|
+
)
|
|
266
|
+
raise SystemExit(1)
|
|
267
|
+
|
|
268
|
+
app.console.print() # Blank line between migrations
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
@app.command
|
|
272
|
+
def status():
|
|
273
|
+
"""Show migration chain and execution status.
|
|
274
|
+
|
|
275
|
+
Reads migration definitions from YAML files (git).
|
|
276
|
+
Shows execution status from database events.
|
|
277
|
+
Displays the parent chain in order.
|
|
278
|
+
|
|
279
|
+
Example:
|
|
280
|
+
$ metaxy migrations status
|
|
281
|
+
|
|
282
|
+
Migration Chain:
|
|
283
|
+
────────────────────────────────────────────
|
|
284
|
+
✓ 20250110_120000 (parent: initial)
|
|
285
|
+
Status: COMPLETED
|
|
286
|
+
Features: 5/5 completed
|
|
287
|
+
|
|
288
|
+
○ 20250113_103000 (parent: 20250110_120000)
|
|
289
|
+
Status: NOT STARTED
|
|
290
|
+
Features: 3 affected
|
|
291
|
+
|
|
292
|
+
⚠ Multiple heads detected: [20250110_120000_a, 20250110_120000_b]
|
|
293
|
+
"""
|
|
294
|
+
from pathlib import Path
|
|
295
|
+
|
|
296
|
+
from metaxy.cli.context import get_store
|
|
297
|
+
from metaxy.entrypoints import load_features
|
|
298
|
+
from metaxy.metadata_store.system_tables import SystemTableStorage
|
|
299
|
+
from metaxy.migrations.loader import build_migration_chain
|
|
300
|
+
|
|
301
|
+
# Load features
|
|
302
|
+
load_features()
|
|
303
|
+
|
|
304
|
+
metadata_store = get_store()
|
|
305
|
+
migrations_dir = Path(".metaxy/migrations")
|
|
306
|
+
|
|
307
|
+
with metadata_store:
|
|
308
|
+
storage = SystemTableStorage(metadata_store)
|
|
309
|
+
|
|
310
|
+
# Try to build migration chain
|
|
311
|
+
try:
|
|
312
|
+
chain = build_migration_chain(migrations_dir)
|
|
313
|
+
except ValueError as e:
|
|
314
|
+
app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
|
|
315
|
+
return
|
|
316
|
+
|
|
317
|
+
if not chain:
|
|
318
|
+
app.console.print("[yellow]No migrations found.[/yellow]")
|
|
319
|
+
app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
|
|
320
|
+
return
|
|
321
|
+
|
|
322
|
+
app.console.print("\n[bold]Migration Chain:[/bold]")
|
|
323
|
+
app.console.print("─" * 60)
|
|
324
|
+
|
|
325
|
+
for migration in chain:
|
|
326
|
+
migration_id = migration.migration_id
|
|
327
|
+
|
|
328
|
+
# Get status from events
|
|
329
|
+
status_str = storage.get_migration_status(migration_id)
|
|
330
|
+
|
|
331
|
+
# Get completion counts
|
|
332
|
+
completed_features = storage.get_completed_features(migration_id)
|
|
333
|
+
failed_features = storage.get_failed_features(migration_id)
|
|
334
|
+
|
|
335
|
+
# Compute total affected
|
|
336
|
+
if status_str == "not_started":
|
|
337
|
+
try:
|
|
338
|
+
total_affected = len(
|
|
339
|
+
migration.get_affected_features(metadata_store)
|
|
340
|
+
)
|
|
341
|
+
except Exception:
|
|
342
|
+
total_affected = "?"
|
|
343
|
+
else:
|
|
344
|
+
total_affected = len(completed_features) + len(failed_features)
|
|
345
|
+
|
|
346
|
+
# Print status icon
|
|
347
|
+
if status_str == "completed":
|
|
348
|
+
app.console.print(
|
|
349
|
+
f"[green]✓[/green] {migration_id} (parent: {migration.parent})"
|
|
350
|
+
)
|
|
351
|
+
app.console.print(" Status: [green]COMPLETED[/green]")
|
|
352
|
+
elif status_str == "failed":
|
|
353
|
+
app.console.print(
|
|
354
|
+
f"[red]✗[/red] {migration_id} (parent: {migration.parent})"
|
|
355
|
+
)
|
|
356
|
+
app.console.print(" Status: [red]FAILED[/red]")
|
|
357
|
+
elif status_str == "in_progress":
|
|
358
|
+
app.console.print(
|
|
359
|
+
f"[yellow]⚠[/yellow] {migration_id} (parent: {migration.parent})"
|
|
360
|
+
)
|
|
361
|
+
app.console.print(" Status: [yellow]IN PROGRESS[/yellow]")
|
|
362
|
+
else:
|
|
363
|
+
app.console.print(
|
|
364
|
+
f"[blue]○[/blue] {migration_id} (parent: {migration.parent})"
|
|
365
|
+
)
|
|
366
|
+
app.console.print(" Status: [blue]NOT STARTED[/blue]")
|
|
367
|
+
|
|
368
|
+
app.console.print(" Snapshots:")
|
|
369
|
+
app.console.print(f" From: {migration.from_snapshot_version}")
|
|
370
|
+
app.console.print(f" To: {migration.to_snapshot_version}")
|
|
371
|
+
app.console.print(
|
|
372
|
+
f" Features: {len(completed_features)}/{total_affected} completed"
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
if failed_features:
|
|
376
|
+
app.console.print(
|
|
377
|
+
f" [red]Failed features ({len(failed_features)}):[/red]"
|
|
378
|
+
)
|
|
379
|
+
for feature_key, error in list(failed_features.items())[:3]:
|
|
380
|
+
app.console.print(f" ✗ {feature_key}: {error}")
|
|
381
|
+
|
|
382
|
+
app.console.print()
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
@app.command(name="list")
|
|
386
|
+
def list_migrations():
|
|
387
|
+
"""List all migrations in chain order.
|
|
388
|
+
|
|
389
|
+
Displays a simple table showing migration ID, creation time, and operations.
|
|
390
|
+
|
|
391
|
+
Example:
|
|
392
|
+
$ metaxy migrations list
|
|
393
|
+
|
|
394
|
+
20250110_120000 2025-01-10 12:00 DataVersionReconciliation
|
|
395
|
+
20250113_103000 2025-01-13 10:30 DataVersionReconciliation
|
|
396
|
+
"""
|
|
397
|
+
from pathlib import Path
|
|
398
|
+
|
|
399
|
+
from rich.table import Table
|
|
400
|
+
|
|
401
|
+
from metaxy.cli.context import get_store
|
|
402
|
+
from metaxy.entrypoints import load_features
|
|
403
|
+
from metaxy.migrations.loader import build_migration_chain
|
|
404
|
+
|
|
405
|
+
# Load features
|
|
406
|
+
load_features()
|
|
407
|
+
|
|
408
|
+
metadata_store = get_store()
|
|
409
|
+
migrations_dir = Path(".metaxy/migrations")
|
|
410
|
+
|
|
411
|
+
with metadata_store:
|
|
412
|
+
# Build migration chain
|
|
413
|
+
try:
|
|
414
|
+
chain = build_migration_chain(migrations_dir)
|
|
415
|
+
except ValueError as e:
|
|
416
|
+
app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
|
|
417
|
+
return
|
|
418
|
+
|
|
419
|
+
if not chain:
|
|
420
|
+
app.console.print("[yellow]No migrations found.[/yellow]")
|
|
421
|
+
app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
|
|
422
|
+
return
|
|
423
|
+
|
|
424
|
+
# Create borderless table with blue headers (no truncation)
|
|
425
|
+
table = Table(
|
|
426
|
+
show_header=True,
|
|
427
|
+
show_edge=False,
|
|
428
|
+
box=None,
|
|
429
|
+
padding=(0, 2),
|
|
430
|
+
header_style="bold blue",
|
|
431
|
+
)
|
|
432
|
+
table.add_column("ID", style="bold", no_wrap=False, overflow="fold")
|
|
433
|
+
table.add_column("Created", style="dim", no_wrap=False, overflow="fold")
|
|
434
|
+
table.add_column("Operations", no_wrap=False, overflow="fold")
|
|
435
|
+
|
|
436
|
+
for migration in chain:
|
|
437
|
+
# Format created_at - simpler format without seconds
|
|
438
|
+
created_str = migration.created_at.strftime("%Y-%m-%d %H:%M")
|
|
439
|
+
|
|
440
|
+
# Format operations - extract short names
|
|
441
|
+
op_names = []
|
|
442
|
+
for op in migration.ops:
|
|
443
|
+
op_type = op.get("type", "unknown")
|
|
444
|
+
# Extract just the class name (last part after final dot)
|
|
445
|
+
op_short = op_type.split(".")[-1]
|
|
446
|
+
op_names.append(op_short)
|
|
447
|
+
|
|
448
|
+
ops_str = ", ".join(op_names)
|
|
449
|
+
|
|
450
|
+
table.add_row(migration.migration_id, created_str, ops_str)
|
|
451
|
+
|
|
452
|
+
app.console.print()
|
|
453
|
+
app.console.print(table)
|
|
454
|
+
app.console.print()
|
|
455
|
+
|
|
456
|
+
|
|
457
|
+
@app.command
|
|
458
|
+
def explain(
|
|
459
|
+
migration_id: Annotated[
|
|
460
|
+
str | None,
|
|
461
|
+
cyclopts.Parameter(
|
|
462
|
+
help="Migration ID to explain (explains latest if not specified)"
|
|
463
|
+
),
|
|
464
|
+
] = None,
|
|
465
|
+
):
|
|
466
|
+
"""Show detailed diff for a migration.
|
|
467
|
+
|
|
468
|
+
Reads migration from YAML file.
|
|
469
|
+
Computes and displays the GraphDiff between the two snapshots on-demand.
|
|
470
|
+
|
|
471
|
+
Examples:
|
|
472
|
+
# Explain latest migration (head of chain)
|
|
473
|
+
$ metaxy migrations explain
|
|
474
|
+
|
|
475
|
+
# Explain specific migration
|
|
476
|
+
$ metaxy migrations explain 20250113_103000
|
|
477
|
+
"""
|
|
478
|
+
from pathlib import Path
|
|
479
|
+
|
|
480
|
+
from metaxy.cli.context import get_store
|
|
481
|
+
from metaxy.entrypoints import load_features
|
|
482
|
+
from metaxy.migrations.loader import (
|
|
483
|
+
find_latest_migration,
|
|
484
|
+
find_migration_yaml,
|
|
485
|
+
load_migration_from_yaml,
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
# Load features
|
|
489
|
+
load_features()
|
|
490
|
+
|
|
491
|
+
metadata_store = get_store()
|
|
492
|
+
migrations_dir = Path(".metaxy/migrations")
|
|
493
|
+
|
|
494
|
+
with metadata_store:
|
|
495
|
+
# Get migration ID
|
|
496
|
+
if migration_id is None:
|
|
497
|
+
# Get latest migration (head)
|
|
498
|
+
try:
|
|
499
|
+
migration_id = find_latest_migration(migrations_dir)
|
|
500
|
+
except ValueError as e:
|
|
501
|
+
app.console.print(f"[red]✗[/red] {e}")
|
|
502
|
+
raise SystemExit(1)
|
|
503
|
+
|
|
504
|
+
if migration_id is None:
|
|
505
|
+
app.console.print("[yellow]No migrations found[/yellow]")
|
|
506
|
+
app.console.print("Run 'metaxy migrations generate' first")
|
|
507
|
+
return
|
|
508
|
+
|
|
509
|
+
# Load migration from YAML
|
|
510
|
+
try:
|
|
511
|
+
yaml_path = find_migration_yaml(migration_id, migrations_dir)
|
|
512
|
+
migration = load_migration_from_yaml(yaml_path)
|
|
513
|
+
except FileNotFoundError as e:
|
|
514
|
+
app.console.print(f"[red]✗[/red] {e}")
|
|
515
|
+
raise SystemExit(1)
|
|
516
|
+
|
|
517
|
+
# Type narrow to DiffMigration for explain command
|
|
518
|
+
from metaxy.migrations.models import DiffMigration
|
|
519
|
+
|
|
520
|
+
if not isinstance(migration, DiffMigration):
|
|
521
|
+
app.console.print(
|
|
522
|
+
f"[red]✗[/red] Migration '{migration_id}' is not a DiffMigration"
|
|
523
|
+
)
|
|
524
|
+
app.console.print(
|
|
525
|
+
f" Type: {type(migration).__name__} (explain only supports DiffMigration)"
|
|
526
|
+
)
|
|
527
|
+
raise SystemExit(1)
|
|
528
|
+
|
|
529
|
+
# Print header
|
|
530
|
+
app.console.print(f"\n[bold]Migration: {migration_id}[/bold]")
|
|
531
|
+
app.console.print(f"From: {migration.from_snapshot_version}")
|
|
532
|
+
app.console.print(f"To: {migration.to_snapshot_version}")
|
|
533
|
+
|
|
534
|
+
# Get description (computed on-demand)
|
|
535
|
+
description = migration.get_description(metadata_store)
|
|
536
|
+
app.console.print(f"Description: {description}")
|
|
537
|
+
app.console.print()
|
|
538
|
+
|
|
539
|
+
# Compute diff on-demand
|
|
540
|
+
try:
|
|
541
|
+
graph_diff = migration.compute_graph_diff(metadata_store)
|
|
542
|
+
except Exception as e:
|
|
543
|
+
app.console.print(f"[red]✗[/red] Failed to compute diff: {e}")
|
|
544
|
+
raise SystemExit(1)
|
|
545
|
+
|
|
546
|
+
# Display detailed diff
|
|
547
|
+
if not graph_diff.has_changes:
|
|
548
|
+
app.console.print("[yellow]No changes detected[/yellow]")
|
|
549
|
+
return
|
|
550
|
+
|
|
551
|
+
# Added nodes
|
|
552
|
+
if graph_diff.added_nodes:
|
|
553
|
+
app.console.print(
|
|
554
|
+
f"[green]Added Features ({len(graph_diff.added_nodes)}):[/green]"
|
|
555
|
+
)
|
|
556
|
+
for node in graph_diff.added_nodes:
|
|
557
|
+
app.console.print(f" ✓ {node.feature_key}")
|
|
558
|
+
if node.fields:
|
|
559
|
+
app.console.print(f" Fields ({len(node.fields)}):")
|
|
560
|
+
for field in node.fields[:3]:
|
|
561
|
+
app.console.print(
|
|
562
|
+
f" - {field['key']} (cv={field.get('code_version', '?')})"
|
|
563
|
+
)
|
|
564
|
+
if len(node.fields) > 3:
|
|
565
|
+
app.console.print(f" ... and {len(node.fields) - 3} more")
|
|
566
|
+
app.console.print()
|
|
567
|
+
|
|
568
|
+
# Removed nodes
|
|
569
|
+
if graph_diff.removed_nodes:
|
|
570
|
+
app.console.print(
|
|
571
|
+
f"[red]Removed Features ({len(graph_diff.removed_nodes)}):[/red]"
|
|
572
|
+
)
|
|
573
|
+
for node in graph_diff.removed_nodes:
|
|
574
|
+
app.console.print(f" ✗ {node.feature_key}")
|
|
575
|
+
if node.fields:
|
|
576
|
+
app.console.print(f" Fields ({len(node.fields)}):")
|
|
577
|
+
for field in node.fields[:3]:
|
|
578
|
+
app.console.print(
|
|
579
|
+
f" - {field['key']} (cv={field.get('code_version', '?')})"
|
|
580
|
+
)
|
|
581
|
+
if len(node.fields) > 3:
|
|
582
|
+
app.console.print(f" ... and {len(node.fields) - 3} more")
|
|
583
|
+
app.console.print()
|
|
584
|
+
|
|
585
|
+
# Changed nodes
|
|
586
|
+
if graph_diff.changed_nodes:
|
|
587
|
+
app.console.print(
|
|
588
|
+
f"[yellow]Changed Features ({len(graph_diff.changed_nodes)}):[/yellow]"
|
|
589
|
+
)
|
|
590
|
+
for node in graph_diff.changed_nodes:
|
|
591
|
+
app.console.print(f" ⚠ {node.feature_key}")
|
|
592
|
+
old_ver = node.old_version if node.old_version else "None"
|
|
593
|
+
new_ver = node.new_version if node.new_version else "None"
|
|
594
|
+
app.console.print(f" Version: {old_ver} → {new_ver}")
|
|
595
|
+
|
|
596
|
+
if (
|
|
597
|
+
node.old_code_version is not None
|
|
598
|
+
or node.new_code_version is not None
|
|
599
|
+
):
|
|
600
|
+
app.console.print(
|
|
601
|
+
f" Code version: {node.old_code_version} → {node.new_code_version}"
|
|
602
|
+
)
|
|
603
|
+
|
|
604
|
+
# Show field changes
|
|
605
|
+
total_field_changes = (
|
|
606
|
+
len(node.added_fields)
|
|
607
|
+
+ len(node.removed_fields)
|
|
608
|
+
+ len(node.changed_fields)
|
|
609
|
+
)
|
|
610
|
+
if total_field_changes > 0:
|
|
611
|
+
app.console.print(f" Field changes ({total_field_changes}):")
|
|
612
|
+
|
|
613
|
+
if node.added_fields:
|
|
614
|
+
app.console.print(
|
|
615
|
+
f" [green]Added ({len(node.added_fields)}):[/green]"
|
|
616
|
+
)
|
|
617
|
+
for field in node.added_fields[:2]:
|
|
618
|
+
app.console.print(
|
|
619
|
+
f" + {field.field_key} (cv={field.new_code_version})"
|
|
620
|
+
)
|
|
621
|
+
if len(node.added_fields) > 2:
|
|
622
|
+
app.console.print(
|
|
623
|
+
f" ... and {len(node.added_fields) - 2} more"
|
|
624
|
+
)
|
|
625
|
+
|
|
626
|
+
if node.removed_fields:
|
|
627
|
+
app.console.print(
|
|
628
|
+
f" [red]Removed ({len(node.removed_fields)}):[/red]"
|
|
629
|
+
)
|
|
630
|
+
for field in node.removed_fields[:2]:
|
|
631
|
+
app.console.print(
|
|
632
|
+
f" - {field.field_key} (cv={field.old_code_version})"
|
|
633
|
+
)
|
|
634
|
+
if len(node.removed_fields) > 2:
|
|
635
|
+
app.console.print(
|
|
636
|
+
f" ... and {len(node.removed_fields) - 2} more"
|
|
637
|
+
)
|
|
638
|
+
|
|
639
|
+
if node.changed_fields:
|
|
640
|
+
app.console.print(
|
|
641
|
+
f" [yellow]Changed ({len(node.changed_fields)}):[/yellow]"
|
|
642
|
+
)
|
|
643
|
+
for field in node.changed_fields[:2]:
|
|
644
|
+
app.console.print(
|
|
645
|
+
f" ~ {field.field_key} (cv={field.old_code_version}→{field.new_code_version})"
|
|
646
|
+
)
|
|
647
|
+
if len(node.changed_fields) > 2:
|
|
648
|
+
app.console.print(
|
|
649
|
+
f" ... and {len(node.changed_fields) - 2} more"
|
|
650
|
+
)
|
|
651
|
+
|
|
652
|
+
app.console.print()
|
|
653
|
+
|
|
654
|
+
# Print affected features summary (computed on-demand)
|
|
655
|
+
affected_features = migration.get_affected_features(metadata_store)
|
|
656
|
+
app.console.print(f"[bold]Affected Features ({len(affected_features)}):[/bold]")
|
|
657
|
+
for feature_key in affected_features[:10]:
|
|
658
|
+
app.console.print(f" • {feature_key}")
|
|
659
|
+
if len(affected_features) > 10:
|
|
660
|
+
app.console.print(f" ... and {len(affected_features) - 10} more")
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
@app.command
|
|
664
|
+
def describe(
|
|
665
|
+
migration_ids: Annotated[
|
|
666
|
+
list[str],
|
|
667
|
+
cyclopts.Parameter(
|
|
668
|
+
help="Migration IDs to describe (default: all migrations in order)"
|
|
669
|
+
),
|
|
670
|
+
] = [],
|
|
671
|
+
):
|
|
672
|
+
"""Show verbose description of migration(s).
|
|
673
|
+
|
|
674
|
+
Displays detailed information about what the migration will do:
|
|
675
|
+
- Migration metadata (ID, parent, snapshots, created timestamp)
|
|
676
|
+
- Operations to execute
|
|
677
|
+
- Affected features with row counts
|
|
678
|
+
- Execution status if already run
|
|
679
|
+
|
|
680
|
+
Examples:
|
|
681
|
+
# Describe all migrations in chain order
|
|
682
|
+
$ metaxy migrations describe
|
|
683
|
+
|
|
684
|
+
# Describe specific migration
|
|
685
|
+
$ metaxy migrations describe 20250127_120000
|
|
686
|
+
|
|
687
|
+
# Describe multiple migrations
|
|
688
|
+
$ metaxy migrations describe 20250101_120000 20250102_090000
|
|
689
|
+
"""
|
|
690
|
+
from pathlib import Path
|
|
691
|
+
|
|
692
|
+
from metaxy.cli.context import get_store
|
|
693
|
+
from metaxy.entrypoints import load_features
|
|
694
|
+
from metaxy.metadata_store.system_tables import SystemTableStorage
|
|
695
|
+
from metaxy.migrations.loader import (
|
|
696
|
+
build_migration_chain,
|
|
697
|
+
find_migration_yaml,
|
|
698
|
+
load_migration_from_yaml,
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
# Load features
|
|
702
|
+
load_features()
|
|
703
|
+
|
|
704
|
+
metadata_store = get_store()
|
|
705
|
+
migrations_dir = Path(".metaxy/migrations")
|
|
706
|
+
|
|
707
|
+
with metadata_store:
|
|
708
|
+
storage = SystemTableStorage(metadata_store)
|
|
709
|
+
|
|
710
|
+
# Determine which migrations to describe
|
|
711
|
+
if not migration_ids:
|
|
712
|
+
# Default: describe all migrations in chain order
|
|
713
|
+
try:
|
|
714
|
+
chain = build_migration_chain(migrations_dir)
|
|
715
|
+
except ValueError as e:
|
|
716
|
+
app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
|
|
717
|
+
raise SystemExit(1)
|
|
718
|
+
|
|
719
|
+
if not chain:
|
|
720
|
+
app.console.print("[yellow]No migrations found.[/yellow]")
|
|
721
|
+
return
|
|
722
|
+
|
|
723
|
+
migrations_to_describe = chain
|
|
724
|
+
else:
|
|
725
|
+
# Load specific migrations
|
|
726
|
+
migrations_to_describe = []
|
|
727
|
+
for migration_id in migration_ids:
|
|
728
|
+
try:
|
|
729
|
+
yaml_path = find_migration_yaml(migration_id, migrations_dir)
|
|
730
|
+
migration_obj = load_migration_from_yaml(yaml_path)
|
|
731
|
+
migrations_to_describe.append(migration_obj)
|
|
732
|
+
except FileNotFoundError as e:
|
|
733
|
+
app.console.print(f"[red]✗[/red] {e}")
|
|
734
|
+
raise SystemExit(1)
|
|
735
|
+
|
|
736
|
+
# Describe each migration
|
|
737
|
+
for i, migration_obj in enumerate(migrations_to_describe):
|
|
738
|
+
if i > 0:
|
|
739
|
+
app.console.print("\n") # Separator between migrations
|
|
740
|
+
|
|
741
|
+
# Find YAML path for display
|
|
742
|
+
yaml_path = find_migration_yaml(migration_obj.migration_id, migrations_dir)
|
|
743
|
+
|
|
744
|
+
# Print header
|
|
745
|
+
app.console.print("\n[bold]Migration Description[/bold]")
|
|
746
|
+
app.console.print("─" * 60)
|
|
747
|
+
app.console.print(f"[bold]ID:[/bold] {migration_obj.migration_id}")
|
|
748
|
+
app.console.print(
|
|
749
|
+
f"[bold]Created:[/bold] {migration_obj.created_at.isoformat()}"
|
|
750
|
+
)
|
|
751
|
+
app.console.print(f"[bold]Parent:[/bold] {migration_obj.parent}")
|
|
752
|
+
app.console.print(f"[bold]YAML:[/bold] {yaml_path}")
|
|
753
|
+
app.console.print()
|
|
754
|
+
|
|
755
|
+
# Snapshots
|
|
756
|
+
app.console.print("[bold]Snapshots:[/bold]")
|
|
757
|
+
app.console.print(f" From: {migration_obj.from_snapshot_version}")
|
|
758
|
+
app.console.print(f" To: {migration_obj.to_snapshot_version}")
|
|
759
|
+
app.console.print()
|
|
760
|
+
|
|
761
|
+
# Operations
|
|
762
|
+
app.console.print("[bold]Operations:[/bold]")
|
|
763
|
+
for j, op in enumerate(migration_obj.ops, 1):
|
|
764
|
+
op_type = op.get("type", "unknown")
|
|
765
|
+
app.console.print(f" {j}. {op_type}")
|
|
766
|
+
app.console.print()
|
|
767
|
+
|
|
768
|
+
# Get affected features
|
|
769
|
+
app.console.print("[bold]Computing affected features...[/bold]")
|
|
770
|
+
try:
|
|
771
|
+
affected_features = migration_obj.get_affected_features(metadata_store)
|
|
772
|
+
except Exception as e:
|
|
773
|
+
app.console.print(
|
|
774
|
+
f"[red]✗[/red] Failed to compute affected features: {e}"
|
|
775
|
+
)
|
|
776
|
+
continue # Skip to next migration
|
|
777
|
+
|
|
778
|
+
app.console.print(
|
|
779
|
+
f"\n[bold]Affected Features ({len(affected_features)}):[/bold]"
|
|
780
|
+
)
|
|
781
|
+
app.console.print("─" * 60)
|
|
782
|
+
|
|
783
|
+
from metaxy.models.feature import FeatureGraph
|
|
784
|
+
from metaxy.models.types import FeatureKey
|
|
785
|
+
|
|
786
|
+
graph = FeatureGraph.get_active()
|
|
787
|
+
|
|
788
|
+
for feature_key_str in affected_features:
|
|
789
|
+
feature_key_obj = FeatureKey(feature_key_str.split("/"))
|
|
790
|
+
|
|
791
|
+
# Get feature class
|
|
792
|
+
if feature_key_obj not in graph.features_by_key:
|
|
793
|
+
app.console.print(f"[yellow]⚠[/yellow] {feature_key_str}")
|
|
794
|
+
app.console.print(
|
|
795
|
+
" [yellow]Feature not in current graph[/yellow]"
|
|
796
|
+
)
|
|
797
|
+
continue
|
|
798
|
+
|
|
799
|
+
feature_cls = graph.features_by_key[feature_key_obj]
|
|
800
|
+
|
|
801
|
+
# Get current row count
|
|
802
|
+
try:
|
|
803
|
+
import narwhals as nw
|
|
804
|
+
|
|
805
|
+
metadata = metadata_store.read_metadata(
|
|
806
|
+
feature_cls,
|
|
807
|
+
current_only=False,
|
|
808
|
+
allow_fallback=False,
|
|
809
|
+
)
|
|
810
|
+
row_count = (
|
|
811
|
+
metadata.select(nw.col("sample_uid").unique())
|
|
812
|
+
.collect()
|
|
813
|
+
.shape[0]
|
|
814
|
+
)
|
|
815
|
+
except Exception:
|
|
816
|
+
row_count = "?"
|
|
817
|
+
|
|
818
|
+
# Check if feature has upstream dependencies
|
|
819
|
+
plan = graph.get_feature_plan(feature_key_obj)
|
|
820
|
+
has_upstream = plan.deps is not None and len(plan.deps) > 0
|
|
821
|
+
|
|
822
|
+
app.console.print(f"[bold]{feature_key_str}[/bold]")
|
|
823
|
+
app.console.print(f" Samples: {row_count}")
|
|
824
|
+
app.console.print(f" Has upstream: {has_upstream}")
|
|
825
|
+
|
|
826
|
+
app.console.print()
|
|
827
|
+
|
|
828
|
+
# Execution status
|
|
829
|
+
status_str = storage.get_migration_status(migration_obj.migration_id)
|
|
830
|
+
completed_features = storage.get_completed_features(
|
|
831
|
+
migration_obj.migration_id
|
|
832
|
+
)
|
|
833
|
+
failed_features = storage.get_failed_features(migration_obj.migration_id)
|
|
834
|
+
|
|
835
|
+
app.console.print("[bold]Execution Status:[/bold]")
|
|
836
|
+
if status_str == "completed":
|
|
837
|
+
app.console.print(" [green]✓ COMPLETED[/green]")
|
|
838
|
+
app.console.print(
|
|
839
|
+
f" Features processed: {len(completed_features)}/{len(affected_features)}"
|
|
840
|
+
)
|
|
841
|
+
elif status_str == "failed":
|
|
842
|
+
app.console.print(" [red]✗ FAILED[/red]")
|
|
843
|
+
app.console.print(
|
|
844
|
+
f" Features completed: {len(completed_features)}/{len(affected_features)}"
|
|
845
|
+
)
|
|
846
|
+
app.console.print(f" Features failed: {len(failed_features)}")
|
|
847
|
+
if failed_features:
|
|
848
|
+
app.console.print(" Failed features:")
|
|
849
|
+
for feature_key, error in list(failed_features.items())[:5]:
|
|
850
|
+
app.console.print(f" • {feature_key}: {error}")
|
|
851
|
+
elif status_str == "in_progress":
|
|
852
|
+
app.console.print(" [yellow]⚠ IN PROGRESS[/yellow]")
|
|
853
|
+
app.console.print(
|
|
854
|
+
f" Features completed: {len(completed_features)}/{len(affected_features)}"
|
|
855
|
+
)
|
|
856
|
+
else:
|
|
857
|
+
app.console.print(" [blue]○ NOT STARTED[/blue]")
|
|
858
|
+
app.console.print(f" Features to process: {len(affected_features)}")
|
|
859
|
+
|
|
860
|
+
|
|
861
|
+
if __name__ == "__main__":
|
|
862
|
+
app()
|