metaxy 0.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaxy might be problematic. Click here for more details.

Files changed (75) hide show
  1. metaxy/__init__.py +61 -0
  2. metaxy/_testing.py +542 -0
  3. metaxy/_utils.py +16 -0
  4. metaxy/_version.py +1 -0
  5. metaxy/cli/app.py +76 -0
  6. metaxy/cli/context.py +71 -0
  7. metaxy/cli/graph.py +576 -0
  8. metaxy/cli/graph_diff.py +290 -0
  9. metaxy/cli/list.py +42 -0
  10. metaxy/cli/metadata.py +271 -0
  11. metaxy/cli/migrations.py +862 -0
  12. metaxy/cli/push.py +55 -0
  13. metaxy/config.py +450 -0
  14. metaxy/data_versioning/__init__.py +24 -0
  15. metaxy/data_versioning/calculators/__init__.py +13 -0
  16. metaxy/data_versioning/calculators/base.py +97 -0
  17. metaxy/data_versioning/calculators/duckdb.py +186 -0
  18. metaxy/data_versioning/calculators/ibis.py +225 -0
  19. metaxy/data_versioning/calculators/polars.py +135 -0
  20. metaxy/data_versioning/diff/__init__.py +15 -0
  21. metaxy/data_versioning/diff/base.py +150 -0
  22. metaxy/data_versioning/diff/narwhals.py +108 -0
  23. metaxy/data_versioning/hash_algorithms.py +19 -0
  24. metaxy/data_versioning/joiners/__init__.py +9 -0
  25. metaxy/data_versioning/joiners/base.py +70 -0
  26. metaxy/data_versioning/joiners/narwhals.py +235 -0
  27. metaxy/entrypoints.py +309 -0
  28. metaxy/ext/__init__.py +1 -0
  29. metaxy/ext/alembic.py +326 -0
  30. metaxy/ext/sqlmodel.py +172 -0
  31. metaxy/ext/sqlmodel_system_tables.py +139 -0
  32. metaxy/graph/__init__.py +21 -0
  33. metaxy/graph/diff/__init__.py +21 -0
  34. metaxy/graph/diff/diff_models.py +399 -0
  35. metaxy/graph/diff/differ.py +740 -0
  36. metaxy/graph/diff/models.py +418 -0
  37. metaxy/graph/diff/rendering/__init__.py +18 -0
  38. metaxy/graph/diff/rendering/base.py +274 -0
  39. metaxy/graph/diff/rendering/cards.py +188 -0
  40. metaxy/graph/diff/rendering/formatter.py +805 -0
  41. metaxy/graph/diff/rendering/graphviz.py +246 -0
  42. metaxy/graph/diff/rendering/mermaid.py +320 -0
  43. metaxy/graph/diff/rendering/rich.py +165 -0
  44. metaxy/graph/diff/rendering/theme.py +48 -0
  45. metaxy/graph/diff/traversal.py +247 -0
  46. metaxy/graph/utils.py +58 -0
  47. metaxy/metadata_store/__init__.py +31 -0
  48. metaxy/metadata_store/_protocols.py +38 -0
  49. metaxy/metadata_store/base.py +1676 -0
  50. metaxy/metadata_store/clickhouse.py +161 -0
  51. metaxy/metadata_store/duckdb.py +167 -0
  52. metaxy/metadata_store/exceptions.py +43 -0
  53. metaxy/metadata_store/ibis.py +451 -0
  54. metaxy/metadata_store/memory.py +228 -0
  55. metaxy/metadata_store/sqlite.py +187 -0
  56. metaxy/metadata_store/system_tables.py +257 -0
  57. metaxy/migrations/__init__.py +34 -0
  58. metaxy/migrations/detector.py +153 -0
  59. metaxy/migrations/executor.py +208 -0
  60. metaxy/migrations/loader.py +260 -0
  61. metaxy/migrations/models.py +718 -0
  62. metaxy/migrations/ops.py +390 -0
  63. metaxy/models/__init__.py +0 -0
  64. metaxy/models/bases.py +6 -0
  65. metaxy/models/constants.py +24 -0
  66. metaxy/models/feature.py +665 -0
  67. metaxy/models/feature_spec.py +105 -0
  68. metaxy/models/field.py +25 -0
  69. metaxy/models/plan.py +155 -0
  70. metaxy/models/types.py +157 -0
  71. metaxy/py.typed +0 -0
  72. metaxy-0.0.0.dist-info/METADATA +247 -0
  73. metaxy-0.0.0.dist-info/RECORD +75 -0
  74. metaxy-0.0.0.dist-info/WHEEL +4 -0
  75. metaxy-0.0.0.dist-info/entry_points.txt +3 -0
@@ -0,0 +1,862 @@
1
+ """New Migration CLI commands using event-based system."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Annotated
6
+
7
+ import cyclopts
8
+ from rich.console import Console
9
+
10
+ # Rich console for formatted output
11
+ console = Console()
12
+
13
+ # Migrations subcommand app
14
+ app = cyclopts.App(
15
+ name="migrations", # pyrefly: ignore[unexpected-keyword]
16
+ help="Metadata migration commands", # pyrefly: ignore[unexpected-keyword]
17
+ console=console, # pyrefly: ignore[unexpected-keyword]
18
+ )
19
+
20
+
21
+ @app.command
22
+ def generate(
23
+ *,
24
+ name: Annotated[
25
+ str | None,
26
+ cyclopts.Parameter(help="Migration name (creates {timestamp}_{name} ID)"),
27
+ ] = None,
28
+ store: Annotated[
29
+ str | None,
30
+ cyclopts.Parameter(help="Store name (defaults to default)"),
31
+ ] = None,
32
+ from_snapshot: Annotated[
33
+ str | None,
34
+ cyclopts.Parameter(
35
+ help="Compare from this historical snapshot version (defaults to latest)"
36
+ ),
37
+ ] = None,
38
+ op: Annotated[
39
+ list[str],
40
+ cyclopts.Parameter(
41
+ help="Operation class path to use (can be repeated). Example: metaxy.migrations.ops.DataVersionReconciliation"
42
+ ),
43
+ ],
44
+ ):
45
+ """Generate migration from detected feature changes.
46
+
47
+ Compares the latest snapshot in the store (or specified from_snapshot)
48
+ with the current active graph to detect changes.
49
+
50
+ The migration is recorded in the system tables (not a YAML file).
51
+
52
+ Examples:
53
+ # Generate with DataVersionReconciliation operation
54
+ $ metaxy migrations generate --op metaxy.migrations.ops.DataVersionReconciliation
55
+
56
+ # Custom operation type
57
+ $ metaxy migrations generate --op myproject.ops.CustomReconciliation
58
+
59
+ # Multiple operations (future use)
60
+ $ metaxy migrations generate \
61
+ --op metaxy.migrations.ops.DataVersionReconciliation \
62
+ --op myproject.ops.CustomBackfill
63
+ """
64
+ from pathlib import Path
65
+
66
+ from metaxy.cli.context import get_config
67
+ from metaxy.entrypoints import load_features
68
+ from metaxy.migrations.detector import detect_migration
69
+
70
+ # Load features from entrypoints
71
+ load_features()
72
+
73
+ # Convert op_type list to ops format
74
+ if len(op) == 0:
75
+ app.console.print(
76
+ "[red]✗[/red] --op is required. "
77
+ "Example: --op metaxy.migrations.ops.DataVersionReconciliation"
78
+ )
79
+ raise SystemExit(1)
80
+
81
+ ops = [{"type": op} for op in op]
82
+
83
+ # Get store
84
+ config = get_config()
85
+ metadata_store = config.get_store(store)
86
+ migrations_dir = Path(config.migrations_dir)
87
+
88
+ with metadata_store:
89
+ # Detect migration and write YAML
90
+ migration = detect_migration(
91
+ metadata_store,
92
+ from_snapshot_version=from_snapshot,
93
+ ops=ops,
94
+ migrations_dir=migrations_dir,
95
+ name=name,
96
+ )
97
+
98
+ if migration is None:
99
+ app.console.print("[yellow]No changes detected[/yellow]")
100
+ app.console.print(" Current graph matches latest snapshot")
101
+ return
102
+
103
+ # Print summary
104
+ yaml_path = migrations_dir / f"{migration.migration_id}.yaml"
105
+ app.console.print("\n[green]✓[/green] Migration generated")
106
+ app.console.print(f" Migration ID: {migration.migration_id}")
107
+ app.console.print(f" YAML file: {yaml_path}")
108
+ app.console.print(f" From snapshot: {migration.from_snapshot_version}")
109
+ app.console.print(f" To snapshot: {migration.to_snapshot_version}")
110
+
111
+ # Show description
112
+ description = migration.get_description(metadata_store)
113
+ app.console.print(f" Description: {description}")
114
+
115
+ # Get affected features (computed on-demand)
116
+ affected_features = migration.get_affected_features(metadata_store)
117
+ app.console.print(f"\n Affected features ({len(affected_features)}):")
118
+ for feature_key in affected_features[:5]:
119
+ app.console.print(f" ✓ {feature_key}")
120
+ if len(affected_features) > 5:
121
+ app.console.print(f" ... and {len(affected_features) - 5} more")
122
+
123
+ app.console.print("\n[bold]NEXT STEPS:[/bold]")
124
+ app.console.print(f"1. Review migration YAML: {yaml_path}")
125
+ app.console.print(
126
+ f"2. Run 'metaxy migrations apply {migration.migration_id}' to execute. Use CD for production!"
127
+ )
128
+
129
+
130
+ @app.command
131
+ def apply(
132
+ migration_id: Annotated[
133
+ str | None,
134
+ cyclopts.Parameter(
135
+ help="Migration ID to apply (applies all unapplied if not specified)"
136
+ ),
137
+ ] = None,
138
+ *,
139
+ dry_run: Annotated[
140
+ bool,
141
+ cyclopts.Parameter(help="Preview changes without executing"),
142
+ ] = False,
143
+ ):
144
+ """Apply migration(s) from YAML files.
145
+
146
+ Reads migration definitions from .metaxy/migrations/ directory (git).
147
+ Follows parent chain to ensure correct order.
148
+ Tracks execution state in database (events).
149
+
150
+ Examples:
151
+ # Apply all unapplied migrations in chain order
152
+ $ metaxy migrations apply
153
+
154
+ # Apply specific migration (and all its unapplied predecessors)
155
+ $ metaxy migrations apply 20250113_103000
156
+
157
+ # Dry run
158
+ $ metaxy migrations apply --dry-run
159
+ """
160
+ from pathlib import Path
161
+
162
+ from metaxy.cli.context import get_store
163
+ from metaxy.entrypoints import load_features
164
+ from metaxy.metadata_store.system_tables import SystemTableStorage
165
+ from metaxy.migrations.executor import MigrationExecutor
166
+ from metaxy.migrations.loader import build_migration_chain
167
+
168
+ # Load features from entrypoints
169
+ load_features()
170
+
171
+ metadata_store = get_store()
172
+ migrations_dir = Path(".metaxy/migrations")
173
+
174
+ with metadata_store:
175
+ storage = SystemTableStorage(metadata_store)
176
+
177
+ # Build migration chain
178
+ try:
179
+ chain = build_migration_chain(migrations_dir)
180
+ except ValueError as e:
181
+ app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
182
+ raise SystemExit(1)
183
+
184
+ if not chain:
185
+ app.console.print("[yellow]No migrations found[/yellow]")
186
+ app.console.print("Run 'metaxy migrations generate' first")
187
+ return
188
+
189
+ # Get completed migrations from events
190
+ completed_ids = set()
191
+ for mid in [m.migration_id for m in chain]:
192
+ if storage.get_migration_status(mid) == "completed":
193
+ completed_ids.add(mid)
194
+
195
+ # Filter to unapplied migrations
196
+ if migration_id is None:
197
+ # Apply all unapplied
198
+ to_apply = [m for m in chain if m.migration_id not in completed_ids]
199
+ else:
200
+ # Apply specific migration and its unapplied predecessors
201
+ target_index = None
202
+ for i, m in enumerate(chain):
203
+ if m.migration_id == migration_id:
204
+ target_index = i
205
+ break
206
+
207
+ if target_index is None:
208
+ app.console.print(
209
+ f"[red]✗[/red] Migration '{migration_id}' not found in chain"
210
+ )
211
+ raise SystemExit(1)
212
+
213
+ # Include all unapplied migrations up to and including target
214
+ to_apply = [
215
+ m
216
+ for m in chain[: target_index + 1]
217
+ if m.migration_id not in completed_ids
218
+ ]
219
+
220
+ if not to_apply:
221
+ app.console.print("[blue]ℹ[/blue] All migrations already completed")
222
+ return
223
+
224
+ # Execute migrations in order
225
+ if dry_run:
226
+ app.console.print("[yellow]=== DRY RUN MODE ===[/yellow]\n")
227
+
228
+ app.console.print(f"Applying {len(to_apply)} migration(s) in chain order:")
229
+ for m in to_apply:
230
+ app.console.print(f" • {m.migration_id}")
231
+ app.console.print()
232
+
233
+ executor = MigrationExecutor(storage)
234
+
235
+ for migration in to_apply:
236
+ app.console.print(f"[bold]Applying: {migration.migration_id}[/bold]")
237
+ app.console.print(
238
+ f" Affecting {len(migration.get_affected_features(metadata_store))} feature(s)"
239
+ )
240
+
241
+ result = executor.execute(migration, metadata_store, dry_run=dry_run)
242
+
243
+ # Print result
244
+ if result.status == "completed":
245
+ app.console.print("[green]✓[/green] Migration completed")
246
+ elif result.status == "skipped":
247
+ app.console.print("[yellow]⊘[/yellow] Migration skipped (dry run)")
248
+ else:
249
+ app.console.print("[red]✗[/red] Migration failed")
250
+
251
+ app.console.print(f" Features completed: {result.features_completed}")
252
+ app.console.print(f" Features failed: {result.features_failed}")
253
+ app.console.print(f" Rows affected: {result.rows_affected}")
254
+ app.console.print(f" Duration: {result.duration_seconds:.2f}s")
255
+
256
+ if result.errors:
257
+ app.console.print("\n[red]Errors:[/red]")
258
+ for feature_key, error in result.errors.items():
259
+ app.console.print(f" ✗ {feature_key}: {error}")
260
+
261
+ if result.status == "failed":
262
+ app.console.print(
263
+ f"\n[red]Migration {migration.migration_id} failed. "
264
+ "Stopping chain execution.[/red]"
265
+ )
266
+ raise SystemExit(1)
267
+
268
+ app.console.print() # Blank line between migrations
269
+
270
+
271
+ @app.command
272
+ def status():
273
+ """Show migration chain and execution status.
274
+
275
+ Reads migration definitions from YAML files (git).
276
+ Shows execution status from database events.
277
+ Displays the parent chain in order.
278
+
279
+ Example:
280
+ $ metaxy migrations status
281
+
282
+ Migration Chain:
283
+ ────────────────────────────────────────────
284
+ ✓ 20250110_120000 (parent: initial)
285
+ Status: COMPLETED
286
+ Features: 5/5 completed
287
+
288
+ ○ 20250113_103000 (parent: 20250110_120000)
289
+ Status: NOT STARTED
290
+ Features: 3 affected
291
+
292
+ ⚠ Multiple heads detected: [20250110_120000_a, 20250110_120000_b]
293
+ """
294
+ from pathlib import Path
295
+
296
+ from metaxy.cli.context import get_store
297
+ from metaxy.entrypoints import load_features
298
+ from metaxy.metadata_store.system_tables import SystemTableStorage
299
+ from metaxy.migrations.loader import build_migration_chain
300
+
301
+ # Load features
302
+ load_features()
303
+
304
+ metadata_store = get_store()
305
+ migrations_dir = Path(".metaxy/migrations")
306
+
307
+ with metadata_store:
308
+ storage = SystemTableStorage(metadata_store)
309
+
310
+ # Try to build migration chain
311
+ try:
312
+ chain = build_migration_chain(migrations_dir)
313
+ except ValueError as e:
314
+ app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
315
+ return
316
+
317
+ if not chain:
318
+ app.console.print("[yellow]No migrations found.[/yellow]")
319
+ app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
320
+ return
321
+
322
+ app.console.print("\n[bold]Migration Chain:[/bold]")
323
+ app.console.print("─" * 60)
324
+
325
+ for migration in chain:
326
+ migration_id = migration.migration_id
327
+
328
+ # Get status from events
329
+ status_str = storage.get_migration_status(migration_id)
330
+
331
+ # Get completion counts
332
+ completed_features = storage.get_completed_features(migration_id)
333
+ failed_features = storage.get_failed_features(migration_id)
334
+
335
+ # Compute total affected
336
+ if status_str == "not_started":
337
+ try:
338
+ total_affected = len(
339
+ migration.get_affected_features(metadata_store)
340
+ )
341
+ except Exception:
342
+ total_affected = "?"
343
+ else:
344
+ total_affected = len(completed_features) + len(failed_features)
345
+
346
+ # Print status icon
347
+ if status_str == "completed":
348
+ app.console.print(
349
+ f"[green]✓[/green] {migration_id} (parent: {migration.parent})"
350
+ )
351
+ app.console.print(" Status: [green]COMPLETED[/green]")
352
+ elif status_str == "failed":
353
+ app.console.print(
354
+ f"[red]✗[/red] {migration_id} (parent: {migration.parent})"
355
+ )
356
+ app.console.print(" Status: [red]FAILED[/red]")
357
+ elif status_str == "in_progress":
358
+ app.console.print(
359
+ f"[yellow]⚠[/yellow] {migration_id} (parent: {migration.parent})"
360
+ )
361
+ app.console.print(" Status: [yellow]IN PROGRESS[/yellow]")
362
+ else:
363
+ app.console.print(
364
+ f"[blue]○[/blue] {migration_id} (parent: {migration.parent})"
365
+ )
366
+ app.console.print(" Status: [blue]NOT STARTED[/blue]")
367
+
368
+ app.console.print(" Snapshots:")
369
+ app.console.print(f" From: {migration.from_snapshot_version}")
370
+ app.console.print(f" To: {migration.to_snapshot_version}")
371
+ app.console.print(
372
+ f" Features: {len(completed_features)}/{total_affected} completed"
373
+ )
374
+
375
+ if failed_features:
376
+ app.console.print(
377
+ f" [red]Failed features ({len(failed_features)}):[/red]"
378
+ )
379
+ for feature_key, error in list(failed_features.items())[:3]:
380
+ app.console.print(f" ✗ {feature_key}: {error}")
381
+
382
+ app.console.print()
383
+
384
+
385
+ @app.command(name="list")
386
+ def list_migrations():
387
+ """List all migrations in chain order.
388
+
389
+ Displays a simple table showing migration ID, creation time, and operations.
390
+
391
+ Example:
392
+ $ metaxy migrations list
393
+
394
+ 20250110_120000 2025-01-10 12:00 DataVersionReconciliation
395
+ 20250113_103000 2025-01-13 10:30 DataVersionReconciliation
396
+ """
397
+ from pathlib import Path
398
+
399
+ from rich.table import Table
400
+
401
+ from metaxy.cli.context import get_store
402
+ from metaxy.entrypoints import load_features
403
+ from metaxy.migrations.loader import build_migration_chain
404
+
405
+ # Load features
406
+ load_features()
407
+
408
+ metadata_store = get_store()
409
+ migrations_dir = Path(".metaxy/migrations")
410
+
411
+ with metadata_store:
412
+ # Build migration chain
413
+ try:
414
+ chain = build_migration_chain(migrations_dir)
415
+ except ValueError as e:
416
+ app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
417
+ return
418
+
419
+ if not chain:
420
+ app.console.print("[yellow]No migrations found.[/yellow]")
421
+ app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
422
+ return
423
+
424
+ # Create borderless table with blue headers (no truncation)
425
+ table = Table(
426
+ show_header=True,
427
+ show_edge=False,
428
+ box=None,
429
+ padding=(0, 2),
430
+ header_style="bold blue",
431
+ )
432
+ table.add_column("ID", style="bold", no_wrap=False, overflow="fold")
433
+ table.add_column("Created", style="dim", no_wrap=False, overflow="fold")
434
+ table.add_column("Operations", no_wrap=False, overflow="fold")
435
+
436
+ for migration in chain:
437
+ # Format created_at - simpler format without seconds
438
+ created_str = migration.created_at.strftime("%Y-%m-%d %H:%M")
439
+
440
+ # Format operations - extract short names
441
+ op_names = []
442
+ for op in migration.ops:
443
+ op_type = op.get("type", "unknown")
444
+ # Extract just the class name (last part after final dot)
445
+ op_short = op_type.split(".")[-1]
446
+ op_names.append(op_short)
447
+
448
+ ops_str = ", ".join(op_names)
449
+
450
+ table.add_row(migration.migration_id, created_str, ops_str)
451
+
452
+ app.console.print()
453
+ app.console.print(table)
454
+ app.console.print()
455
+
456
+
457
+ @app.command
458
+ def explain(
459
+ migration_id: Annotated[
460
+ str | None,
461
+ cyclopts.Parameter(
462
+ help="Migration ID to explain (explains latest if not specified)"
463
+ ),
464
+ ] = None,
465
+ ):
466
+ """Show detailed diff for a migration.
467
+
468
+ Reads migration from YAML file.
469
+ Computes and displays the GraphDiff between the two snapshots on-demand.
470
+
471
+ Examples:
472
+ # Explain latest migration (head of chain)
473
+ $ metaxy migrations explain
474
+
475
+ # Explain specific migration
476
+ $ metaxy migrations explain 20250113_103000
477
+ """
478
+ from pathlib import Path
479
+
480
+ from metaxy.cli.context import get_store
481
+ from metaxy.entrypoints import load_features
482
+ from metaxy.migrations.loader import (
483
+ find_latest_migration,
484
+ find_migration_yaml,
485
+ load_migration_from_yaml,
486
+ )
487
+
488
+ # Load features
489
+ load_features()
490
+
491
+ metadata_store = get_store()
492
+ migrations_dir = Path(".metaxy/migrations")
493
+
494
+ with metadata_store:
495
+ # Get migration ID
496
+ if migration_id is None:
497
+ # Get latest migration (head)
498
+ try:
499
+ migration_id = find_latest_migration(migrations_dir)
500
+ except ValueError as e:
501
+ app.console.print(f"[red]✗[/red] {e}")
502
+ raise SystemExit(1)
503
+
504
+ if migration_id is None:
505
+ app.console.print("[yellow]No migrations found[/yellow]")
506
+ app.console.print("Run 'metaxy migrations generate' first")
507
+ return
508
+
509
+ # Load migration from YAML
510
+ try:
511
+ yaml_path = find_migration_yaml(migration_id, migrations_dir)
512
+ migration = load_migration_from_yaml(yaml_path)
513
+ except FileNotFoundError as e:
514
+ app.console.print(f"[red]✗[/red] {e}")
515
+ raise SystemExit(1)
516
+
517
+ # Type narrow to DiffMigration for explain command
518
+ from metaxy.migrations.models import DiffMigration
519
+
520
+ if not isinstance(migration, DiffMigration):
521
+ app.console.print(
522
+ f"[red]✗[/red] Migration '{migration_id}' is not a DiffMigration"
523
+ )
524
+ app.console.print(
525
+ f" Type: {type(migration).__name__} (explain only supports DiffMigration)"
526
+ )
527
+ raise SystemExit(1)
528
+
529
+ # Print header
530
+ app.console.print(f"\n[bold]Migration: {migration_id}[/bold]")
531
+ app.console.print(f"From: {migration.from_snapshot_version}")
532
+ app.console.print(f"To: {migration.to_snapshot_version}")
533
+
534
+ # Get description (computed on-demand)
535
+ description = migration.get_description(metadata_store)
536
+ app.console.print(f"Description: {description}")
537
+ app.console.print()
538
+
539
+ # Compute diff on-demand
540
+ try:
541
+ graph_diff = migration.compute_graph_diff(metadata_store)
542
+ except Exception as e:
543
+ app.console.print(f"[red]✗[/red] Failed to compute diff: {e}")
544
+ raise SystemExit(1)
545
+
546
+ # Display detailed diff
547
+ if not graph_diff.has_changes:
548
+ app.console.print("[yellow]No changes detected[/yellow]")
549
+ return
550
+
551
+ # Added nodes
552
+ if graph_diff.added_nodes:
553
+ app.console.print(
554
+ f"[green]Added Features ({len(graph_diff.added_nodes)}):[/green]"
555
+ )
556
+ for node in graph_diff.added_nodes:
557
+ app.console.print(f" ✓ {node.feature_key}")
558
+ if node.fields:
559
+ app.console.print(f" Fields ({len(node.fields)}):")
560
+ for field in node.fields[:3]:
561
+ app.console.print(
562
+ f" - {field['key']} (cv={field.get('code_version', '?')})"
563
+ )
564
+ if len(node.fields) > 3:
565
+ app.console.print(f" ... and {len(node.fields) - 3} more")
566
+ app.console.print()
567
+
568
+ # Removed nodes
569
+ if graph_diff.removed_nodes:
570
+ app.console.print(
571
+ f"[red]Removed Features ({len(graph_diff.removed_nodes)}):[/red]"
572
+ )
573
+ for node in graph_diff.removed_nodes:
574
+ app.console.print(f" ✗ {node.feature_key}")
575
+ if node.fields:
576
+ app.console.print(f" Fields ({len(node.fields)}):")
577
+ for field in node.fields[:3]:
578
+ app.console.print(
579
+ f" - {field['key']} (cv={field.get('code_version', '?')})"
580
+ )
581
+ if len(node.fields) > 3:
582
+ app.console.print(f" ... and {len(node.fields) - 3} more")
583
+ app.console.print()
584
+
585
+ # Changed nodes
586
+ if graph_diff.changed_nodes:
587
+ app.console.print(
588
+ f"[yellow]Changed Features ({len(graph_diff.changed_nodes)}):[/yellow]"
589
+ )
590
+ for node in graph_diff.changed_nodes:
591
+ app.console.print(f" ⚠ {node.feature_key}")
592
+ old_ver = node.old_version if node.old_version else "None"
593
+ new_ver = node.new_version if node.new_version else "None"
594
+ app.console.print(f" Version: {old_ver} → {new_ver}")
595
+
596
+ if (
597
+ node.old_code_version is not None
598
+ or node.new_code_version is not None
599
+ ):
600
+ app.console.print(
601
+ f" Code version: {node.old_code_version} → {node.new_code_version}"
602
+ )
603
+
604
+ # Show field changes
605
+ total_field_changes = (
606
+ len(node.added_fields)
607
+ + len(node.removed_fields)
608
+ + len(node.changed_fields)
609
+ )
610
+ if total_field_changes > 0:
611
+ app.console.print(f" Field changes ({total_field_changes}):")
612
+
613
+ if node.added_fields:
614
+ app.console.print(
615
+ f" [green]Added ({len(node.added_fields)}):[/green]"
616
+ )
617
+ for field in node.added_fields[:2]:
618
+ app.console.print(
619
+ f" + {field.field_key} (cv={field.new_code_version})"
620
+ )
621
+ if len(node.added_fields) > 2:
622
+ app.console.print(
623
+ f" ... and {len(node.added_fields) - 2} more"
624
+ )
625
+
626
+ if node.removed_fields:
627
+ app.console.print(
628
+ f" [red]Removed ({len(node.removed_fields)}):[/red]"
629
+ )
630
+ for field in node.removed_fields[:2]:
631
+ app.console.print(
632
+ f" - {field.field_key} (cv={field.old_code_version})"
633
+ )
634
+ if len(node.removed_fields) > 2:
635
+ app.console.print(
636
+ f" ... and {len(node.removed_fields) - 2} more"
637
+ )
638
+
639
+ if node.changed_fields:
640
+ app.console.print(
641
+ f" [yellow]Changed ({len(node.changed_fields)}):[/yellow]"
642
+ )
643
+ for field in node.changed_fields[:2]:
644
+ app.console.print(
645
+ f" ~ {field.field_key} (cv={field.old_code_version}→{field.new_code_version})"
646
+ )
647
+ if len(node.changed_fields) > 2:
648
+ app.console.print(
649
+ f" ... and {len(node.changed_fields) - 2} more"
650
+ )
651
+
652
+ app.console.print()
653
+
654
+ # Print affected features summary (computed on-demand)
655
+ affected_features = migration.get_affected_features(metadata_store)
656
+ app.console.print(f"[bold]Affected Features ({len(affected_features)}):[/bold]")
657
+ for feature_key in affected_features[:10]:
658
+ app.console.print(f" • {feature_key}")
659
+ if len(affected_features) > 10:
660
+ app.console.print(f" ... and {len(affected_features) - 10} more")
661
+
662
+
663
+ @app.command
664
+ def describe(
665
+ migration_ids: Annotated[
666
+ list[str],
667
+ cyclopts.Parameter(
668
+ help="Migration IDs to describe (default: all migrations in order)"
669
+ ),
670
+ ] = [],
671
+ ):
672
+ """Show verbose description of migration(s).
673
+
674
+ Displays detailed information about what the migration will do:
675
+ - Migration metadata (ID, parent, snapshots, created timestamp)
676
+ - Operations to execute
677
+ - Affected features with row counts
678
+ - Execution status if already run
679
+
680
+ Examples:
681
+ # Describe all migrations in chain order
682
+ $ metaxy migrations describe
683
+
684
+ # Describe specific migration
685
+ $ metaxy migrations describe 20250127_120000
686
+
687
+ # Describe multiple migrations
688
+ $ metaxy migrations describe 20250101_120000 20250102_090000
689
+ """
690
+ from pathlib import Path
691
+
692
+ from metaxy.cli.context import get_store
693
+ from metaxy.entrypoints import load_features
694
+ from metaxy.metadata_store.system_tables import SystemTableStorage
695
+ from metaxy.migrations.loader import (
696
+ build_migration_chain,
697
+ find_migration_yaml,
698
+ load_migration_from_yaml,
699
+ )
700
+
701
+ # Load features
702
+ load_features()
703
+
704
+ metadata_store = get_store()
705
+ migrations_dir = Path(".metaxy/migrations")
706
+
707
+ with metadata_store:
708
+ storage = SystemTableStorage(metadata_store)
709
+
710
+ # Determine which migrations to describe
711
+ if not migration_ids:
712
+ # Default: describe all migrations in chain order
713
+ try:
714
+ chain = build_migration_chain(migrations_dir)
715
+ except ValueError as e:
716
+ app.console.print(f"[red]✗[/red] Invalid migration chain: {e}")
717
+ raise SystemExit(1)
718
+
719
+ if not chain:
720
+ app.console.print("[yellow]No migrations found.[/yellow]")
721
+ return
722
+
723
+ migrations_to_describe = chain
724
+ else:
725
+ # Load specific migrations
726
+ migrations_to_describe = []
727
+ for migration_id in migration_ids:
728
+ try:
729
+ yaml_path = find_migration_yaml(migration_id, migrations_dir)
730
+ migration_obj = load_migration_from_yaml(yaml_path)
731
+ migrations_to_describe.append(migration_obj)
732
+ except FileNotFoundError as e:
733
+ app.console.print(f"[red]✗[/red] {e}")
734
+ raise SystemExit(1)
735
+
736
+ # Describe each migration
737
+ for i, migration_obj in enumerate(migrations_to_describe):
738
+ if i > 0:
739
+ app.console.print("\n") # Separator between migrations
740
+
741
+ # Find YAML path for display
742
+ yaml_path = find_migration_yaml(migration_obj.migration_id, migrations_dir)
743
+
744
+ # Print header
745
+ app.console.print("\n[bold]Migration Description[/bold]")
746
+ app.console.print("─" * 60)
747
+ app.console.print(f"[bold]ID:[/bold] {migration_obj.migration_id}")
748
+ app.console.print(
749
+ f"[bold]Created:[/bold] {migration_obj.created_at.isoformat()}"
750
+ )
751
+ app.console.print(f"[bold]Parent:[/bold] {migration_obj.parent}")
752
+ app.console.print(f"[bold]YAML:[/bold] {yaml_path}")
753
+ app.console.print()
754
+
755
+ # Snapshots
756
+ app.console.print("[bold]Snapshots:[/bold]")
757
+ app.console.print(f" From: {migration_obj.from_snapshot_version}")
758
+ app.console.print(f" To: {migration_obj.to_snapshot_version}")
759
+ app.console.print()
760
+
761
+ # Operations
762
+ app.console.print("[bold]Operations:[/bold]")
763
+ for j, op in enumerate(migration_obj.ops, 1):
764
+ op_type = op.get("type", "unknown")
765
+ app.console.print(f" {j}. {op_type}")
766
+ app.console.print()
767
+
768
+ # Get affected features
769
+ app.console.print("[bold]Computing affected features...[/bold]")
770
+ try:
771
+ affected_features = migration_obj.get_affected_features(metadata_store)
772
+ except Exception as e:
773
+ app.console.print(
774
+ f"[red]✗[/red] Failed to compute affected features: {e}"
775
+ )
776
+ continue # Skip to next migration
777
+
778
+ app.console.print(
779
+ f"\n[bold]Affected Features ({len(affected_features)}):[/bold]"
780
+ )
781
+ app.console.print("─" * 60)
782
+
783
+ from metaxy.models.feature import FeatureGraph
784
+ from metaxy.models.types import FeatureKey
785
+
786
+ graph = FeatureGraph.get_active()
787
+
788
+ for feature_key_str in affected_features:
789
+ feature_key_obj = FeatureKey(feature_key_str.split("/"))
790
+
791
+ # Get feature class
792
+ if feature_key_obj not in graph.features_by_key:
793
+ app.console.print(f"[yellow]⚠[/yellow] {feature_key_str}")
794
+ app.console.print(
795
+ " [yellow]Feature not in current graph[/yellow]"
796
+ )
797
+ continue
798
+
799
+ feature_cls = graph.features_by_key[feature_key_obj]
800
+
801
+ # Get current row count
802
+ try:
803
+ import narwhals as nw
804
+
805
+ metadata = metadata_store.read_metadata(
806
+ feature_cls,
807
+ current_only=False,
808
+ allow_fallback=False,
809
+ )
810
+ row_count = (
811
+ metadata.select(nw.col("sample_uid").unique())
812
+ .collect()
813
+ .shape[0]
814
+ )
815
+ except Exception:
816
+ row_count = "?"
817
+
818
+ # Check if feature has upstream dependencies
819
+ plan = graph.get_feature_plan(feature_key_obj)
820
+ has_upstream = plan.deps is not None and len(plan.deps) > 0
821
+
822
+ app.console.print(f"[bold]{feature_key_str}[/bold]")
823
+ app.console.print(f" Samples: {row_count}")
824
+ app.console.print(f" Has upstream: {has_upstream}")
825
+
826
+ app.console.print()
827
+
828
+ # Execution status
829
+ status_str = storage.get_migration_status(migration_obj.migration_id)
830
+ completed_features = storage.get_completed_features(
831
+ migration_obj.migration_id
832
+ )
833
+ failed_features = storage.get_failed_features(migration_obj.migration_id)
834
+
835
+ app.console.print("[bold]Execution Status:[/bold]")
836
+ if status_str == "completed":
837
+ app.console.print(" [green]✓ COMPLETED[/green]")
838
+ app.console.print(
839
+ f" Features processed: {len(completed_features)}/{len(affected_features)}"
840
+ )
841
+ elif status_str == "failed":
842
+ app.console.print(" [red]✗ FAILED[/red]")
843
+ app.console.print(
844
+ f" Features completed: {len(completed_features)}/{len(affected_features)}"
845
+ )
846
+ app.console.print(f" Features failed: {len(failed_features)}")
847
+ if failed_features:
848
+ app.console.print(" Failed features:")
849
+ for feature_key, error in list(failed_features.items())[:5]:
850
+ app.console.print(f" • {feature_key}: {error}")
851
+ elif status_str == "in_progress":
852
+ app.console.print(" [yellow]⚠ IN PROGRESS[/yellow]")
853
+ app.console.print(
854
+ f" Features completed: {len(completed_features)}/{len(affected_features)}"
855
+ )
856
+ else:
857
+ app.console.print(" [blue]○ NOT STARTED[/blue]")
858
+ app.console.print(f" Features to process: {len(affected_features)}")
859
+
860
+
861
+ if __name__ == "__main__":
862
+ app()