metaxy 0.0.1.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. metaxy/__init__.py +170 -0
  2. metaxy/_packaging.py +96 -0
  3. metaxy/_testing/__init__.py +55 -0
  4. metaxy/_testing/config.py +43 -0
  5. metaxy/_testing/metaxy_project.py +780 -0
  6. metaxy/_testing/models.py +111 -0
  7. metaxy/_testing/parametric/__init__.py +13 -0
  8. metaxy/_testing/parametric/metadata.py +664 -0
  9. metaxy/_testing/pytest_helpers.py +74 -0
  10. metaxy/_testing/runbook.py +533 -0
  11. metaxy/_utils.py +35 -0
  12. metaxy/_version.py +1 -0
  13. metaxy/cli/app.py +97 -0
  14. metaxy/cli/console.py +13 -0
  15. metaxy/cli/context.py +167 -0
  16. metaxy/cli/graph.py +610 -0
  17. metaxy/cli/graph_diff.py +290 -0
  18. metaxy/cli/list.py +46 -0
  19. metaxy/cli/metadata.py +317 -0
  20. metaxy/cli/migrations.py +999 -0
  21. metaxy/cli/utils.py +268 -0
  22. metaxy/config.py +680 -0
  23. metaxy/entrypoints.py +296 -0
  24. metaxy/ext/__init__.py +1 -0
  25. metaxy/ext/dagster/__init__.py +54 -0
  26. metaxy/ext/dagster/constants.py +10 -0
  27. metaxy/ext/dagster/dagster_type.py +156 -0
  28. metaxy/ext/dagster/io_manager.py +200 -0
  29. metaxy/ext/dagster/metaxify.py +512 -0
  30. metaxy/ext/dagster/observable.py +115 -0
  31. metaxy/ext/dagster/resources.py +27 -0
  32. metaxy/ext/dagster/selection.py +73 -0
  33. metaxy/ext/dagster/table_metadata.py +417 -0
  34. metaxy/ext/dagster/utils.py +462 -0
  35. metaxy/ext/sqlalchemy/__init__.py +23 -0
  36. metaxy/ext/sqlalchemy/config.py +29 -0
  37. metaxy/ext/sqlalchemy/plugin.py +353 -0
  38. metaxy/ext/sqlmodel/__init__.py +13 -0
  39. metaxy/ext/sqlmodel/config.py +29 -0
  40. metaxy/ext/sqlmodel/plugin.py +499 -0
  41. metaxy/graph/__init__.py +29 -0
  42. metaxy/graph/describe.py +325 -0
  43. metaxy/graph/diff/__init__.py +21 -0
  44. metaxy/graph/diff/diff_models.py +446 -0
  45. metaxy/graph/diff/differ.py +769 -0
  46. metaxy/graph/diff/models.py +443 -0
  47. metaxy/graph/diff/rendering/__init__.py +18 -0
  48. metaxy/graph/diff/rendering/base.py +323 -0
  49. metaxy/graph/diff/rendering/cards.py +188 -0
  50. metaxy/graph/diff/rendering/formatter.py +805 -0
  51. metaxy/graph/diff/rendering/graphviz.py +246 -0
  52. metaxy/graph/diff/rendering/mermaid.py +326 -0
  53. metaxy/graph/diff/rendering/rich.py +169 -0
  54. metaxy/graph/diff/rendering/theme.py +48 -0
  55. metaxy/graph/diff/traversal.py +247 -0
  56. metaxy/graph/status.py +329 -0
  57. metaxy/graph/utils.py +58 -0
  58. metaxy/metadata_store/__init__.py +32 -0
  59. metaxy/metadata_store/_ducklake_support.py +419 -0
  60. metaxy/metadata_store/base.py +1792 -0
  61. metaxy/metadata_store/bigquery.py +354 -0
  62. metaxy/metadata_store/clickhouse.py +184 -0
  63. metaxy/metadata_store/delta.py +371 -0
  64. metaxy/metadata_store/duckdb.py +446 -0
  65. metaxy/metadata_store/exceptions.py +61 -0
  66. metaxy/metadata_store/ibis.py +542 -0
  67. metaxy/metadata_store/lancedb.py +391 -0
  68. metaxy/metadata_store/memory.py +292 -0
  69. metaxy/metadata_store/system/__init__.py +57 -0
  70. metaxy/metadata_store/system/events.py +264 -0
  71. metaxy/metadata_store/system/keys.py +9 -0
  72. metaxy/metadata_store/system/models.py +129 -0
  73. metaxy/metadata_store/system/storage.py +957 -0
  74. metaxy/metadata_store/types.py +10 -0
  75. metaxy/metadata_store/utils.py +104 -0
  76. metaxy/metadata_store/warnings.py +36 -0
  77. metaxy/migrations/__init__.py +32 -0
  78. metaxy/migrations/detector.py +291 -0
  79. metaxy/migrations/executor.py +516 -0
  80. metaxy/migrations/generator.py +319 -0
  81. metaxy/migrations/loader.py +231 -0
  82. metaxy/migrations/models.py +528 -0
  83. metaxy/migrations/ops.py +447 -0
  84. metaxy/models/__init__.py +0 -0
  85. metaxy/models/bases.py +12 -0
  86. metaxy/models/constants.py +139 -0
  87. metaxy/models/feature.py +1335 -0
  88. metaxy/models/feature_spec.py +338 -0
  89. metaxy/models/field.py +263 -0
  90. metaxy/models/fields_mapping.py +307 -0
  91. metaxy/models/filter_expression.py +297 -0
  92. metaxy/models/lineage.py +285 -0
  93. metaxy/models/plan.py +232 -0
  94. metaxy/models/types.py +475 -0
  95. metaxy/py.typed +0 -0
  96. metaxy/utils/__init__.py +1 -0
  97. metaxy/utils/constants.py +2 -0
  98. metaxy/utils/exceptions.py +23 -0
  99. metaxy/utils/hashing.py +230 -0
  100. metaxy/versioning/__init__.py +31 -0
  101. metaxy/versioning/engine.py +656 -0
  102. metaxy/versioning/feature_dep_transformer.py +151 -0
  103. metaxy/versioning/ibis.py +249 -0
  104. metaxy/versioning/lineage_handler.py +205 -0
  105. metaxy/versioning/polars.py +189 -0
  106. metaxy/versioning/renamed_df.py +35 -0
  107. metaxy/versioning/types.py +63 -0
  108. metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
  109. metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
  110. metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
  111. metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
@@ -0,0 +1,999 @@
1
+ """New Migration CLI commands using event-based system."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Annotated, Literal
6
+
7
+ import cyclopts
8
+
9
+ from metaxy.cli.console import console, data_console, error_console
10
+
11
+ # Migrations subcommand app
12
+ app = cyclopts.App(
13
+ name="migrations", # pyrefly: ignore[unexpected-keyword]
14
+ help="Metadata migration commands", # pyrefly: ignore[unexpected-keyword]
15
+ console=console, # pyrefly: ignore[unexpected-keyword]
16
+ error_console=error_console, # pyrefly: ignore[unexpected-keyword]
17
+ )
18
+
19
+
20
+ @app.command
21
+ def generate(
22
+ *,
23
+ name: Annotated[
24
+ str | None,
25
+ cyclopts.Parameter(help="Migration name (creates {timestamp}_{name} ID)"),
26
+ ] = None,
27
+ store: Annotated[
28
+ str | None,
29
+ cyclopts.Parameter(help="Store name (defaults to default)"),
30
+ ] = None,
31
+ from_snapshot: Annotated[
32
+ str | None,
33
+ cyclopts.Parameter(
34
+ help="Compare from this historical snapshot version (defaults to latest)"
35
+ ),
36
+ ] = None,
37
+ op: Annotated[
38
+ list[str],
39
+ cyclopts.Parameter(
40
+ help="Operation class path to use (can be repeated). Example: metaxy.migrations.ops.DataVersionReconciliation"
41
+ ),
42
+ ],
43
+ type: Annotated[
44
+ Literal["diff", "full"],
45
+ cyclopts.Parameter(
46
+ help="Migration type: 'diff' (compare different graph snapshots) or 'full' (operates on a single graph snapshot)"
47
+ ),
48
+ ] = "diff",
49
+ ):
50
+ """Generate migration from detected feature changes.
51
+
52
+ Two migration types are supported:
53
+
54
+ - **diff** : Compares the latest snapshot in the store (or specified
55
+ from_snapshot) with the current active graph to detect changes. Only affected
56
+ features are included.
57
+
58
+ - **full**: Creates a migration that includes ALL features in the current graph.
59
+ Each operation will have a 'features' list with all feature keys.
60
+
61
+ Examples:
62
+ # Generate diff migration with DataVersionReconciliation operation
63
+ $ metaxy migrations generate --op metaxy.migrations.ops.DataVersionReconciliation
64
+
65
+ # Generate full graph migration (all features)
66
+ $ metaxy migrations generate --migration-type full --op myproject.ops.CustomBackfill
67
+
68
+ # Custom operation type
69
+ $ metaxy migrations generate --op myproject.ops.CustomReconciliation
70
+
71
+ # Multiple operations
72
+ $ metaxy migrations generate \
73
+ --op metaxy.migrations.ops.DataVersionReconciliation \
74
+ --op myproject.ops.CustomBackfill
75
+ """
76
+ import shlex
77
+ import sys
78
+ from pathlib import Path
79
+
80
+ from metaxy.cli.context import AppContext
81
+ from metaxy.migrations.detector import (
82
+ detect_diff_migration,
83
+ generate_full_graph_migration,
84
+ )
85
+
86
+ context = AppContext.get()
87
+ context.raise_command_cannot_override_project()
88
+ config = context.config
89
+
90
+ # Convert op_type list to ops format
91
+ if len(op) == 0:
92
+ app.console.print(
93
+ "[red]✗[/red] --op is required. "
94
+ "Example: --op metaxy.migrations.ops.DataVersionReconciliation"
95
+ )
96
+ raise SystemExit(1)
97
+
98
+ ops = [{"type": o} for o in op]
99
+
100
+ # Get store and project from config
101
+ metadata_store = context.get_store(store)
102
+ migrations_dir = Path(config.migrations_dir)
103
+ project = context.get_required_project() # This command needs a specific project
104
+
105
+ # Reconstruct CLI command for YAML comment
106
+ cli_command = shlex.join(sys.argv)
107
+
108
+ with metadata_store.open("write"):
109
+ if type == "diff":
110
+ # Detect migration and write YAML
111
+ migration = detect_diff_migration(
112
+ metadata_store,
113
+ project=project,
114
+ from_snapshot_version=from_snapshot,
115
+ ops=ops,
116
+ migrations_dir=migrations_dir,
117
+ name=name,
118
+ command=cli_command,
119
+ )
120
+
121
+ if migration is None:
122
+ app.console.print("[yellow]No changes detected[/yellow]")
123
+ app.console.print(" Current graph matches latest snapshot")
124
+ return
125
+
126
+ # Print summary for DiffMigration
127
+ yaml_path = migrations_dir / f"{migration.migration_id}.yaml"
128
+ app.console.print("\n[green]✓[/green] DiffMigration generated")
129
+ app.console.print(f" Migration ID: {migration.migration_id}")
130
+ app.console.print(f" YAML file: {yaml_path}")
131
+ app.console.print(f" From snapshot: {migration.from_snapshot_version}")
132
+ app.console.print(f" To snapshot: {migration.to_snapshot_version}")
133
+
134
+ else: # type == "full"
135
+ if from_snapshot is not None:
136
+ app.console.print(
137
+ "[yellow]Warning:[/yellow] --from-snapshot is ignored for full graph migrations"
138
+ )
139
+
140
+ migration = generate_full_graph_migration(
141
+ metadata_store,
142
+ project=project,
143
+ ops=ops,
144
+ migrations_dir=migrations_dir,
145
+ name=name,
146
+ command=cli_command,
147
+ )
148
+
149
+ # Print summary for FullGraphMigration
150
+ yaml_path = migrations_dir / f"{migration.migration_id}.yaml"
151
+ app.console.print("\n[green]✓[/green] FullGraphMigration generated")
152
+ app.console.print(f" Migration ID: {migration.migration_id}")
153
+ app.console.print(f" YAML file: {yaml_path}")
154
+ app.console.print(f" Snapshot: {migration.snapshot_version}")
155
+
156
+ # Output migration ID to stdout for scripting
157
+ data_console.print(migration.migration_id)
158
+
159
+ # Get affected features (computed on-demand)
160
+ affected_features = migration.get_affected_features(metadata_store, project)
161
+ app.console.print(f"\n Affected features ({len(affected_features)}):")
162
+ for feature_key in affected_features[:5]:
163
+ app.console.print(f" ✓ {feature_key}")
164
+ if len(affected_features) > 5:
165
+ app.console.print(f" ... and {len(affected_features) - 5} more")
166
+
167
+ app.console.print("\n[bold]NEXT STEPS:[/bold]")
168
+ app.console.print(f"1. Review migration YAML: {yaml_path}")
169
+ app.console.print(
170
+ f"2. Run 'metaxy migrations apply {migration.migration_id}' to execute. Use CD for production!"
171
+ )
172
+
173
+
174
+ @app.command
175
+ def apply(
176
+ migration_id: Annotated[
177
+ str | None,
178
+ cyclopts.Parameter(
179
+ help="Migration ID to apply (applies all unapplied if not specified)"
180
+ ),
181
+ ] = None,
182
+ store: Annotated[
183
+ str | None,
184
+ cyclopts.Parameter(help="Metadata store to use."),
185
+ ] = None,
186
+ *,
187
+ dry_run: Annotated[
188
+ bool,
189
+ cyclopts.Parameter(help="Preview changes without executing"),
190
+ ] = False,
191
+ ):
192
+ """Apply migration(s) from YAML files.
193
+
194
+ Reads migration definitions from .metaxy/migrations/ directory (git).
195
+ Follows parent chain to ensure correct order.
196
+ Tracks execution state in database (events).
197
+
198
+ Examples:
199
+ # Apply all unapplied migrations in chain order
200
+ $ metaxy migrations apply
201
+
202
+ # Apply specific migration (and all its unapplied predecessors)
203
+ $ metaxy migrations apply 20250113_103000
204
+
205
+ # Dry run
206
+ $ metaxy migrations apply --dry-run
207
+ """
208
+ from pathlib import Path
209
+
210
+ from metaxy.cli.context import AppContext
211
+ from metaxy.metadata_store.system import SystemTableStorage
212
+ from metaxy.migrations.executor import MigrationExecutor
213
+ from metaxy.migrations.loader import build_migration_chain
214
+
215
+ context = AppContext.get()
216
+ context.raise_command_cannot_override_project()
217
+
218
+ # Get context and project from config
219
+ project = context.get_required_project() # This command needs a specific project
220
+ metadata_store = context.get_store(store)
221
+ migrations_dir = Path(".metaxy/migrations")
222
+
223
+ with metadata_store.open("write"):
224
+ storage = SystemTableStorage(metadata_store)
225
+
226
+ # Build migration chain
227
+ try:
228
+ chain = build_migration_chain(migrations_dir)
229
+ except ValueError as e:
230
+ from metaxy.cli.utils import print_error
231
+
232
+ print_error(app.console, "Invalid migration chain", e)
233
+ raise SystemExit(1)
234
+
235
+ if not chain:
236
+ app.console.print("[yellow]No migrations found[/yellow]")
237
+ app.console.print("Run 'metaxy migrations generate' first")
238
+ return
239
+
240
+ # Get completed migrations from events
241
+ from metaxy.metadata_store.system import MigrationStatus
242
+
243
+ completed_ids = set()
244
+ for m in chain:
245
+ # Get expected features from migration to verify all operations completed
246
+ expected_features = m.get_affected_features(metadata_store, project)
247
+ if (
248
+ storage.get_migration_status(
249
+ m.migration_id, project, expected_features=expected_features
250
+ )
251
+ == MigrationStatus.COMPLETED
252
+ ):
253
+ completed_ids.add(m.migration_id)
254
+
255
+ # Filter to unapplied migrations
256
+ if migration_id is None:
257
+ # Apply all unapplied
258
+ to_apply = [m for m in chain if m.migration_id not in completed_ids]
259
+ else:
260
+ # Apply specific migration and its unapplied predecessors
261
+ target_index = None
262
+ for i, m in enumerate(chain):
263
+ if m.migration_id == migration_id:
264
+ target_index = i
265
+ break
266
+
267
+ if target_index is None:
268
+ app.console.print(
269
+ f"[red]✗[/red] Migration '{migration_id}' not found in chain"
270
+ )
271
+ raise SystemExit(1)
272
+
273
+ # Include all unapplied migrations up to and including target
274
+ to_apply = [
275
+ m
276
+ for m in chain[: target_index + 1]
277
+ if m.migration_id not in completed_ids
278
+ ]
279
+
280
+ if not to_apply:
281
+ app.console.print("[blue]ℹ[/blue] All migrations already completed")
282
+ return
283
+
284
+ # Execute migrations in order
285
+ if dry_run:
286
+ app.console.print("[yellow]=== DRY RUN MODE ===[/yellow]\n")
287
+
288
+ app.console.print(f"Applying {len(to_apply)} migration(s) in chain order:")
289
+ for m in to_apply:
290
+ app.console.print(f" • {m.migration_id}")
291
+ app.console.print()
292
+
293
+ executor = MigrationExecutor(storage)
294
+
295
+ for migration in to_apply:
296
+ # Get status info to show accurate progress
297
+ status_info = migration.get_status_info(metadata_store, project)
298
+
299
+ app.console.print(f"[bold]Applying: {migration.migration_id}[/bold]")
300
+ if status_info.features_remaining > 0:
301
+ app.console.print(
302
+ f" Processing {status_info.features_remaining} feature(s) "
303
+ f"({len(status_info.completed_features)} already completed)"
304
+ )
305
+ else:
306
+ app.console.print(
307
+ f" All {status_info.features_total} feature(s) already completed"
308
+ )
309
+
310
+ result = executor.execute(
311
+ migration, metadata_store, project, dry_run=dry_run
312
+ )
313
+
314
+ # Print result
315
+ if result.status == "completed":
316
+ app.console.print("[green]✓[/green] Migration completed")
317
+ elif result.status == "skipped":
318
+ app.console.print("[yellow]⊘[/yellow] Migration skipped (dry run)")
319
+ else:
320
+ app.console.print("[red]✗[/red] Migration failed")
321
+
322
+ app.console.print(f" Features completed: {result.features_completed}")
323
+ app.console.print(f" Features failed: {result.features_failed}")
324
+ app.console.print(f" Rows affected: {result.rows_affected}")
325
+ app.console.print(f" Duration: {result.duration_seconds:.2f}s")
326
+
327
+ if result.errors:
328
+ from metaxy.cli.utils import print_error_list
329
+
330
+ print_error_list(
331
+ app.console,
332
+ result.errors,
333
+ header="\n[red]Errors:[/red]",
334
+ )
335
+
336
+ if result.status == "failed":
337
+ app.console.print(
338
+ f"\n[red]Migration {migration.migration_id} failed. "
339
+ "Stopping chain execution.[/red]"
340
+ )
341
+ raise SystemExit(1)
342
+
343
+ app.console.print() # Blank line between migrations
344
+
345
+
346
+ @app.command
347
+ def status():
348
+ """Show migrations and execution status.
349
+
350
+ Reads migration definitions from YAML files (git).
351
+ Shows execution status from database events.
352
+ Displays the parent chain in order.
353
+
354
+ Example:
355
+ $ metaxy migrations status
356
+
357
+ Migration:
358
+ ────────────────────────────────────────────
359
+ ✓ 20250110_120000 (parent: initial)
360
+ Status: COMPLETED
361
+ Features: 5/5 completed
362
+
363
+ ○ 20250113_103000 (parent: 20250110_120000)
364
+ Status: NOT STARTED
365
+ Features: 3 affected
366
+
367
+ ⚠ Multiple heads detected: [20250110_120000_a, 20250110_120000_b]
368
+ """
369
+ from pathlib import Path
370
+
371
+ from metaxy.cli.context import AppContext
372
+ from metaxy.migrations.loader import build_migration_chain
373
+
374
+ context = AppContext.get()
375
+
376
+ # Get context and project from config
377
+ project = context.get_required_project() # This command needs a specific project
378
+ metadata_store = context.get_store()
379
+ migrations_dir = Path(".metaxy/migrations")
380
+
381
+ with metadata_store:
382
+ # Try to build migration chain
383
+ try:
384
+ chain = build_migration_chain(migrations_dir)
385
+ except ValueError as e:
386
+ from metaxy.cli.utils import print_error
387
+
388
+ print_error(app.console, "Invalid migrations", e)
389
+ return
390
+
391
+ if not chain:
392
+ app.console.print("[yellow]No migrations found.[/yellow]")
393
+ app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
394
+ return
395
+
396
+ app.console.print("\n[bold]Migration:[/bold]")
397
+ app.console.print("─" * 60)
398
+
399
+ for migration in chain:
400
+ migration_id = migration.migration_id
401
+
402
+ # Get comprehensive status info from migration
403
+ from metaxy.metadata_store.system import MigrationStatus
404
+
405
+ status_info = migration.get_status_info(metadata_store, project)
406
+ migration_status = status_info.status
407
+ completed_features = status_info.completed_features
408
+ failed_features = status_info.failed_features
409
+ total_affected = status_info.features_total
410
+
411
+ # Print status icon
412
+ if migration_status == MigrationStatus.COMPLETED:
413
+ app.console.print(
414
+ f"[green]✓[/green] {migration_id} (parent: {migration.parent})"
415
+ )
416
+ app.console.print(" Status: [green]COMPLETED[/green]")
417
+ elif migration_status == MigrationStatus.FAILED:
418
+ app.console.print(
419
+ f"[red]✗[/red] {migration_id} (parent: {migration.parent})"
420
+ )
421
+ app.console.print(" Status: [red]FAILED[/red]")
422
+ elif migration_status == MigrationStatus.IN_PROGRESS:
423
+ app.console.print(
424
+ f"[yellow]⚠[/yellow] {migration_id} (parent: {migration.parent})"
425
+ )
426
+ app.console.print(" Status: [yellow]IN PROGRESS[/yellow]")
427
+ else:
428
+ app.console.print(
429
+ f"[blue]○[/blue] {migration_id} (parent: {migration.parent})"
430
+ )
431
+ app.console.print(" Status: [blue]NOT STARTED[/blue]")
432
+
433
+ # Show snapshot info for DiffMigration
434
+ from metaxy.migrations.models import DiffMigration
435
+
436
+ if isinstance(migration, DiffMigration):
437
+ app.console.print(" Snapshots:")
438
+ app.console.print(f" From: {migration.from_snapshot_version}")
439
+ app.console.print(f" To: {migration.to_snapshot_version}")
440
+
441
+ # Show operation-level progress for FullGraphMigration
442
+ from metaxy.migrations.models import FullGraphMigration, OperationConfig
443
+
444
+ if isinstance(migration, FullGraphMigration) and migration.ops:
445
+ app.console.print(" Operations:")
446
+ completed_set = set(status_info.completed_features)
447
+
448
+ for i, op_dict in enumerate(migration.ops, 1):
449
+ op_config = OperationConfig.model_validate(op_dict)
450
+ op_type_short = op_config.type.split(".")[-1] # Extract class name
451
+ op_features = set(op_config.features)
452
+ op_completed = op_features & completed_set
453
+
454
+ # Determine status icon
455
+ if len(op_completed) == 0:
456
+ icon = "[blue]○[/blue]" # Not started
457
+ elif len(op_completed) == len(op_features):
458
+ icon = "[green]✓[/green]" # Completed
459
+ else:
460
+ icon = "[yellow]⚠[/yellow]" # In progress
461
+
462
+ app.console.print(
463
+ f" {icon} {i}. {op_type_short} "
464
+ f"({len(op_completed)}/{len(op_features)} features)"
465
+ )
466
+
467
+ app.console.print(
468
+ f" Features: {len(completed_features)}/{total_affected} completed"
469
+ )
470
+
471
+ if failed_features:
472
+ from metaxy.cli.utils import print_error_list
473
+
474
+ print_error_list(
475
+ app.console,
476
+ failed_features,
477
+ header=f" [red]Failed features ({len(failed_features)}):[/red]",
478
+ indent=" ",
479
+ max_items=3,
480
+ )
481
+
482
+ app.console.print()
483
+
484
+
485
+ @app.command(name="list")
486
+ def list_migrations():
487
+ """List all migrations in chain order as defined in code.
488
+
489
+ Displays a simple table showing migration ID, creation time, and operations.
490
+
491
+ Example:
492
+ $ metaxy migrations list
493
+
494
+ 20250110_120000 2025-01-10 12:00 DataVersionReconciliation
495
+ 20250113_103000 2025-01-13 10:30 DataVersionReconciliation
496
+ """
497
+ from pathlib import Path
498
+
499
+ from rich.table import Table
500
+
501
+ from metaxy.cli.context import AppContext
502
+ from metaxy.migrations.loader import build_migration_chain
503
+
504
+ AppContext.get()
505
+ migrations_dir = Path(".metaxy/migrations")
506
+
507
+ # Build migration chain
508
+ try:
509
+ chain = build_migration_chain(migrations_dir)
510
+ except ValueError as e:
511
+ from metaxy.cli.utils import print_error
512
+
513
+ print_error(app.console, "Invalid migration", e)
514
+ return
515
+
516
+ if not chain:
517
+ app.console.print("[yellow]No migrations found.[/yellow]")
518
+ app.console.print(f" Migrations directory: {migrations_dir.resolve()}")
519
+ return
520
+
521
+ # Create borderless table with blue headers (no truncation)
522
+ table = Table(
523
+ show_header=True,
524
+ show_edge=False,
525
+ box=None,
526
+ padding=(0, 2),
527
+ header_style="bold blue",
528
+ )
529
+ table.add_column("ID", style="bold", no_wrap=False, overflow="fold")
530
+ table.add_column("Created", style="dim", no_wrap=False, overflow="fold")
531
+ table.add_column("Operations", no_wrap=False, overflow="fold")
532
+
533
+ for migration in chain:
534
+ # Format created_at - simpler format without seconds
535
+ created_str = migration.created_at.strftime("%Y-%m-%d %H:%M")
536
+
537
+ # Format operations - extract short names from raw ops dicts
538
+ # Use .ops (raw dicts) instead of .operations (instantiated) to avoid
539
+ # importing operation classes that might not exist
540
+ op_names = []
541
+ ops = getattr(migration, "ops", [])
542
+ for op_dict in ops:
543
+ op_type = op_dict.get("type", "unknown")
544
+ # Extract just the class name (last part after final dot)
545
+ op_short = op_type.split(".")[-1]
546
+ op_names.append(op_short)
547
+
548
+ ops_str = ", ".join(op_names)
549
+
550
+ table.add_row(migration.migration_id, created_str, ops_str)
551
+
552
+ app.console.print()
553
+ app.console.print(table)
554
+ app.console.print()
555
+
556
+
557
+ @app.command
558
+ def explain(
559
+ migration_id: Annotated[
560
+ str | None,
561
+ cyclopts.Parameter(
562
+ help="Migration ID to explain (explains latest if not specified)"
563
+ ),
564
+ ] = None,
565
+ ):
566
+ """Show detailed diff for a migration.
567
+
568
+ Reads migration from YAML file.
569
+ Computes and displays the GraphDiff between the two snapshots on-demand.
570
+
571
+ Examples:
572
+ # Explain latest migration (head of chain)
573
+ $ metaxy migrations explain
574
+
575
+ # Explain specific migration
576
+ $ metaxy migrations explain 20250113_103000
577
+ """
578
+ from pathlib import Path
579
+
580
+ from metaxy.cli.context import AppContext
581
+ from metaxy.migrations.loader import (
582
+ find_latest_migration,
583
+ find_migration_yaml,
584
+ load_migration_from_yaml,
585
+ )
586
+
587
+ context = AppContext.get()
588
+ # Get context and project from config
589
+ project = context.get_required_project() # This command needs a specific project
590
+ metadata_store = context.get_store(None)
591
+ migrations_dir = Path(".metaxy/migrations")
592
+
593
+ with metadata_store:
594
+ # Get migration ID
595
+ if migration_id is None:
596
+ # Get latest migration (head)
597
+ try:
598
+ migration_id = find_latest_migration(migrations_dir)
599
+ except ValueError as e:
600
+ app.console.print(f"[red]✗[/red] {e}")
601
+ raise SystemExit(1)
602
+
603
+ if migration_id is None:
604
+ app.console.print("[yellow]No migrations found[/yellow]")
605
+ app.console.print("Run 'metaxy migrations generate' first")
606
+ return
607
+
608
+ # Load migration from YAML
609
+ try:
610
+ yaml_path = find_migration_yaml(migration_id, migrations_dir)
611
+ migration = load_migration_from_yaml(yaml_path)
612
+ except FileNotFoundError as e:
613
+ app.console.print(f"[red]✗[/red] {e}")
614
+ raise SystemExit(1)
615
+
616
+ # Type narrow to DiffMigration for explain command
617
+ from metaxy.migrations.models import DiffMigration
618
+
619
+ if not isinstance(migration, DiffMigration):
620
+ app.console.print(
621
+ f"[red]✗[/red] Migration '{migration_id}' is not a DiffMigration"
622
+ )
623
+ app.console.print(
624
+ f" Type: {type(migration).__name__} (explain only supports DiffMigration)"
625
+ )
626
+ raise SystemExit(1)
627
+
628
+ # Print header
629
+ app.console.print(f"\n[bold]Migration: {migration_id}[/bold]")
630
+ app.console.print(f"From: {migration.from_snapshot_version}")
631
+ app.console.print(f"To: {migration.to_snapshot_version}")
632
+ app.console.print()
633
+
634
+ # Compute diff on-demand
635
+ try:
636
+ graph_diff = migration.compute_graph_diff(metadata_store, project)
637
+ except Exception as e:
638
+ from metaxy.cli.utils import print_error
639
+
640
+ print_error(app.console, "Failed to compute diff", e)
641
+ raise SystemExit(1)
642
+
643
+ # Display detailed diff
644
+ if not graph_diff.has_changes:
645
+ app.console.print("[yellow]No changes detected[/yellow]")
646
+ return
647
+
648
+ # Added nodes
649
+ if graph_diff.added_nodes:
650
+ app.console.print(
651
+ f"[green]Added Features ({len(graph_diff.added_nodes)}):[/green]"
652
+ )
653
+ for node in graph_diff.added_nodes:
654
+ app.console.print(f" ✓ {node.feature_key}")
655
+ if node.fields:
656
+ app.console.print(f" Fields ({len(node.fields)}):")
657
+ for field in node.fields[:3]:
658
+ app.console.print(
659
+ f" - {field['key']} (cv={field.get('code_version', '?')})"
660
+ )
661
+ if len(node.fields) > 3:
662
+ app.console.print(f" ... and {len(node.fields) - 3} more")
663
+ app.console.print()
664
+
665
+ # Removed nodes
666
+ if graph_diff.removed_nodes:
667
+ app.console.print(
668
+ f"[red]Removed Features ({len(graph_diff.removed_nodes)}):[/red]"
669
+ )
670
+ for node in graph_diff.removed_nodes:
671
+ app.console.print(f" ✗ {node.feature_key}")
672
+ if node.fields:
673
+ app.console.print(f" Fields ({len(node.fields)}):")
674
+ for field in node.fields[:3]:
675
+ app.console.print(
676
+ f" - {field['key']} (cv={field.get('code_version', '?')})"
677
+ )
678
+ if len(node.fields) > 3:
679
+ app.console.print(f" ... and {len(node.fields) - 3} more")
680
+ app.console.print()
681
+
682
+ # Changed nodes
683
+ if graph_diff.changed_nodes:
684
+ app.console.print(
685
+ f"[yellow]Changed Features ({len(graph_diff.changed_nodes)}):[/yellow]"
686
+ )
687
+ for node in graph_diff.changed_nodes:
688
+ app.console.print(f" ⚠ {node.feature_key}")
689
+ old_ver = node.old_version if node.old_version else "None"
690
+ new_ver = node.new_version if node.new_version else "None"
691
+ app.console.print(f" Version: {old_ver} → {new_ver}")
692
+
693
+ if (
694
+ node.old_code_version is not None
695
+ or node.new_code_version is not None
696
+ ):
697
+ app.console.print(
698
+ f" Code version: {node.old_code_version} → {node.new_code_version}"
699
+ )
700
+
701
+ # Show field changes
702
+ total_field_changes = (
703
+ len(node.added_fields)
704
+ + len(node.removed_fields)
705
+ + len(node.changed_fields)
706
+ )
707
+ if total_field_changes > 0:
708
+ app.console.print(f" Field changes ({total_field_changes}):")
709
+
710
+ if node.added_fields:
711
+ app.console.print(
712
+ f" [green]Added ({len(node.added_fields)}):[/green]"
713
+ )
714
+ for field in node.added_fields[:2]:
715
+ app.console.print(
716
+ f" + {field.field_key} (cv={field.new_code_version})"
717
+ )
718
+ if len(node.added_fields) > 2:
719
+ app.console.print(
720
+ f" ... and {len(node.added_fields) - 2} more"
721
+ )
722
+
723
+ if node.removed_fields:
724
+ app.console.print(
725
+ f" [red]Removed ({len(node.removed_fields)}):[/red]"
726
+ )
727
+ for field in node.removed_fields[:2]:
728
+ app.console.print(
729
+ f" - {field.field_key} (cv={field.old_code_version})"
730
+ )
731
+ if len(node.removed_fields) > 2:
732
+ app.console.print(
733
+ f" ... and {len(node.removed_fields) - 2} more"
734
+ )
735
+
736
+ if node.changed_fields:
737
+ app.console.print(
738
+ f" [yellow]Changed ({len(node.changed_fields)}):[/yellow]"
739
+ )
740
+ for field in node.changed_fields[:2]:
741
+ app.console.print(
742
+ f" ~ {field.field_key} (cv={field.old_code_version}→{field.new_code_version})"
743
+ )
744
+ if len(node.changed_fields) > 2:
745
+ app.console.print(
746
+ f" ... and {len(node.changed_fields) - 2} more"
747
+ )
748
+
749
+ app.console.print()
750
+
751
+ # Print affected features summary (computed on-demand)
752
+ affected_features = migration.get_affected_features(metadata_store, project)
753
+ app.console.print(f"[bold]Affected Features ({len(affected_features)}):[/bold]")
754
+ for feature_key in affected_features[:10]:
755
+ app.console.print(f" • {feature_key}")
756
+ if len(affected_features) > 10:
757
+ app.console.print(f" ... and {len(affected_features) - 10} more")
758
+
759
+
760
+ @app.command
761
+ def describe(
762
+ migration_ids: Annotated[
763
+ list[str],
764
+ cyclopts.Parameter(
765
+ help="Migration IDs to describe (default: all migrations in order)"
766
+ ),
767
+ ] = [],
768
+ store: Annotated[
769
+ str | None,
770
+ cyclopts.Parameter(None, help="Metadata store to use."),
771
+ ] = None,
772
+ ):
773
+ """Show verbose description of migration(s).
774
+
775
+ Displays detailed information about what the migration will do:
776
+ - Migration metadata (ID, parent, snapshots, created timestamp)
777
+ - Operations to execute
778
+ - Affected features with row counts
779
+ - Execution status if already run
780
+
781
+ Examples:
782
+ # Describe all migrations in chain order
783
+ $ metaxy migrations describe
784
+
785
+ # Describe specific migration
786
+ $ metaxy migrations describe 20250127_120000
787
+
788
+ # Describe multiple migrations
789
+ $ metaxy migrations describe 20250101_120000 20250102_090000
790
+ """
791
+ from pathlib import Path
792
+
793
+ from metaxy.cli.context import AppContext
794
+ from metaxy.metadata_store.system import SystemTableStorage
795
+ from metaxy.migrations.loader import (
796
+ build_migration_chain,
797
+ find_migration_yaml,
798
+ load_migration_from_yaml,
799
+ )
800
+
801
+ context = AppContext.get()
802
+ # Get context and project from config
803
+ project = context.get_required_project() # This command needs a specific project
804
+ metadata_store = context.get_store(store)
805
+ migrations_dir = Path(".metaxy/migrations")
806
+
807
+ with metadata_store:
808
+ storage = SystemTableStorage(metadata_store)
809
+
810
+ # Determine which migrations to describe
811
+ if not migration_ids:
812
+ # Default: describe all migrations in chain order
813
+ try:
814
+ chain = build_migration_chain(migrations_dir)
815
+ except ValueError as e:
816
+ from metaxy.cli.utils import print_error
817
+
818
+ print_error(app.console, "Invalid migration chain", e)
819
+ raise SystemExit(1)
820
+
821
+ if not chain:
822
+ app.console.print("[yellow]No migrations found.[/yellow]")
823
+ return
824
+
825
+ migrations_to_describe = chain
826
+ else:
827
+ # Load specific migrations
828
+ migrations_to_describe = []
829
+ for migration_id in migration_ids:
830
+ try:
831
+ yaml_path = find_migration_yaml(migration_id, migrations_dir)
832
+ migration_obj = load_migration_from_yaml(yaml_path)
833
+ migrations_to_describe.append(migration_obj)
834
+ except FileNotFoundError as e:
835
+ app.console.print(f"[red]✗[/red] {e}")
836
+ raise SystemExit(1)
837
+
838
+ # Describe each migration
839
+ for i, migration_obj in enumerate(migrations_to_describe):
840
+ if i > 0:
841
+ app.console.print("\n") # Separator between migrations
842
+
843
+ # Find YAML path for display
844
+ yaml_path = find_migration_yaml(migration_obj.migration_id, migrations_dir)
845
+
846
+ # Print header
847
+ app.console.print("\n[bold]Migration Description[/bold]")
848
+ app.console.print("─" * 60)
849
+ app.console.print(f"[bold]ID:[/bold] {migration_obj.migration_id}")
850
+ app.console.print(
851
+ f"[bold]Created:[/bold] {migration_obj.created_at.isoformat()}"
852
+ )
853
+ app.console.print(f"[bold]Parent:[/bold] {migration_obj.parent}")
854
+ app.console.print(f"[bold]YAML:[/bold] {yaml_path}")
855
+ app.console.print()
856
+
857
+ # Snapshots (for DiffMigration)
858
+ from metaxy.migrations.models import DiffMigration, FullGraphMigration
859
+
860
+ if isinstance(migration_obj, DiffMigration):
861
+ app.console.print("[bold]Snapshots:[/bold]")
862
+ app.console.print(f" From: {migration_obj.from_snapshot_version}")
863
+ app.console.print(f" To: {migration_obj.to_snapshot_version}")
864
+ app.console.print()
865
+
866
+ # Operations (for FullGraphMigration)
867
+ if isinstance(migration_obj, FullGraphMigration):
868
+ app.console.print("[bold]Operations:[/bold]")
869
+ for j, op in enumerate(migration_obj.ops, 1):
870
+ op_type = op.get("type", "unknown")
871
+ app.console.print(f" {j}. {op_type}")
872
+ app.console.print()
873
+
874
+ # Get affected features
875
+ app.console.print("[bold]Computing affected features...[/bold]")
876
+ try:
877
+ affected_features = migration_obj.get_affected_features(
878
+ metadata_store, project
879
+ )
880
+ except Exception as e:
881
+ app.console.print(
882
+ f"[red]✗[/red] Failed to compute affected features: {e}"
883
+ )
884
+ continue # Skip to next migration
885
+
886
+ app.console.print(
887
+ f"\n[bold]Affected Features ({len(affected_features)}):[/bold]"
888
+ )
889
+ app.console.print("─" * 60)
890
+
891
+ from metaxy.models.feature import FeatureGraph
892
+ from metaxy.models.types import FeatureKey
893
+
894
+ graph = FeatureGraph.get_active()
895
+
896
+ # Get events for this migration to extract rows_affected per feature
897
+ events_df = storage.get_migration_events(
898
+ migration_obj.migration_id, project
899
+ )
900
+
901
+ for feature_key_str in affected_features:
902
+ feature_key_obj = FeatureKey(feature_key_str.split("/"))
903
+
904
+ # Get feature class
905
+ if feature_key_obj not in graph.features_by_key:
906
+ app.console.print(f"[yellow]⚠[/yellow] {feature_key_str}")
907
+ app.console.print(
908
+ " [yellow]Feature not in current graph[/yellow]"
909
+ )
910
+ continue
911
+
912
+ graph.features_by_key[feature_key_obj]
913
+
914
+ # Get rows affected from events (sum of all completed events for this feature)
915
+ import polars as pl
916
+
917
+ feature_events = events_df.filter(
918
+ (pl.col("feature_key") == feature_key_str)
919
+ & (pl.col("event_type") == "feature_migration_completed")
920
+ )
921
+
922
+ if feature_events.height > 0:
923
+ # Extract rows_affected from JSON payload
924
+ feature_events = feature_events.with_columns(
925
+ pl.col("payload")
926
+ .str.json_path_match("$.rows_affected")
927
+ .cast(pl.Int64, strict=False)
928
+ .fill_null(0)
929
+ .alias("rows_affected")
930
+ )
931
+ rows_affected = int(feature_events["rows_affected"].sum())
932
+
933
+ # Count number of attempts (feature_migration_started events)
934
+ attempts = events_df.filter(
935
+ (pl.col("feature_key") == feature_key_str)
936
+ & (pl.col("event_type") == "feature_migration_started")
937
+ ).height
938
+ else:
939
+ rows_affected = 0
940
+ attempts = 0
941
+
942
+ # Check if feature has upstream dependencies
943
+ plan = graph.get_feature_plan(feature_key_obj)
944
+ has_upstream = plan.deps is not None and len(plan.deps) > 0
945
+
946
+ app.console.print(f"[bold]{feature_key_str}[/bold]")
947
+ app.console.print(f" Rows Affected: {rows_affected}")
948
+ app.console.print(f" Attempts: {attempts}")
949
+ app.console.print(f" Has upstream: {has_upstream}")
950
+
951
+ app.console.print()
952
+
953
+ # Execution status
954
+ from metaxy.metadata_store.system import MigrationStatus
955
+
956
+ summary = storage.get_migration_summary(
957
+ migration_obj.migration_id,
958
+ project,
959
+ expected_features=affected_features,
960
+ )
961
+ migration_status = summary["status"]
962
+ completed_features = summary["completed_features"]
963
+ failed_features = summary["failed_features"]
964
+
965
+ app.console.print("[bold]Execution Status:[/bold]")
966
+ if migration_status == MigrationStatus.COMPLETED:
967
+ app.console.print(" [green]✓ COMPLETED[/green]")
968
+ app.console.print(
969
+ f" Features processed: {len(completed_features)}/{len(affected_features)}"
970
+ )
971
+ elif migration_status == MigrationStatus.FAILED:
972
+ app.console.print(" [red]✗ FAILED[/red]")
973
+ app.console.print(
974
+ f" Features completed: {len(completed_features)}/{len(affected_features)}"
975
+ )
976
+ app.console.print(f" Features failed: {len(failed_features)}")
977
+ if failed_features:
978
+ from metaxy.cli.utils import print_error_list
979
+
980
+ print_error_list(
981
+ app.console,
982
+ failed_features,
983
+ header=" Failed features:",
984
+ prefix=" •",
985
+ indent=" ",
986
+ max_items=5,
987
+ )
988
+ elif migration_status == MigrationStatus.IN_PROGRESS:
989
+ app.console.print(" [yellow]⚠ IN PROGRESS[/yellow]")
990
+ app.console.print(
991
+ f" Features completed: {len(completed_features)}/{len(affected_features)}"
992
+ )
993
+ else:
994
+ app.console.print(" [blue]○ NOT STARTED[/blue]")
995
+ app.console.print(f" Features to process: {len(affected_features)}")
996
+
997
+
998
+ if __name__ == "__main__":
999
+ app()