metaxy 0.0.1.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. metaxy/__init__.py +170 -0
  2. metaxy/_packaging.py +96 -0
  3. metaxy/_testing/__init__.py +55 -0
  4. metaxy/_testing/config.py +43 -0
  5. metaxy/_testing/metaxy_project.py +780 -0
  6. metaxy/_testing/models.py +111 -0
  7. metaxy/_testing/parametric/__init__.py +13 -0
  8. metaxy/_testing/parametric/metadata.py +664 -0
  9. metaxy/_testing/pytest_helpers.py +74 -0
  10. metaxy/_testing/runbook.py +533 -0
  11. metaxy/_utils.py +35 -0
  12. metaxy/_version.py +1 -0
  13. metaxy/cli/app.py +97 -0
  14. metaxy/cli/console.py +13 -0
  15. metaxy/cli/context.py +167 -0
  16. metaxy/cli/graph.py +610 -0
  17. metaxy/cli/graph_diff.py +290 -0
  18. metaxy/cli/list.py +46 -0
  19. metaxy/cli/metadata.py +317 -0
  20. metaxy/cli/migrations.py +999 -0
  21. metaxy/cli/utils.py +268 -0
  22. metaxy/config.py +680 -0
  23. metaxy/entrypoints.py +296 -0
  24. metaxy/ext/__init__.py +1 -0
  25. metaxy/ext/dagster/__init__.py +54 -0
  26. metaxy/ext/dagster/constants.py +10 -0
  27. metaxy/ext/dagster/dagster_type.py +156 -0
  28. metaxy/ext/dagster/io_manager.py +200 -0
  29. metaxy/ext/dagster/metaxify.py +512 -0
  30. metaxy/ext/dagster/observable.py +115 -0
  31. metaxy/ext/dagster/resources.py +27 -0
  32. metaxy/ext/dagster/selection.py +73 -0
  33. metaxy/ext/dagster/table_metadata.py +417 -0
  34. metaxy/ext/dagster/utils.py +462 -0
  35. metaxy/ext/sqlalchemy/__init__.py +23 -0
  36. metaxy/ext/sqlalchemy/config.py +29 -0
  37. metaxy/ext/sqlalchemy/plugin.py +353 -0
  38. metaxy/ext/sqlmodel/__init__.py +13 -0
  39. metaxy/ext/sqlmodel/config.py +29 -0
  40. metaxy/ext/sqlmodel/plugin.py +499 -0
  41. metaxy/graph/__init__.py +29 -0
  42. metaxy/graph/describe.py +325 -0
  43. metaxy/graph/diff/__init__.py +21 -0
  44. metaxy/graph/diff/diff_models.py +446 -0
  45. metaxy/graph/diff/differ.py +769 -0
  46. metaxy/graph/diff/models.py +443 -0
  47. metaxy/graph/diff/rendering/__init__.py +18 -0
  48. metaxy/graph/diff/rendering/base.py +323 -0
  49. metaxy/graph/diff/rendering/cards.py +188 -0
  50. metaxy/graph/diff/rendering/formatter.py +805 -0
  51. metaxy/graph/diff/rendering/graphviz.py +246 -0
  52. metaxy/graph/diff/rendering/mermaid.py +326 -0
  53. metaxy/graph/diff/rendering/rich.py +169 -0
  54. metaxy/graph/diff/rendering/theme.py +48 -0
  55. metaxy/graph/diff/traversal.py +247 -0
  56. metaxy/graph/status.py +329 -0
  57. metaxy/graph/utils.py +58 -0
  58. metaxy/metadata_store/__init__.py +32 -0
  59. metaxy/metadata_store/_ducklake_support.py +419 -0
  60. metaxy/metadata_store/base.py +1792 -0
  61. metaxy/metadata_store/bigquery.py +354 -0
  62. metaxy/metadata_store/clickhouse.py +184 -0
  63. metaxy/metadata_store/delta.py +371 -0
  64. metaxy/metadata_store/duckdb.py +446 -0
  65. metaxy/metadata_store/exceptions.py +61 -0
  66. metaxy/metadata_store/ibis.py +542 -0
  67. metaxy/metadata_store/lancedb.py +391 -0
  68. metaxy/metadata_store/memory.py +292 -0
  69. metaxy/metadata_store/system/__init__.py +57 -0
  70. metaxy/metadata_store/system/events.py +264 -0
  71. metaxy/metadata_store/system/keys.py +9 -0
  72. metaxy/metadata_store/system/models.py +129 -0
  73. metaxy/metadata_store/system/storage.py +957 -0
  74. metaxy/metadata_store/types.py +10 -0
  75. metaxy/metadata_store/utils.py +104 -0
  76. metaxy/metadata_store/warnings.py +36 -0
  77. metaxy/migrations/__init__.py +32 -0
  78. metaxy/migrations/detector.py +291 -0
  79. metaxy/migrations/executor.py +516 -0
  80. metaxy/migrations/generator.py +319 -0
  81. metaxy/migrations/loader.py +231 -0
  82. metaxy/migrations/models.py +528 -0
  83. metaxy/migrations/ops.py +447 -0
  84. metaxy/models/__init__.py +0 -0
  85. metaxy/models/bases.py +12 -0
  86. metaxy/models/constants.py +139 -0
  87. metaxy/models/feature.py +1335 -0
  88. metaxy/models/feature_spec.py +338 -0
  89. metaxy/models/field.py +263 -0
  90. metaxy/models/fields_mapping.py +307 -0
  91. metaxy/models/filter_expression.py +297 -0
  92. metaxy/models/lineage.py +285 -0
  93. metaxy/models/plan.py +232 -0
  94. metaxy/models/types.py +475 -0
  95. metaxy/py.typed +0 -0
  96. metaxy/utils/__init__.py +1 -0
  97. metaxy/utils/constants.py +2 -0
  98. metaxy/utils/exceptions.py +23 -0
  99. metaxy/utils/hashing.py +230 -0
  100. metaxy/versioning/__init__.py +31 -0
  101. metaxy/versioning/engine.py +656 -0
  102. metaxy/versioning/feature_dep_transformer.py +151 -0
  103. metaxy/versioning/ibis.py +249 -0
  104. metaxy/versioning/lineage_handler.py +205 -0
  105. metaxy/versioning/polars.py +189 -0
  106. metaxy/versioning/renamed_df.py +35 -0
  107. metaxy/versioning/types.py +63 -0
  108. metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
  109. metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
  110. metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
  111. metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
metaxy/cli/utils.py ADDED
@@ -0,0 +1,268 @@
1
+ """Shared utilities for Metaxy CLI commands."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass, field
8
+ from typing import TYPE_CHECKING, Annotated, Any, Literal, NoReturn
9
+
10
+ import cyclopts
11
+ from rich.console import Console
12
+ from rich.markup import escape as escape_markup
13
+
14
+ from metaxy.cli.console import data_console
15
+ from metaxy.models.types import FeatureKey
16
+
17
+ if TYPE_CHECKING:
18
+ from metaxy.cli.context import AppContext
19
+ from metaxy.metadata_store.base import MetadataStore
20
+ from metaxy.models.feature import FeatureGraph
21
+
22
+ # Standard output format type used across CLI commands
23
+ OutputFormat = Literal["plain", "json"]
24
+
25
+
26
+ def print_error(
27
+ console: Console,
28
+ message: str,
29
+ error: str | Exception | None = None,
30
+ *,
31
+ prefix: str = "[red]✗[/red]",
32
+ ) -> None:
33
+ """Print an error message, safely escaping dynamic content.
34
+
35
+ Args:
36
+ console: Rich console to print to
37
+ message: Static message (Rich markup allowed)
38
+ error: Optional exception/error to append (will be escaped)
39
+ prefix: Symbol/text prefix (default: red ✗)
40
+ """
41
+ if error is not None:
42
+ safe_error = escape_markup(str(error))
43
+ console.print(f"{prefix} {message}: {safe_error}")
44
+ else:
45
+ console.print(f"{prefix} {message}")
46
+
47
+
48
+ def print_error_item(
49
+ console: Console,
50
+ key: str,
51
+ error: str | Exception,
52
+ *,
53
+ prefix: str = " ✗",
54
+ indent: str = "",
55
+ ) -> None:
56
+ """Print a single error item with key and error message, safely escaping markup.
57
+
58
+ Args:
59
+ console: Rich console to print to
60
+ key: The identifier (e.g., feature key) - will be escaped
61
+ error: The error message or exception - will be escaped
62
+ prefix: Symbol/text before the key (default: " ✗")
63
+ indent: Additional indentation
64
+ """
65
+ safe_key = escape_markup(str(key))
66
+ safe_error = escape_markup(str(error))
67
+ console.print(f"{indent}{prefix} {safe_key}: {safe_error}")
68
+
69
+
70
+ def print_error_list(
71
+ console: Console,
72
+ errors: Mapping[str, str | Exception],
73
+ *,
74
+ header: str | None = None,
75
+ prefix: str = " ✗",
76
+ indent: str = "",
77
+ max_items: int | None = None,
78
+ ) -> None:
79
+ """Print a list of errors with optional header, safely escaping all content.
80
+
81
+ Args:
82
+ console: Rich console to print to
83
+ errors: Mapping of keys to error messages (dict[str, str] or dict[str, Exception])
84
+ header: Optional header line with Rich markup (not escaped)
85
+ prefix: Symbol/text before each key (default: " ✗")
86
+ indent: Additional indentation for each error line
87
+ max_items: Maximum number of items to display (None = all)
88
+ """
89
+ if header:
90
+ console.print(header)
91
+
92
+ items = list(errors.items())
93
+ if max_items is not None:
94
+ items = items[:max_items]
95
+
96
+ for key, error in items:
97
+ print_error_item(console, key, error, prefix=prefix, indent=indent)
98
+
99
+ if max_items is not None and len(errors) > max_items:
100
+ remaining = len(errors) - max_items
101
+ console.print(f"{indent} ... and {remaining} more")
102
+
103
+
104
+ @dataclass
105
+ class CLIError:
106
+ """Structured CLI error that can be rendered as JSON or plain text."""
107
+
108
+ code: str # e.g., "MISSING_REQUIRED_FLAG", "CONFLICTING_FLAGS"
109
+ message: str
110
+ details: dict[str, Any] = field(default_factory=dict)
111
+ hint: str | None = None
112
+
113
+ def to_json(self) -> dict[str, Any]:
114
+ """Convert to JSON-serializable dict."""
115
+ result: dict[str, Any] = {"error": self.code, "message": self.message}
116
+ result.update(self.details)
117
+ return result
118
+
119
+ def to_plain(self) -> str:
120
+ """Convert to plain text with Rich markup.
121
+
122
+ The message and hint are escaped to prevent Rich markup injection
123
+ from error messages containing brackets (e.g., file paths like [/tmp/...]).
124
+ """
125
+ safe_message = escape_markup(self.message)
126
+ lines = [f"[red]Error:[/red] {safe_message}"]
127
+ if self.hint:
128
+ safe_hint = escape_markup(self.hint)
129
+ lines.append(f"[yellow]Hint:[/yellow] {safe_hint}")
130
+ return "\n".join(lines)
131
+
132
+
133
+ def exit_with_error(error: CLIError, output_format: OutputFormat) -> NoReturn:
134
+ """Print error in appropriate format and exit with code 1."""
135
+ if output_format == "json":
136
+ print(json.dumps(error.to_json()))
137
+ else:
138
+ data_console.print(error.to_plain())
139
+ raise SystemExit(1)
140
+
141
+
142
+ @cyclopts.Parameter(name="*")
143
+ @dataclass(kw_only=True)
144
+ class FeatureSelector:
145
+ """Encapsulates feature selection logic for CLI commands.
146
+
147
+ Handles the common pattern of --feature vs --all-features arguments.
148
+ """
149
+
150
+ features: Annotated[
151
+ list[str] | None,
152
+ cyclopts.Parameter(
153
+ name="--feature",
154
+ help="Feature key (e.g., 'my_feature' or 'namespace/feature'). Can be repeated.",
155
+ ),
156
+ ] = None
157
+ all_features: Annotated[
158
+ bool,
159
+ cyclopts.Parameter(
160
+ name="--all-features",
161
+ help="Apply to all features in the project's feature graph.",
162
+ ),
163
+ ] = False
164
+
165
+ def validate(self, output_format: OutputFormat) -> None:
166
+ """Validate that exactly one selection mode is specified."""
167
+ if not self.all_features and not self.features:
168
+ exit_with_error(
169
+ CLIError(
170
+ code="MISSING_REQUIRED_FLAG",
171
+ message="Must specify either --all-features or --feature",
172
+ details={"required_flags": ["--all-features", "--feature"]},
173
+ ),
174
+ output_format,
175
+ )
176
+ if self.all_features and self.features:
177
+ exit_with_error(
178
+ CLIError(
179
+ code="CONFLICTING_FLAGS",
180
+ message="Cannot specify both --all-features and --feature",
181
+ details={"conflicting_flags": ["--all-features", "--feature"]},
182
+ ),
183
+ output_format,
184
+ )
185
+
186
+ def resolve_keys(
187
+ self,
188
+ graph: FeatureGraph,
189
+ output_format: OutputFormat,
190
+ ) -> tuple[list[FeatureKey], list[FeatureKey]]:
191
+ """Resolve feature selection to keys.
192
+
193
+ Args:
194
+ graph: The feature graph to resolve against
195
+ output_format: Output format for error messages
196
+
197
+ Returns:
198
+ Tuple of (valid_keys, missing_keys) where:
199
+ - valid_keys: Keys that exist in the graph
200
+ - missing_keys: Keys that were requested but don't exist
201
+ """
202
+ if self.all_features:
203
+ return graph.list_features(only_current_project=True), []
204
+
205
+ # Parse explicit feature keys
206
+ parsed_keys: list[FeatureKey] = []
207
+ for raw_key in self.features or []:
208
+ try:
209
+ parsed_keys.append(FeatureKey(raw_key))
210
+ except ValueError as exc:
211
+ exit_with_error(
212
+ CLIError(
213
+ code="INVALID_FEATURE_KEY",
214
+ message=f"Invalid feature key '{raw_key}': {exc}",
215
+ details={"key": raw_key},
216
+ ),
217
+ output_format,
218
+ )
219
+
220
+ # Check which keys exist in graph
221
+ valid = [k for k in parsed_keys if k in graph.features_by_key]
222
+ missing = [k for k in parsed_keys if k not in graph.features_by_key]
223
+
224
+ return valid, missing
225
+
226
+
227
+ def load_graph_for_command(
228
+ context: AppContext,
229
+ snapshot_version: str | None,
230
+ metadata_store: MetadataStore,
231
+ output_format: OutputFormat,
232
+ ) -> FeatureGraph:
233
+ """Load feature graph from snapshot or use current.
234
+
235
+ Args:
236
+ context: CLI application context
237
+ snapshot_version: Optional snapshot version to load from
238
+ metadata_store: Store to load snapshot from
239
+ output_format: Output format for error messages
240
+
241
+ Returns:
242
+ FeatureGraph from snapshot or current context
243
+ """
244
+ if snapshot_version is None:
245
+ return context.graph
246
+
247
+ from metaxy.metadata_store.system.storage import SystemTableStorage
248
+
249
+ storage = SystemTableStorage(metadata_store)
250
+ try:
251
+ return storage.load_graph_from_snapshot(
252
+ snapshot_version=snapshot_version,
253
+ project=context.project,
254
+ )
255
+ except ValueError as e:
256
+ exit_with_error(
257
+ CLIError(code="SNAPSHOT_ERROR", message=str(e)),
258
+ output_format,
259
+ )
260
+ except ImportError as e:
261
+ exit_with_error(
262
+ CLIError(
263
+ code="SNAPSHOT_LOAD_FAILED",
264
+ message=f"Failed to load snapshot: {e}",
265
+ hint="Feature classes may have been moved or deleted.",
266
+ ),
267
+ output_format,
268
+ )