metaxy 0.0.1.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaxy/__init__.py +170 -0
- metaxy/_packaging.py +96 -0
- metaxy/_testing/__init__.py +55 -0
- metaxy/_testing/config.py +43 -0
- metaxy/_testing/metaxy_project.py +780 -0
- metaxy/_testing/models.py +111 -0
- metaxy/_testing/parametric/__init__.py +13 -0
- metaxy/_testing/parametric/metadata.py +664 -0
- metaxy/_testing/pytest_helpers.py +74 -0
- metaxy/_testing/runbook.py +533 -0
- metaxy/_utils.py +35 -0
- metaxy/_version.py +1 -0
- metaxy/cli/app.py +97 -0
- metaxy/cli/console.py +13 -0
- metaxy/cli/context.py +167 -0
- metaxy/cli/graph.py +610 -0
- metaxy/cli/graph_diff.py +290 -0
- metaxy/cli/list.py +46 -0
- metaxy/cli/metadata.py +317 -0
- metaxy/cli/migrations.py +999 -0
- metaxy/cli/utils.py +268 -0
- metaxy/config.py +680 -0
- metaxy/entrypoints.py +296 -0
- metaxy/ext/__init__.py +1 -0
- metaxy/ext/dagster/__init__.py +54 -0
- metaxy/ext/dagster/constants.py +10 -0
- metaxy/ext/dagster/dagster_type.py +156 -0
- metaxy/ext/dagster/io_manager.py +200 -0
- metaxy/ext/dagster/metaxify.py +512 -0
- metaxy/ext/dagster/observable.py +115 -0
- metaxy/ext/dagster/resources.py +27 -0
- metaxy/ext/dagster/selection.py +73 -0
- metaxy/ext/dagster/table_metadata.py +417 -0
- metaxy/ext/dagster/utils.py +462 -0
- metaxy/ext/sqlalchemy/__init__.py +23 -0
- metaxy/ext/sqlalchemy/config.py +29 -0
- metaxy/ext/sqlalchemy/plugin.py +353 -0
- metaxy/ext/sqlmodel/__init__.py +13 -0
- metaxy/ext/sqlmodel/config.py +29 -0
- metaxy/ext/sqlmodel/plugin.py +499 -0
- metaxy/graph/__init__.py +29 -0
- metaxy/graph/describe.py +325 -0
- metaxy/graph/diff/__init__.py +21 -0
- metaxy/graph/diff/diff_models.py +446 -0
- metaxy/graph/diff/differ.py +769 -0
- metaxy/graph/diff/models.py +443 -0
- metaxy/graph/diff/rendering/__init__.py +18 -0
- metaxy/graph/diff/rendering/base.py +323 -0
- metaxy/graph/diff/rendering/cards.py +188 -0
- metaxy/graph/diff/rendering/formatter.py +805 -0
- metaxy/graph/diff/rendering/graphviz.py +246 -0
- metaxy/graph/diff/rendering/mermaid.py +326 -0
- metaxy/graph/diff/rendering/rich.py +169 -0
- metaxy/graph/diff/rendering/theme.py +48 -0
- metaxy/graph/diff/traversal.py +247 -0
- metaxy/graph/status.py +329 -0
- metaxy/graph/utils.py +58 -0
- metaxy/metadata_store/__init__.py +32 -0
- metaxy/metadata_store/_ducklake_support.py +419 -0
- metaxy/metadata_store/base.py +1792 -0
- metaxy/metadata_store/bigquery.py +354 -0
- metaxy/metadata_store/clickhouse.py +184 -0
- metaxy/metadata_store/delta.py +371 -0
- metaxy/metadata_store/duckdb.py +446 -0
- metaxy/metadata_store/exceptions.py +61 -0
- metaxy/metadata_store/ibis.py +542 -0
- metaxy/metadata_store/lancedb.py +391 -0
- metaxy/metadata_store/memory.py +292 -0
- metaxy/metadata_store/system/__init__.py +57 -0
- metaxy/metadata_store/system/events.py +264 -0
- metaxy/metadata_store/system/keys.py +9 -0
- metaxy/metadata_store/system/models.py +129 -0
- metaxy/metadata_store/system/storage.py +957 -0
- metaxy/metadata_store/types.py +10 -0
- metaxy/metadata_store/utils.py +104 -0
- metaxy/metadata_store/warnings.py +36 -0
- metaxy/migrations/__init__.py +32 -0
- metaxy/migrations/detector.py +291 -0
- metaxy/migrations/executor.py +516 -0
- metaxy/migrations/generator.py +319 -0
- metaxy/migrations/loader.py +231 -0
- metaxy/migrations/models.py +528 -0
- metaxy/migrations/ops.py +447 -0
- metaxy/models/__init__.py +0 -0
- metaxy/models/bases.py +12 -0
- metaxy/models/constants.py +139 -0
- metaxy/models/feature.py +1335 -0
- metaxy/models/feature_spec.py +338 -0
- metaxy/models/field.py +263 -0
- metaxy/models/fields_mapping.py +307 -0
- metaxy/models/filter_expression.py +297 -0
- metaxy/models/lineage.py +285 -0
- metaxy/models/plan.py +232 -0
- metaxy/models/types.py +475 -0
- metaxy/py.typed +0 -0
- metaxy/utils/__init__.py +1 -0
- metaxy/utils/constants.py +2 -0
- metaxy/utils/exceptions.py +23 -0
- metaxy/utils/hashing.py +230 -0
- metaxy/versioning/__init__.py +31 -0
- metaxy/versioning/engine.py +656 -0
- metaxy/versioning/feature_dep_transformer.py +151 -0
- metaxy/versioning/ibis.py +249 -0
- metaxy/versioning/lineage_handler.py +205 -0
- metaxy/versioning/polars.py +189 -0
- metaxy/versioning/renamed_df.py +35 -0
- metaxy/versioning/types.py +63 -0
- metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
- metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
- metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
- metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
metaxy/cli/utils.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
"""Shared utilities for Metaxy CLI commands."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import TYPE_CHECKING, Annotated, Any, Literal, NoReturn
|
|
9
|
+
|
|
10
|
+
import cyclopts
|
|
11
|
+
from rich.console import Console
|
|
12
|
+
from rich.markup import escape as escape_markup
|
|
13
|
+
|
|
14
|
+
from metaxy.cli.console import data_console
|
|
15
|
+
from metaxy.models.types import FeatureKey
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from metaxy.cli.context import AppContext
|
|
19
|
+
from metaxy.metadata_store.base import MetadataStore
|
|
20
|
+
from metaxy.models.feature import FeatureGraph
|
|
21
|
+
|
|
22
|
+
# Standard output format type used across CLI commands
|
|
23
|
+
OutputFormat = Literal["plain", "json"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def print_error(
|
|
27
|
+
console: Console,
|
|
28
|
+
message: str,
|
|
29
|
+
error: str | Exception | None = None,
|
|
30
|
+
*,
|
|
31
|
+
prefix: str = "[red]✗[/red]",
|
|
32
|
+
) -> None:
|
|
33
|
+
"""Print an error message, safely escaping dynamic content.
|
|
34
|
+
|
|
35
|
+
Args:
|
|
36
|
+
console: Rich console to print to
|
|
37
|
+
message: Static message (Rich markup allowed)
|
|
38
|
+
error: Optional exception/error to append (will be escaped)
|
|
39
|
+
prefix: Symbol/text prefix (default: red ✗)
|
|
40
|
+
"""
|
|
41
|
+
if error is not None:
|
|
42
|
+
safe_error = escape_markup(str(error))
|
|
43
|
+
console.print(f"{prefix} {message}: {safe_error}")
|
|
44
|
+
else:
|
|
45
|
+
console.print(f"{prefix} {message}")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def print_error_item(
|
|
49
|
+
console: Console,
|
|
50
|
+
key: str,
|
|
51
|
+
error: str | Exception,
|
|
52
|
+
*,
|
|
53
|
+
prefix: str = " ✗",
|
|
54
|
+
indent: str = "",
|
|
55
|
+
) -> None:
|
|
56
|
+
"""Print a single error item with key and error message, safely escaping markup.
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
console: Rich console to print to
|
|
60
|
+
key: The identifier (e.g., feature key) - will be escaped
|
|
61
|
+
error: The error message or exception - will be escaped
|
|
62
|
+
prefix: Symbol/text before the key (default: " ✗")
|
|
63
|
+
indent: Additional indentation
|
|
64
|
+
"""
|
|
65
|
+
safe_key = escape_markup(str(key))
|
|
66
|
+
safe_error = escape_markup(str(error))
|
|
67
|
+
console.print(f"{indent}{prefix} {safe_key}: {safe_error}")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def print_error_list(
|
|
71
|
+
console: Console,
|
|
72
|
+
errors: Mapping[str, str | Exception],
|
|
73
|
+
*,
|
|
74
|
+
header: str | None = None,
|
|
75
|
+
prefix: str = " ✗",
|
|
76
|
+
indent: str = "",
|
|
77
|
+
max_items: int | None = None,
|
|
78
|
+
) -> None:
|
|
79
|
+
"""Print a list of errors with optional header, safely escaping all content.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
console: Rich console to print to
|
|
83
|
+
errors: Mapping of keys to error messages (dict[str, str] or dict[str, Exception])
|
|
84
|
+
header: Optional header line with Rich markup (not escaped)
|
|
85
|
+
prefix: Symbol/text before each key (default: " ✗")
|
|
86
|
+
indent: Additional indentation for each error line
|
|
87
|
+
max_items: Maximum number of items to display (None = all)
|
|
88
|
+
"""
|
|
89
|
+
if header:
|
|
90
|
+
console.print(header)
|
|
91
|
+
|
|
92
|
+
items = list(errors.items())
|
|
93
|
+
if max_items is not None:
|
|
94
|
+
items = items[:max_items]
|
|
95
|
+
|
|
96
|
+
for key, error in items:
|
|
97
|
+
print_error_item(console, key, error, prefix=prefix, indent=indent)
|
|
98
|
+
|
|
99
|
+
if max_items is not None and len(errors) > max_items:
|
|
100
|
+
remaining = len(errors) - max_items
|
|
101
|
+
console.print(f"{indent} ... and {remaining} more")
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass
|
|
105
|
+
class CLIError:
|
|
106
|
+
"""Structured CLI error that can be rendered as JSON or plain text."""
|
|
107
|
+
|
|
108
|
+
code: str # e.g., "MISSING_REQUIRED_FLAG", "CONFLICTING_FLAGS"
|
|
109
|
+
message: str
|
|
110
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
111
|
+
hint: str | None = None
|
|
112
|
+
|
|
113
|
+
def to_json(self) -> dict[str, Any]:
|
|
114
|
+
"""Convert to JSON-serializable dict."""
|
|
115
|
+
result: dict[str, Any] = {"error": self.code, "message": self.message}
|
|
116
|
+
result.update(self.details)
|
|
117
|
+
return result
|
|
118
|
+
|
|
119
|
+
def to_plain(self) -> str:
|
|
120
|
+
"""Convert to plain text with Rich markup.
|
|
121
|
+
|
|
122
|
+
The message and hint are escaped to prevent Rich markup injection
|
|
123
|
+
from error messages containing brackets (e.g., file paths like [/tmp/...]).
|
|
124
|
+
"""
|
|
125
|
+
safe_message = escape_markup(self.message)
|
|
126
|
+
lines = [f"[red]Error:[/red] {safe_message}"]
|
|
127
|
+
if self.hint:
|
|
128
|
+
safe_hint = escape_markup(self.hint)
|
|
129
|
+
lines.append(f"[yellow]Hint:[/yellow] {safe_hint}")
|
|
130
|
+
return "\n".join(lines)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def exit_with_error(error: CLIError, output_format: OutputFormat) -> NoReturn:
|
|
134
|
+
"""Print error in appropriate format and exit with code 1."""
|
|
135
|
+
if output_format == "json":
|
|
136
|
+
print(json.dumps(error.to_json()))
|
|
137
|
+
else:
|
|
138
|
+
data_console.print(error.to_plain())
|
|
139
|
+
raise SystemExit(1)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@cyclopts.Parameter(name="*")
|
|
143
|
+
@dataclass(kw_only=True)
|
|
144
|
+
class FeatureSelector:
|
|
145
|
+
"""Encapsulates feature selection logic for CLI commands.
|
|
146
|
+
|
|
147
|
+
Handles the common pattern of --feature vs --all-features arguments.
|
|
148
|
+
"""
|
|
149
|
+
|
|
150
|
+
features: Annotated[
|
|
151
|
+
list[str] | None,
|
|
152
|
+
cyclopts.Parameter(
|
|
153
|
+
name="--feature",
|
|
154
|
+
help="Feature key (e.g., 'my_feature' or 'namespace/feature'). Can be repeated.",
|
|
155
|
+
),
|
|
156
|
+
] = None
|
|
157
|
+
all_features: Annotated[
|
|
158
|
+
bool,
|
|
159
|
+
cyclopts.Parameter(
|
|
160
|
+
name="--all-features",
|
|
161
|
+
help="Apply to all features in the project's feature graph.",
|
|
162
|
+
),
|
|
163
|
+
] = False
|
|
164
|
+
|
|
165
|
+
def validate(self, output_format: OutputFormat) -> None:
|
|
166
|
+
"""Validate that exactly one selection mode is specified."""
|
|
167
|
+
if not self.all_features and not self.features:
|
|
168
|
+
exit_with_error(
|
|
169
|
+
CLIError(
|
|
170
|
+
code="MISSING_REQUIRED_FLAG",
|
|
171
|
+
message="Must specify either --all-features or --feature",
|
|
172
|
+
details={"required_flags": ["--all-features", "--feature"]},
|
|
173
|
+
),
|
|
174
|
+
output_format,
|
|
175
|
+
)
|
|
176
|
+
if self.all_features and self.features:
|
|
177
|
+
exit_with_error(
|
|
178
|
+
CLIError(
|
|
179
|
+
code="CONFLICTING_FLAGS",
|
|
180
|
+
message="Cannot specify both --all-features and --feature",
|
|
181
|
+
details={"conflicting_flags": ["--all-features", "--feature"]},
|
|
182
|
+
),
|
|
183
|
+
output_format,
|
|
184
|
+
)
|
|
185
|
+
|
|
186
|
+
def resolve_keys(
|
|
187
|
+
self,
|
|
188
|
+
graph: FeatureGraph,
|
|
189
|
+
output_format: OutputFormat,
|
|
190
|
+
) -> tuple[list[FeatureKey], list[FeatureKey]]:
|
|
191
|
+
"""Resolve feature selection to keys.
|
|
192
|
+
|
|
193
|
+
Args:
|
|
194
|
+
graph: The feature graph to resolve against
|
|
195
|
+
output_format: Output format for error messages
|
|
196
|
+
|
|
197
|
+
Returns:
|
|
198
|
+
Tuple of (valid_keys, missing_keys) where:
|
|
199
|
+
- valid_keys: Keys that exist in the graph
|
|
200
|
+
- missing_keys: Keys that were requested but don't exist
|
|
201
|
+
"""
|
|
202
|
+
if self.all_features:
|
|
203
|
+
return graph.list_features(only_current_project=True), []
|
|
204
|
+
|
|
205
|
+
# Parse explicit feature keys
|
|
206
|
+
parsed_keys: list[FeatureKey] = []
|
|
207
|
+
for raw_key in self.features or []:
|
|
208
|
+
try:
|
|
209
|
+
parsed_keys.append(FeatureKey(raw_key))
|
|
210
|
+
except ValueError as exc:
|
|
211
|
+
exit_with_error(
|
|
212
|
+
CLIError(
|
|
213
|
+
code="INVALID_FEATURE_KEY",
|
|
214
|
+
message=f"Invalid feature key '{raw_key}': {exc}",
|
|
215
|
+
details={"key": raw_key},
|
|
216
|
+
),
|
|
217
|
+
output_format,
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
# Check which keys exist in graph
|
|
221
|
+
valid = [k for k in parsed_keys if k in graph.features_by_key]
|
|
222
|
+
missing = [k for k in parsed_keys if k not in graph.features_by_key]
|
|
223
|
+
|
|
224
|
+
return valid, missing
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def load_graph_for_command(
|
|
228
|
+
context: AppContext,
|
|
229
|
+
snapshot_version: str | None,
|
|
230
|
+
metadata_store: MetadataStore,
|
|
231
|
+
output_format: OutputFormat,
|
|
232
|
+
) -> FeatureGraph:
|
|
233
|
+
"""Load feature graph from snapshot or use current.
|
|
234
|
+
|
|
235
|
+
Args:
|
|
236
|
+
context: CLI application context
|
|
237
|
+
snapshot_version: Optional snapshot version to load from
|
|
238
|
+
metadata_store: Store to load snapshot from
|
|
239
|
+
output_format: Output format for error messages
|
|
240
|
+
|
|
241
|
+
Returns:
|
|
242
|
+
FeatureGraph from snapshot or current context
|
|
243
|
+
"""
|
|
244
|
+
if snapshot_version is None:
|
|
245
|
+
return context.graph
|
|
246
|
+
|
|
247
|
+
from metaxy.metadata_store.system.storage import SystemTableStorage
|
|
248
|
+
|
|
249
|
+
storage = SystemTableStorage(metadata_store)
|
|
250
|
+
try:
|
|
251
|
+
return storage.load_graph_from_snapshot(
|
|
252
|
+
snapshot_version=snapshot_version,
|
|
253
|
+
project=context.project,
|
|
254
|
+
)
|
|
255
|
+
except ValueError as e:
|
|
256
|
+
exit_with_error(
|
|
257
|
+
CLIError(code="SNAPSHOT_ERROR", message=str(e)),
|
|
258
|
+
output_format,
|
|
259
|
+
)
|
|
260
|
+
except ImportError as e:
|
|
261
|
+
exit_with_error(
|
|
262
|
+
CLIError(
|
|
263
|
+
code="SNAPSHOT_LOAD_FAILED",
|
|
264
|
+
message=f"Failed to load snapshot: {e}",
|
|
265
|
+
hint="Feature classes may have been moved or deleted.",
|
|
266
|
+
),
|
|
267
|
+
output_format,
|
|
268
|
+
)
|