metaxy 0.0.1.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaxy/__init__.py +170 -0
- metaxy/_packaging.py +96 -0
- metaxy/_testing/__init__.py +55 -0
- metaxy/_testing/config.py +43 -0
- metaxy/_testing/metaxy_project.py +780 -0
- metaxy/_testing/models.py +111 -0
- metaxy/_testing/parametric/__init__.py +13 -0
- metaxy/_testing/parametric/metadata.py +664 -0
- metaxy/_testing/pytest_helpers.py +74 -0
- metaxy/_testing/runbook.py +533 -0
- metaxy/_utils.py +35 -0
- metaxy/_version.py +1 -0
- metaxy/cli/app.py +97 -0
- metaxy/cli/console.py +13 -0
- metaxy/cli/context.py +167 -0
- metaxy/cli/graph.py +610 -0
- metaxy/cli/graph_diff.py +290 -0
- metaxy/cli/list.py +46 -0
- metaxy/cli/metadata.py +317 -0
- metaxy/cli/migrations.py +999 -0
- metaxy/cli/utils.py +268 -0
- metaxy/config.py +680 -0
- metaxy/entrypoints.py +296 -0
- metaxy/ext/__init__.py +1 -0
- metaxy/ext/dagster/__init__.py +54 -0
- metaxy/ext/dagster/constants.py +10 -0
- metaxy/ext/dagster/dagster_type.py +156 -0
- metaxy/ext/dagster/io_manager.py +200 -0
- metaxy/ext/dagster/metaxify.py +512 -0
- metaxy/ext/dagster/observable.py +115 -0
- metaxy/ext/dagster/resources.py +27 -0
- metaxy/ext/dagster/selection.py +73 -0
- metaxy/ext/dagster/table_metadata.py +417 -0
- metaxy/ext/dagster/utils.py +462 -0
- metaxy/ext/sqlalchemy/__init__.py +23 -0
- metaxy/ext/sqlalchemy/config.py +29 -0
- metaxy/ext/sqlalchemy/plugin.py +353 -0
- metaxy/ext/sqlmodel/__init__.py +13 -0
- metaxy/ext/sqlmodel/config.py +29 -0
- metaxy/ext/sqlmodel/plugin.py +499 -0
- metaxy/graph/__init__.py +29 -0
- metaxy/graph/describe.py +325 -0
- metaxy/graph/diff/__init__.py +21 -0
- metaxy/graph/diff/diff_models.py +446 -0
- metaxy/graph/diff/differ.py +769 -0
- metaxy/graph/diff/models.py +443 -0
- metaxy/graph/diff/rendering/__init__.py +18 -0
- metaxy/graph/diff/rendering/base.py +323 -0
- metaxy/graph/diff/rendering/cards.py +188 -0
- metaxy/graph/diff/rendering/formatter.py +805 -0
- metaxy/graph/diff/rendering/graphviz.py +246 -0
- metaxy/graph/diff/rendering/mermaid.py +326 -0
- metaxy/graph/diff/rendering/rich.py +169 -0
- metaxy/graph/diff/rendering/theme.py +48 -0
- metaxy/graph/diff/traversal.py +247 -0
- metaxy/graph/status.py +329 -0
- metaxy/graph/utils.py +58 -0
- metaxy/metadata_store/__init__.py +32 -0
- metaxy/metadata_store/_ducklake_support.py +419 -0
- metaxy/metadata_store/base.py +1792 -0
- metaxy/metadata_store/bigquery.py +354 -0
- metaxy/metadata_store/clickhouse.py +184 -0
- metaxy/metadata_store/delta.py +371 -0
- metaxy/metadata_store/duckdb.py +446 -0
- metaxy/metadata_store/exceptions.py +61 -0
- metaxy/metadata_store/ibis.py +542 -0
- metaxy/metadata_store/lancedb.py +391 -0
- metaxy/metadata_store/memory.py +292 -0
- metaxy/metadata_store/system/__init__.py +57 -0
- metaxy/metadata_store/system/events.py +264 -0
- metaxy/metadata_store/system/keys.py +9 -0
- metaxy/metadata_store/system/models.py +129 -0
- metaxy/metadata_store/system/storage.py +957 -0
- metaxy/metadata_store/types.py +10 -0
- metaxy/metadata_store/utils.py +104 -0
- metaxy/metadata_store/warnings.py +36 -0
- metaxy/migrations/__init__.py +32 -0
- metaxy/migrations/detector.py +291 -0
- metaxy/migrations/executor.py +516 -0
- metaxy/migrations/generator.py +319 -0
- metaxy/migrations/loader.py +231 -0
- metaxy/migrations/models.py +528 -0
- metaxy/migrations/ops.py +447 -0
- metaxy/models/__init__.py +0 -0
- metaxy/models/bases.py +12 -0
- metaxy/models/constants.py +139 -0
- metaxy/models/feature.py +1335 -0
- metaxy/models/feature_spec.py +338 -0
- metaxy/models/field.py +263 -0
- metaxy/models/fields_mapping.py +307 -0
- metaxy/models/filter_expression.py +297 -0
- metaxy/models/lineage.py +285 -0
- metaxy/models/plan.py +232 -0
- metaxy/models/types.py +475 -0
- metaxy/py.typed +0 -0
- metaxy/utils/__init__.py +1 -0
- metaxy/utils/constants.py +2 -0
- metaxy/utils/exceptions.py +23 -0
- metaxy/utils/hashing.py +230 -0
- metaxy/versioning/__init__.py +31 -0
- metaxy/versioning/engine.py +656 -0
- metaxy/versioning/feature_dep_transformer.py +151 -0
- metaxy/versioning/ibis.py +249 -0
- metaxy/versioning/lineage_handler.py +205 -0
- metaxy/versioning/polars.py +189 -0
- metaxy/versioning/renamed_df.py +35 -0
- metaxy/versioning/types.py +63 -0
- metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
- metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
- metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
- metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
metaxy/cli/app.py
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"""Main Metaxy CLI application."""
|
|
2
|
+
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Annotated
|
|
5
|
+
|
|
6
|
+
import cyclopts
|
|
7
|
+
|
|
8
|
+
from metaxy import init_metaxy
|
|
9
|
+
from metaxy._version import __version__
|
|
10
|
+
from metaxy.cli.console import console, error_console
|
|
11
|
+
|
|
12
|
+
# Main app
|
|
13
|
+
app = cyclopts.App(
|
|
14
|
+
name="metaxy", # pyrefly: ignore[unexpected-keyword]
|
|
15
|
+
version=__version__, # pyrefly: ignore[unexpected-keyword]
|
|
16
|
+
console=console, # pyrefly: ignore[unexpected-keyword]
|
|
17
|
+
error_console=error_console, # pyrefly: ignore[unexpected-keyword]
|
|
18
|
+
config=cyclopts.config.Env( # pyrefly: ignore[unexpected-keyword,implicit-import]
|
|
19
|
+
"METAXY_", # Every environment variable for setting the arguments will begin with this. # pyrefly: ignore[bad-argument-count]
|
|
20
|
+
),
|
|
21
|
+
help_epilogue="Learn more in [Metaxy docs](https://anam-org.github.io/metaxy)",
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.command
|
|
26
|
+
def shell():
|
|
27
|
+
"""Start interactive shell."""
|
|
28
|
+
app.interactive_shell()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# Meta app for global parameters
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@app.meta.default
|
|
35
|
+
def launcher(
|
|
36
|
+
*tokens: Annotated[str, cyclopts.Parameter(show=False, allow_leading_hyphen=True)],
|
|
37
|
+
config_file: Annotated[
|
|
38
|
+
Path | None,
|
|
39
|
+
cyclopts.Parameter(
|
|
40
|
+
None,
|
|
41
|
+
help="Global option. Path to the Metaxy configuration file. Defaults to auto-discovery.",
|
|
42
|
+
),
|
|
43
|
+
] = None,
|
|
44
|
+
project: Annotated[
|
|
45
|
+
str | None,
|
|
46
|
+
cyclopts.Parameter(
|
|
47
|
+
None,
|
|
48
|
+
help="Global option. Metaxy project to work with. Some commands may forbid setting this argument.",
|
|
49
|
+
),
|
|
50
|
+
] = None,
|
|
51
|
+
all_projects: Annotated[
|
|
52
|
+
bool,
|
|
53
|
+
cyclopts.Parameter(
|
|
54
|
+
name=["--all-projects"],
|
|
55
|
+
help="Global option. Operate on all available Metaxy projects. Some commands may forbid setting this argument.",
|
|
56
|
+
),
|
|
57
|
+
] = False,
|
|
58
|
+
):
|
|
59
|
+
"""Metaxy CLI.
|
|
60
|
+
|
|
61
|
+
Auto-discovers configuration (`metaxy.toml` or `pyproject.toml`) in current or parent directories.
|
|
62
|
+
Feature definitions are collected via [feature discovery](https://anam-org.github.io/metaxy/main/learn/feature-discovery/).
|
|
63
|
+
"""
|
|
64
|
+
import logging
|
|
65
|
+
import os
|
|
66
|
+
|
|
67
|
+
logging.getLogger().setLevel(os.environ.get("METAXY_LOG_LEVEL", "INFO"))
|
|
68
|
+
|
|
69
|
+
# Load Metaxy configuration with parent directory search
|
|
70
|
+
# This handles TOML discovery, env vars, and entrypoint loading
|
|
71
|
+
config = init_metaxy(config_file=config_file, search_parents=True)
|
|
72
|
+
|
|
73
|
+
# Store config in context for commands to access
|
|
74
|
+
# Commands will instantiate and open store as needed
|
|
75
|
+
from metaxy.cli.context import AppContext
|
|
76
|
+
|
|
77
|
+
AppContext.set(config, cli_project=project, all_projects=all_projects)
|
|
78
|
+
|
|
79
|
+
# Run the actual command
|
|
80
|
+
app(tokens)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# Register subcommands (lazy loading via import strings)
|
|
84
|
+
app.command("metaxy.cli.migrations:app", name="migrations")
|
|
85
|
+
app.command("metaxy.cli.graph:app", name="graph")
|
|
86
|
+
app.command("metaxy.cli.graph_diff:app", name="graph-diff")
|
|
87
|
+
app.command("metaxy.cli.list:app", name="list")
|
|
88
|
+
app.command("metaxy.cli.metadata:app", name="metadata")
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def main():
|
|
92
|
+
"""Entry point for the CLI."""
|
|
93
|
+
app.meta()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
if __name__ == "__main__":
|
|
97
|
+
main()
|
metaxy/cli/console.py
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
|
|
3
|
+
from rich.console import Console
|
|
4
|
+
|
|
5
|
+
# Rich console for logs and status messages (goes to stderr)
|
|
6
|
+
console = Console(file=sys.stderr, stderr=True)
|
|
7
|
+
|
|
8
|
+
# Console for data output (goes to stdout)
|
|
9
|
+
# This is used for outputting important data that scripts might want to capture
|
|
10
|
+
data_console = Console(file=sys.stdout, highlight=False)
|
|
11
|
+
|
|
12
|
+
# Error console (also goes to stderr)
|
|
13
|
+
error_console = Console(file=sys.stderr, stderr=True, style="bold red", highlight=False)
|
metaxy/cli/context.py
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
"""CLI application context for sharing state across commands."""
|
|
2
|
+
|
|
3
|
+
from contextvars import ContextVar
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from functools import cached_property
|
|
6
|
+
from typing import TYPE_CHECKING
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from metaxy.config import MetaxyConfig
|
|
10
|
+
from metaxy.metadata_store.base import MetadataStore
|
|
11
|
+
from metaxy.models.feature import FeatureGraph
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
# Context variable for storing the app context
|
|
15
|
+
_app_context: ContextVar["AppContext | None"] = ContextVar("_app_context", default=None)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class AppContext:
|
|
20
|
+
"""CLI application context.
|
|
21
|
+
|
|
22
|
+
Stores the config initialized by the meta app launcher.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
config: "MetaxyConfig"
|
|
26
|
+
cli_project: (
|
|
27
|
+
str | None
|
|
28
|
+
) # some CLI commands can be executed with a project different from the one in the Metaxy config
|
|
29
|
+
all_projects: bool = False # some CLI commands can work with all projects
|
|
30
|
+
|
|
31
|
+
@classmethod
|
|
32
|
+
def set(
|
|
33
|
+
cls, config: "MetaxyConfig", cli_project: str | None, all_projects: bool = False
|
|
34
|
+
) -> None:
|
|
35
|
+
"""Initialize the app context.
|
|
36
|
+
|
|
37
|
+
Should be called at CLI startup.
|
|
38
|
+
|
|
39
|
+
Args:
|
|
40
|
+
config: Metaxy configuration
|
|
41
|
+
cli_project: CLI project override
|
|
42
|
+
all_projects: Whether to include all projects
|
|
43
|
+
"""
|
|
44
|
+
if _app_context.get() is not None:
|
|
45
|
+
raise RuntimeError(
|
|
46
|
+
"AppContext already initialized. It is not allowed to call AppContext.set() again."
|
|
47
|
+
)
|
|
48
|
+
else:
|
|
49
|
+
from metaxy import load_features
|
|
50
|
+
from metaxy.config import MetaxyConfig
|
|
51
|
+
|
|
52
|
+
MetaxyConfig.set(config)
|
|
53
|
+
load_features()
|
|
54
|
+
_app_context.set(AppContext(config, cli_project, all_projects))
|
|
55
|
+
|
|
56
|
+
@classmethod
|
|
57
|
+
def get(cls) -> "AppContext":
|
|
58
|
+
"""Get the app context.
|
|
59
|
+
|
|
60
|
+
Returns:
|
|
61
|
+
AppContext instance
|
|
62
|
+
|
|
63
|
+
Raises:
|
|
64
|
+
RuntimeError: If context not initialized
|
|
65
|
+
"""
|
|
66
|
+
ctx = _app_context.get()
|
|
67
|
+
if ctx is None:
|
|
68
|
+
raise RuntimeError(
|
|
69
|
+
"CLI context not initialized. AppContext.set(config) should be called at CLI startup."
|
|
70
|
+
)
|
|
71
|
+
else:
|
|
72
|
+
return ctx
|
|
73
|
+
|
|
74
|
+
def reset(self) -> None:
|
|
75
|
+
"""Reset the app context.
|
|
76
|
+
|
|
77
|
+
Raises:
|
|
78
|
+
RuntimeError: If context not initialized
|
|
79
|
+
"""
|
|
80
|
+
ctx = _app_context.get()
|
|
81
|
+
if ctx is None:
|
|
82
|
+
raise RuntimeError(
|
|
83
|
+
"CLI context not initialized. AppContext.set(config) should be called at CLI startup."
|
|
84
|
+
)
|
|
85
|
+
else:
|
|
86
|
+
_app_context.set(None)
|
|
87
|
+
|
|
88
|
+
def get_store(self, name: str | None = None) -> "MetadataStore":
|
|
89
|
+
"""Get and open a metadata store from config.
|
|
90
|
+
|
|
91
|
+
Store is retrieved from config context.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
Opened metadata store instance (within context manager)
|
|
95
|
+
|
|
96
|
+
Raises:
|
|
97
|
+
RuntimeError: If context not initialized
|
|
98
|
+
"""
|
|
99
|
+
return self.config.get_store(name)
|
|
100
|
+
|
|
101
|
+
@cached_property
|
|
102
|
+
def graph(self) -> "FeatureGraph":
|
|
103
|
+
"""Get the graph instance.
|
|
104
|
+
|
|
105
|
+
Returns:
|
|
106
|
+
Graph instance
|
|
107
|
+
|
|
108
|
+
Raises:
|
|
109
|
+
RuntimeError: If context not initialized
|
|
110
|
+
"""
|
|
111
|
+
from metaxy.models.feature import FeatureGraph
|
|
112
|
+
|
|
113
|
+
return FeatureGraph.get_active()
|
|
114
|
+
|
|
115
|
+
def raise_command_cannot_override_project(self) -> None:
|
|
116
|
+
"""Raise an error if the command cannot override project.
|
|
117
|
+
|
|
118
|
+
Raises:
|
|
119
|
+
SystemExit: If command cannot override project
|
|
120
|
+
"""
|
|
121
|
+
if self.cli_project or self.all_projects:
|
|
122
|
+
from metaxy.cli.console import console
|
|
123
|
+
|
|
124
|
+
console.print(
|
|
125
|
+
f"[red]Error:[/red] This command can only be used with the project from Metaxy configuration: {self.config.project}.",
|
|
126
|
+
style="bold",
|
|
127
|
+
)
|
|
128
|
+
raise SystemExit(1)
|
|
129
|
+
|
|
130
|
+
@cached_property
|
|
131
|
+
def project(self) -> str | None:
|
|
132
|
+
"""Get the project for the app invocation, in order of precedence:
|
|
133
|
+
- None if all projects are selected
|
|
134
|
+
- project from CLI input
|
|
135
|
+
- project from Metaxy configuration
|
|
136
|
+
|
|
137
|
+
Returns:
|
|
138
|
+
Project name or None if not set
|
|
139
|
+
"""
|
|
140
|
+
return (
|
|
141
|
+
(self.cli_project or self.config.project) if not self.all_projects else None
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
def get_required_project(self) -> str:
|
|
145
|
+
"""Get the project for commands that require a specific project.
|
|
146
|
+
|
|
147
|
+
This method ensures we have a valid project string, raising an error
|
|
148
|
+
if all_projects is True (which would make project None).
|
|
149
|
+
|
|
150
|
+
Returns:
|
|
151
|
+
Project name (never None)
|
|
152
|
+
|
|
153
|
+
Raises:
|
|
154
|
+
SystemExit: If all_projects is True (project would be None)
|
|
155
|
+
"""
|
|
156
|
+
if self.all_projects:
|
|
157
|
+
from metaxy.cli.console import console
|
|
158
|
+
|
|
159
|
+
console.print(
|
|
160
|
+
"[red]Error:[/red] This command requires a specific project. "
|
|
161
|
+
"Cannot use --all-projects flag.",
|
|
162
|
+
style="bold",
|
|
163
|
+
)
|
|
164
|
+
raise SystemExit(1)
|
|
165
|
+
|
|
166
|
+
# Return the project (either from CLI or config, guaranteed to have a default)
|
|
167
|
+
return self.cli_project or self.config.project
|