metaxy 0.0.1.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. metaxy/__init__.py +170 -0
  2. metaxy/_packaging.py +96 -0
  3. metaxy/_testing/__init__.py +55 -0
  4. metaxy/_testing/config.py +43 -0
  5. metaxy/_testing/metaxy_project.py +780 -0
  6. metaxy/_testing/models.py +111 -0
  7. metaxy/_testing/parametric/__init__.py +13 -0
  8. metaxy/_testing/parametric/metadata.py +664 -0
  9. metaxy/_testing/pytest_helpers.py +74 -0
  10. metaxy/_testing/runbook.py +533 -0
  11. metaxy/_utils.py +35 -0
  12. metaxy/_version.py +1 -0
  13. metaxy/cli/app.py +97 -0
  14. metaxy/cli/console.py +13 -0
  15. metaxy/cli/context.py +167 -0
  16. metaxy/cli/graph.py +610 -0
  17. metaxy/cli/graph_diff.py +290 -0
  18. metaxy/cli/list.py +46 -0
  19. metaxy/cli/metadata.py +317 -0
  20. metaxy/cli/migrations.py +999 -0
  21. metaxy/cli/utils.py +268 -0
  22. metaxy/config.py +680 -0
  23. metaxy/entrypoints.py +296 -0
  24. metaxy/ext/__init__.py +1 -0
  25. metaxy/ext/dagster/__init__.py +54 -0
  26. metaxy/ext/dagster/constants.py +10 -0
  27. metaxy/ext/dagster/dagster_type.py +156 -0
  28. metaxy/ext/dagster/io_manager.py +200 -0
  29. metaxy/ext/dagster/metaxify.py +512 -0
  30. metaxy/ext/dagster/observable.py +115 -0
  31. metaxy/ext/dagster/resources.py +27 -0
  32. metaxy/ext/dagster/selection.py +73 -0
  33. metaxy/ext/dagster/table_metadata.py +417 -0
  34. metaxy/ext/dagster/utils.py +462 -0
  35. metaxy/ext/sqlalchemy/__init__.py +23 -0
  36. metaxy/ext/sqlalchemy/config.py +29 -0
  37. metaxy/ext/sqlalchemy/plugin.py +353 -0
  38. metaxy/ext/sqlmodel/__init__.py +13 -0
  39. metaxy/ext/sqlmodel/config.py +29 -0
  40. metaxy/ext/sqlmodel/plugin.py +499 -0
  41. metaxy/graph/__init__.py +29 -0
  42. metaxy/graph/describe.py +325 -0
  43. metaxy/graph/diff/__init__.py +21 -0
  44. metaxy/graph/diff/diff_models.py +446 -0
  45. metaxy/graph/diff/differ.py +769 -0
  46. metaxy/graph/diff/models.py +443 -0
  47. metaxy/graph/diff/rendering/__init__.py +18 -0
  48. metaxy/graph/diff/rendering/base.py +323 -0
  49. metaxy/graph/diff/rendering/cards.py +188 -0
  50. metaxy/graph/diff/rendering/formatter.py +805 -0
  51. metaxy/graph/diff/rendering/graphviz.py +246 -0
  52. metaxy/graph/diff/rendering/mermaid.py +326 -0
  53. metaxy/graph/diff/rendering/rich.py +169 -0
  54. metaxy/graph/diff/rendering/theme.py +48 -0
  55. metaxy/graph/diff/traversal.py +247 -0
  56. metaxy/graph/status.py +329 -0
  57. metaxy/graph/utils.py +58 -0
  58. metaxy/metadata_store/__init__.py +32 -0
  59. metaxy/metadata_store/_ducklake_support.py +419 -0
  60. metaxy/metadata_store/base.py +1792 -0
  61. metaxy/metadata_store/bigquery.py +354 -0
  62. metaxy/metadata_store/clickhouse.py +184 -0
  63. metaxy/metadata_store/delta.py +371 -0
  64. metaxy/metadata_store/duckdb.py +446 -0
  65. metaxy/metadata_store/exceptions.py +61 -0
  66. metaxy/metadata_store/ibis.py +542 -0
  67. metaxy/metadata_store/lancedb.py +391 -0
  68. metaxy/metadata_store/memory.py +292 -0
  69. metaxy/metadata_store/system/__init__.py +57 -0
  70. metaxy/metadata_store/system/events.py +264 -0
  71. metaxy/metadata_store/system/keys.py +9 -0
  72. metaxy/metadata_store/system/models.py +129 -0
  73. metaxy/metadata_store/system/storage.py +957 -0
  74. metaxy/metadata_store/types.py +10 -0
  75. metaxy/metadata_store/utils.py +104 -0
  76. metaxy/metadata_store/warnings.py +36 -0
  77. metaxy/migrations/__init__.py +32 -0
  78. metaxy/migrations/detector.py +291 -0
  79. metaxy/migrations/executor.py +516 -0
  80. metaxy/migrations/generator.py +319 -0
  81. metaxy/migrations/loader.py +231 -0
  82. metaxy/migrations/models.py +528 -0
  83. metaxy/migrations/ops.py +447 -0
  84. metaxy/models/__init__.py +0 -0
  85. metaxy/models/bases.py +12 -0
  86. metaxy/models/constants.py +139 -0
  87. metaxy/models/feature.py +1335 -0
  88. metaxy/models/feature_spec.py +338 -0
  89. metaxy/models/field.py +263 -0
  90. metaxy/models/fields_mapping.py +307 -0
  91. metaxy/models/filter_expression.py +297 -0
  92. metaxy/models/lineage.py +285 -0
  93. metaxy/models/plan.py +232 -0
  94. metaxy/models/types.py +475 -0
  95. metaxy/py.typed +0 -0
  96. metaxy/utils/__init__.py +1 -0
  97. metaxy/utils/constants.py +2 -0
  98. metaxy/utils/exceptions.py +23 -0
  99. metaxy/utils/hashing.py +230 -0
  100. metaxy/versioning/__init__.py +31 -0
  101. metaxy/versioning/engine.py +656 -0
  102. metaxy/versioning/feature_dep_transformer.py +151 -0
  103. metaxy/versioning/ibis.py +249 -0
  104. metaxy/versioning/lineage_handler.py +205 -0
  105. metaxy/versioning/polars.py +189 -0
  106. metaxy/versioning/renamed_df.py +35 -0
  107. metaxy/versioning/types.py +63 -0
  108. metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
  109. metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
  110. metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
  111. metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
metaxy/cli/app.py ADDED
@@ -0,0 +1,97 @@
1
+ """Main Metaxy CLI application."""
2
+
3
+ from pathlib import Path
4
+ from typing import Annotated
5
+
6
+ import cyclopts
7
+
8
+ from metaxy import init_metaxy
9
+ from metaxy._version import __version__
10
+ from metaxy.cli.console import console, error_console
11
+
12
+ # Main app
13
+ app = cyclopts.App(
14
+ name="metaxy", # pyrefly: ignore[unexpected-keyword]
15
+ version=__version__, # pyrefly: ignore[unexpected-keyword]
16
+ console=console, # pyrefly: ignore[unexpected-keyword]
17
+ error_console=error_console, # pyrefly: ignore[unexpected-keyword]
18
+ config=cyclopts.config.Env( # pyrefly: ignore[unexpected-keyword,implicit-import]
19
+ "METAXY_", # Every environment variable for setting the arguments will begin with this. # pyrefly: ignore[bad-argument-count]
20
+ ),
21
+ help_epilogue="Learn more in [Metaxy docs](https://anam-org.github.io/metaxy)",
22
+ )
23
+
24
+
25
+ @app.command
26
+ def shell():
27
+ """Start interactive shell."""
28
+ app.interactive_shell()
29
+
30
+
31
+ # Meta app for global parameters
32
+
33
+
34
+ @app.meta.default
35
+ def launcher(
36
+ *tokens: Annotated[str, cyclopts.Parameter(show=False, allow_leading_hyphen=True)],
37
+ config_file: Annotated[
38
+ Path | None,
39
+ cyclopts.Parameter(
40
+ None,
41
+ help="Global option. Path to the Metaxy configuration file. Defaults to auto-discovery.",
42
+ ),
43
+ ] = None,
44
+ project: Annotated[
45
+ str | None,
46
+ cyclopts.Parameter(
47
+ None,
48
+ help="Global option. Metaxy project to work with. Some commands may forbid setting this argument.",
49
+ ),
50
+ ] = None,
51
+ all_projects: Annotated[
52
+ bool,
53
+ cyclopts.Parameter(
54
+ name=["--all-projects"],
55
+ help="Global option. Operate on all available Metaxy projects. Some commands may forbid setting this argument.",
56
+ ),
57
+ ] = False,
58
+ ):
59
+ """Metaxy CLI.
60
+
61
+ Auto-discovers configuration (`metaxy.toml` or `pyproject.toml`) in current or parent directories.
62
+ Feature definitions are collected via [feature discovery](https://anam-org.github.io/metaxy/main/learn/feature-discovery/).
63
+ """
64
+ import logging
65
+ import os
66
+
67
+ logging.getLogger().setLevel(os.environ.get("METAXY_LOG_LEVEL", "INFO"))
68
+
69
+ # Load Metaxy configuration with parent directory search
70
+ # This handles TOML discovery, env vars, and entrypoint loading
71
+ config = init_metaxy(config_file=config_file, search_parents=True)
72
+
73
+ # Store config in context for commands to access
74
+ # Commands will instantiate and open store as needed
75
+ from metaxy.cli.context import AppContext
76
+
77
+ AppContext.set(config, cli_project=project, all_projects=all_projects)
78
+
79
+ # Run the actual command
80
+ app(tokens)
81
+
82
+
83
+ # Register subcommands (lazy loading via import strings)
84
+ app.command("metaxy.cli.migrations:app", name="migrations")
85
+ app.command("metaxy.cli.graph:app", name="graph")
86
+ app.command("metaxy.cli.graph_diff:app", name="graph-diff")
87
+ app.command("metaxy.cli.list:app", name="list")
88
+ app.command("metaxy.cli.metadata:app", name="metadata")
89
+
90
+
91
+ def main():
92
+ """Entry point for the CLI."""
93
+ app.meta()
94
+
95
+
96
+ if __name__ == "__main__":
97
+ main()
metaxy/cli/console.py ADDED
@@ -0,0 +1,13 @@
1
+ import sys
2
+
3
+ from rich.console import Console
4
+
5
+ # Rich console for logs and status messages (goes to stderr)
6
+ console = Console(file=sys.stderr, stderr=True)
7
+
8
+ # Console for data output (goes to stdout)
9
+ # This is used for outputting important data that scripts might want to capture
10
+ data_console = Console(file=sys.stdout, highlight=False)
11
+
12
+ # Error console (also goes to stderr)
13
+ error_console = Console(file=sys.stderr, stderr=True, style="bold red", highlight=False)
metaxy/cli/context.py ADDED
@@ -0,0 +1,167 @@
1
+ """CLI application context for sharing state across commands."""
2
+
3
+ from contextvars import ContextVar
4
+ from dataclasses import dataclass
5
+ from functools import cached_property
6
+ from typing import TYPE_CHECKING
7
+
8
+ if TYPE_CHECKING:
9
+ from metaxy.config import MetaxyConfig
10
+ from metaxy.metadata_store.base import MetadataStore
11
+ from metaxy.models.feature import FeatureGraph
12
+
13
+
14
+ # Context variable for storing the app context
15
+ _app_context: ContextVar["AppContext | None"] = ContextVar("_app_context", default=None)
16
+
17
+
18
+ @dataclass
19
+ class AppContext:
20
+ """CLI application context.
21
+
22
+ Stores the config initialized by the meta app launcher.
23
+ """
24
+
25
+ config: "MetaxyConfig"
26
+ cli_project: (
27
+ str | None
28
+ ) # some CLI commands can be executed with a project different from the one in the Metaxy config
29
+ all_projects: bool = False # some CLI commands can work with all projects
30
+
31
+ @classmethod
32
+ def set(
33
+ cls, config: "MetaxyConfig", cli_project: str | None, all_projects: bool = False
34
+ ) -> None:
35
+ """Initialize the app context.
36
+
37
+ Should be called at CLI startup.
38
+
39
+ Args:
40
+ config: Metaxy configuration
41
+ cli_project: CLI project override
42
+ all_projects: Whether to include all projects
43
+ """
44
+ if _app_context.get() is not None:
45
+ raise RuntimeError(
46
+ "AppContext already initialized. It is not allowed to call AppContext.set() again."
47
+ )
48
+ else:
49
+ from metaxy import load_features
50
+ from metaxy.config import MetaxyConfig
51
+
52
+ MetaxyConfig.set(config)
53
+ load_features()
54
+ _app_context.set(AppContext(config, cli_project, all_projects))
55
+
56
+ @classmethod
57
+ def get(cls) -> "AppContext":
58
+ """Get the app context.
59
+
60
+ Returns:
61
+ AppContext instance
62
+
63
+ Raises:
64
+ RuntimeError: If context not initialized
65
+ """
66
+ ctx = _app_context.get()
67
+ if ctx is None:
68
+ raise RuntimeError(
69
+ "CLI context not initialized. AppContext.set(config) should be called at CLI startup."
70
+ )
71
+ else:
72
+ return ctx
73
+
74
+ def reset(self) -> None:
75
+ """Reset the app context.
76
+
77
+ Raises:
78
+ RuntimeError: If context not initialized
79
+ """
80
+ ctx = _app_context.get()
81
+ if ctx is None:
82
+ raise RuntimeError(
83
+ "CLI context not initialized. AppContext.set(config) should be called at CLI startup."
84
+ )
85
+ else:
86
+ _app_context.set(None)
87
+
88
+ def get_store(self, name: str | None = None) -> "MetadataStore":
89
+ """Get and open a metadata store from config.
90
+
91
+ Store is retrieved from config context.
92
+
93
+ Returns:
94
+ Opened metadata store instance (within context manager)
95
+
96
+ Raises:
97
+ RuntimeError: If context not initialized
98
+ """
99
+ return self.config.get_store(name)
100
+
101
+ @cached_property
102
+ def graph(self) -> "FeatureGraph":
103
+ """Get the graph instance.
104
+
105
+ Returns:
106
+ Graph instance
107
+
108
+ Raises:
109
+ RuntimeError: If context not initialized
110
+ """
111
+ from metaxy.models.feature import FeatureGraph
112
+
113
+ return FeatureGraph.get_active()
114
+
115
+ def raise_command_cannot_override_project(self) -> None:
116
+ """Raise an error if the command cannot override project.
117
+
118
+ Raises:
119
+ SystemExit: If command cannot override project
120
+ """
121
+ if self.cli_project or self.all_projects:
122
+ from metaxy.cli.console import console
123
+
124
+ console.print(
125
+ f"[red]Error:[/red] This command can only be used with the project from Metaxy configuration: {self.config.project}.",
126
+ style="bold",
127
+ )
128
+ raise SystemExit(1)
129
+
130
+ @cached_property
131
+ def project(self) -> str | None:
132
+ """Get the project for the app invocation, in order of precedence:
133
+ - None if all projects are selected
134
+ - project from CLI input
135
+ - project from Metaxy configuration
136
+
137
+ Returns:
138
+ Project name or None if not set
139
+ """
140
+ return (
141
+ (self.cli_project or self.config.project) if not self.all_projects else None
142
+ )
143
+
144
+ def get_required_project(self) -> str:
145
+ """Get the project for commands that require a specific project.
146
+
147
+ This method ensures we have a valid project string, raising an error
148
+ if all_projects is True (which would make project None).
149
+
150
+ Returns:
151
+ Project name (never None)
152
+
153
+ Raises:
154
+ SystemExit: If all_projects is True (project would be None)
155
+ """
156
+ if self.all_projects:
157
+ from metaxy.cli.console import console
158
+
159
+ console.print(
160
+ "[red]Error:[/red] This command requires a specific project. "
161
+ "Cannot use --all-projects flag.",
162
+ style="bold",
163
+ )
164
+ raise SystemExit(1)
165
+
166
+ # Return the project (either from CLI or config, guaranteed to have a default)
167
+ return self.cli_project or self.config.project