metaxy 0.0.1.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. metaxy/__init__.py +170 -0
  2. metaxy/_packaging.py +96 -0
  3. metaxy/_testing/__init__.py +55 -0
  4. metaxy/_testing/config.py +43 -0
  5. metaxy/_testing/metaxy_project.py +780 -0
  6. metaxy/_testing/models.py +111 -0
  7. metaxy/_testing/parametric/__init__.py +13 -0
  8. metaxy/_testing/parametric/metadata.py +664 -0
  9. metaxy/_testing/pytest_helpers.py +74 -0
  10. metaxy/_testing/runbook.py +533 -0
  11. metaxy/_utils.py +35 -0
  12. metaxy/_version.py +1 -0
  13. metaxy/cli/app.py +97 -0
  14. metaxy/cli/console.py +13 -0
  15. metaxy/cli/context.py +167 -0
  16. metaxy/cli/graph.py +610 -0
  17. metaxy/cli/graph_diff.py +290 -0
  18. metaxy/cli/list.py +46 -0
  19. metaxy/cli/metadata.py +317 -0
  20. metaxy/cli/migrations.py +999 -0
  21. metaxy/cli/utils.py +268 -0
  22. metaxy/config.py +680 -0
  23. metaxy/entrypoints.py +296 -0
  24. metaxy/ext/__init__.py +1 -0
  25. metaxy/ext/dagster/__init__.py +54 -0
  26. metaxy/ext/dagster/constants.py +10 -0
  27. metaxy/ext/dagster/dagster_type.py +156 -0
  28. metaxy/ext/dagster/io_manager.py +200 -0
  29. metaxy/ext/dagster/metaxify.py +512 -0
  30. metaxy/ext/dagster/observable.py +115 -0
  31. metaxy/ext/dagster/resources.py +27 -0
  32. metaxy/ext/dagster/selection.py +73 -0
  33. metaxy/ext/dagster/table_metadata.py +417 -0
  34. metaxy/ext/dagster/utils.py +462 -0
  35. metaxy/ext/sqlalchemy/__init__.py +23 -0
  36. metaxy/ext/sqlalchemy/config.py +29 -0
  37. metaxy/ext/sqlalchemy/plugin.py +353 -0
  38. metaxy/ext/sqlmodel/__init__.py +13 -0
  39. metaxy/ext/sqlmodel/config.py +29 -0
  40. metaxy/ext/sqlmodel/plugin.py +499 -0
  41. metaxy/graph/__init__.py +29 -0
  42. metaxy/graph/describe.py +325 -0
  43. metaxy/graph/diff/__init__.py +21 -0
  44. metaxy/graph/diff/diff_models.py +446 -0
  45. metaxy/graph/diff/differ.py +769 -0
  46. metaxy/graph/diff/models.py +443 -0
  47. metaxy/graph/diff/rendering/__init__.py +18 -0
  48. metaxy/graph/diff/rendering/base.py +323 -0
  49. metaxy/graph/diff/rendering/cards.py +188 -0
  50. metaxy/graph/diff/rendering/formatter.py +805 -0
  51. metaxy/graph/diff/rendering/graphviz.py +246 -0
  52. metaxy/graph/diff/rendering/mermaid.py +326 -0
  53. metaxy/graph/diff/rendering/rich.py +169 -0
  54. metaxy/graph/diff/rendering/theme.py +48 -0
  55. metaxy/graph/diff/traversal.py +247 -0
  56. metaxy/graph/status.py +329 -0
  57. metaxy/graph/utils.py +58 -0
  58. metaxy/metadata_store/__init__.py +32 -0
  59. metaxy/metadata_store/_ducklake_support.py +419 -0
  60. metaxy/metadata_store/base.py +1792 -0
  61. metaxy/metadata_store/bigquery.py +354 -0
  62. metaxy/metadata_store/clickhouse.py +184 -0
  63. metaxy/metadata_store/delta.py +371 -0
  64. metaxy/metadata_store/duckdb.py +446 -0
  65. metaxy/metadata_store/exceptions.py +61 -0
  66. metaxy/metadata_store/ibis.py +542 -0
  67. metaxy/metadata_store/lancedb.py +391 -0
  68. metaxy/metadata_store/memory.py +292 -0
  69. metaxy/metadata_store/system/__init__.py +57 -0
  70. metaxy/metadata_store/system/events.py +264 -0
  71. metaxy/metadata_store/system/keys.py +9 -0
  72. metaxy/metadata_store/system/models.py +129 -0
  73. metaxy/metadata_store/system/storage.py +957 -0
  74. metaxy/metadata_store/types.py +10 -0
  75. metaxy/metadata_store/utils.py +104 -0
  76. metaxy/metadata_store/warnings.py +36 -0
  77. metaxy/migrations/__init__.py +32 -0
  78. metaxy/migrations/detector.py +291 -0
  79. metaxy/migrations/executor.py +516 -0
  80. metaxy/migrations/generator.py +319 -0
  81. metaxy/migrations/loader.py +231 -0
  82. metaxy/migrations/models.py +528 -0
  83. metaxy/migrations/ops.py +447 -0
  84. metaxy/models/__init__.py +0 -0
  85. metaxy/models/bases.py +12 -0
  86. metaxy/models/constants.py +139 -0
  87. metaxy/models/feature.py +1335 -0
  88. metaxy/models/feature_spec.py +338 -0
  89. metaxy/models/field.py +263 -0
  90. metaxy/models/fields_mapping.py +307 -0
  91. metaxy/models/filter_expression.py +297 -0
  92. metaxy/models/lineage.py +285 -0
  93. metaxy/models/plan.py +232 -0
  94. metaxy/models/types.py +475 -0
  95. metaxy/py.typed +0 -0
  96. metaxy/utils/__init__.py +1 -0
  97. metaxy/utils/constants.py +2 -0
  98. metaxy/utils/exceptions.py +23 -0
  99. metaxy/utils/hashing.py +230 -0
  100. metaxy/versioning/__init__.py +31 -0
  101. metaxy/versioning/engine.py +656 -0
  102. metaxy/versioning/feature_dep_transformer.py +151 -0
  103. metaxy/versioning/ibis.py +249 -0
  104. metaxy/versioning/lineage_handler.py +205 -0
  105. metaxy/versioning/polars.py +189 -0
  106. metaxy/versioning/renamed_df.py +35 -0
  107. metaxy/versioning/types.py +63 -0
  108. metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
  109. metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
  110. metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
  111. metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
@@ -0,0 +1,323 @@
1
+ """Base classes and configuration for graph rendering."""
2
+
3
+ from dataclasses import dataclass, field
4
+
5
+ from metaxy.graph.diff.models import GraphData, NodeStatus
6
+ from metaxy.graph.diff.rendering.theme import Theme
7
+ from metaxy.graph.diff.traversal import GraphWalker
8
+ from metaxy.graph.utils import format_feature_key, format_field_key, format_hash
9
+ from metaxy.models.feature import FeatureGraph
10
+ from metaxy.models.types import FeatureKey, FieldKey
11
+
12
+
13
+ @dataclass(kw_only=True)
14
+ class RenderConfig:
15
+ """Configuration for graph rendering.
16
+
17
+ Controls what information is displayed and how it's formatted.
18
+ """
19
+
20
+ # What to show
21
+ show_fields: bool = field(
22
+ default=True,
23
+ metadata={"help": "Show field-level details within features"},
24
+ )
25
+
26
+ show_feature_versions: bool = field(
27
+ default=True,
28
+ metadata={"help": "Show feature version hashes"},
29
+ )
30
+
31
+ show_field_versions: bool = field(
32
+ default=True,
33
+ metadata={"help": "Show field version hashes (requires --show-fields)"},
34
+ )
35
+
36
+ show_code_versions: bool = field(
37
+ default=False,
38
+ metadata={"help": "Show feature and field code versions"},
39
+ )
40
+
41
+ show_snapshot_version: bool = field(
42
+ default=True,
43
+ metadata={"help": "Show graph snapshot version in output"},
44
+ )
45
+
46
+ # Display options
47
+ hash_length: int = field(
48
+ default=8,
49
+ metadata={
50
+ "help": "Number of characters to show for version hashes (0 for full)"
51
+ },
52
+ )
53
+
54
+ direction: str = field(
55
+ default="TB",
56
+ metadata={"help": "Graph layout direction: TB (top-bottom) or LR (left-right)"},
57
+ )
58
+
59
+ # Filtering options
60
+ feature: str | None = field(
61
+ default=None,
62
+ metadata={
63
+ "help": "Focus on a specific feature (e.g., 'video/files' or 'video__files')"
64
+ },
65
+ )
66
+
67
+ up: int | None = field(
68
+ default=None,
69
+ metadata={
70
+ "help": "Number of dependency levels to render upstream (default: all)"
71
+ },
72
+ )
73
+
74
+ down: int | None = field(
75
+ default=None,
76
+ metadata={
77
+ "help": "Number of dependency levels to render downstream (default: all)"
78
+ },
79
+ )
80
+
81
+ project: str | None = field(
82
+ default=None,
83
+ metadata={
84
+ "help": "Filter nodes by project (show only features from this project)"
85
+ },
86
+ )
87
+
88
+ show_projects: bool = field(
89
+ default=True,
90
+ metadata={"help": "Show project names in feature nodes"},
91
+ )
92
+
93
+ def get_feature_key(self) -> FeatureKey | None:
94
+ """Parse feature string into FeatureKey.
95
+
96
+ Returns:
97
+ FeatureKey if feature is set, None otherwise
98
+ """
99
+ if self.feature is None:
100
+ return None
101
+
102
+ # Support both formats: "video__files" or "video/files"
103
+ if "/" in self.feature:
104
+ return FeatureKey(self.feature.split("/"))
105
+ else:
106
+ return FeatureKey(self.feature.split("__"))
107
+
108
+ @classmethod
109
+ def minimal(cls, show_projects: bool = True) -> "RenderConfig":
110
+ """Preset: minimal information (structure only)."""
111
+ return cls(
112
+ show_fields=True,
113
+ show_feature_versions=False,
114
+ show_field_versions=False,
115
+ show_code_versions=False,
116
+ show_snapshot_version=False,
117
+ show_projects=show_projects,
118
+ )
119
+
120
+ @classmethod
121
+ def default(cls) -> "RenderConfig":
122
+ """Preset: default information level (balanced)."""
123
+ return cls(
124
+ show_fields=True,
125
+ show_feature_versions=True,
126
+ show_field_versions=True,
127
+ show_code_versions=False,
128
+ show_snapshot_version=True,
129
+ hash_length=8,
130
+ )
131
+
132
+ @classmethod
133
+ def verbose(cls, show_projects: bool = True) -> "RenderConfig":
134
+ """Preset: maximum information (everything)."""
135
+ return cls(
136
+ show_fields=True,
137
+ show_feature_versions=True,
138
+ show_field_versions=True,
139
+ show_code_versions=True,
140
+ show_snapshot_version=True,
141
+ hash_length=0, # Full hashes
142
+ show_projects=show_projects,
143
+ )
144
+
145
+
146
+ class BaseRenderer:
147
+ """Base class for graph renderers.
148
+
149
+ Provides common utilities for formatting keys and hashes.
150
+ Uses unified GraphData model and Theme system.
151
+ """
152
+
153
+ def __init__(
154
+ self,
155
+ graph: FeatureGraph | None = None,
156
+ config: RenderConfig | None = None,
157
+ graph_data: GraphData | None = None,
158
+ theme: Theme | None = None,
159
+ ):
160
+ """Initialize renderer.
161
+
162
+ Args:
163
+ graph: FeatureGraph (converted to GraphData automatically)
164
+ config: Render configuration
165
+ graph_data: GraphData (alternative to feature graph)
166
+ theme: Color theme (uses default if None)
167
+
168
+ Note:
169
+ Either graph or graph_data must be provided.
170
+ If both are provided, an error is raised.
171
+ """
172
+ if graph_data is None and graph is None:
173
+ raise ValueError("Either graph or graph_data must be provided")
174
+
175
+ # Prefer graph_data if provided, otherwise convert from graph
176
+ if graph_data is not None:
177
+ self.graph_data: GraphData = graph_data
178
+ else:
179
+ # graph is not None (validated above)
180
+ assert graph is not None
181
+ self.graph_data = GraphData.from_feature_graph(graph)
182
+
183
+ self.config = config or RenderConfig()
184
+ self.theme = theme or Theme.default()
185
+ self.walker = GraphWalker(self.graph_data)
186
+
187
+ def _format_hash(self, hash_str: str | None) -> str:
188
+ """Format hash according to config.
189
+
190
+ Args:
191
+ hash_str: Full hash string (or None for removed nodes)
192
+
193
+ Returns:
194
+ Truncated hash if hash_length > 0, otherwise full hash
195
+ """
196
+ if hash_str is None:
197
+ return "none"
198
+ return format_hash(hash_str, length=self.config.hash_length)
199
+
200
+ def _get_status_color(self, status: NodeStatus) -> str:
201
+ """Get color for a given status.
202
+
203
+ Args:
204
+ status: Node or field status
205
+
206
+ Returns:
207
+ Color string for Rich markup
208
+ """
209
+ if status == NodeStatus.ADDED:
210
+ return self.theme.added_color
211
+ elif status == NodeStatus.REMOVED:
212
+ return self.theme.removed_color
213
+ elif status == NodeStatus.CHANGED:
214
+ return self.theme.changed_color
215
+ elif status == NodeStatus.UNCHANGED:
216
+ return self.theme.unchanged_color
217
+ else:
218
+ return self.theme.feature_color
219
+
220
+ def _format_version_transition(
221
+ self, old_version: str | None, new_version: str | None
222
+ ) -> str:
223
+ """Format version transition for diff display.
224
+
225
+ Args:
226
+ old_version: Old version hash
227
+ new_version: New version hash
228
+
229
+ Returns:
230
+ Formatted string like "old... → new..."
231
+ """
232
+ old_str = self._format_hash(old_version)
233
+ new_str = self._format_hash(new_version)
234
+ return (
235
+ f"[{self.theme.old_version_color}]{old_str}[/{self.theme.old_version_color}]... → "
236
+ f"[{self.theme.new_version_color}]{new_str}[/{self.theme.new_version_color}]..."
237
+ )
238
+
239
+ def _format_feature_key(self, key: FeatureKey) -> str:
240
+ """Format feature key for display.
241
+
242
+ Uses / separator instead of __ for better readability.
243
+
244
+ Args:
245
+ key: Feature key
246
+
247
+ Returns:
248
+ Formatted string like "my/feature/key"
249
+ """
250
+ return format_feature_key(key)
251
+
252
+ def _format_field_key(self, key: FieldKey) -> str:
253
+ """Format field key for display.
254
+
255
+ Args:
256
+ key: Field key
257
+
258
+ Returns:
259
+ Formatted string like "field_name"
260
+ """
261
+ return format_field_key(key)
262
+
263
+ def _get_filtered_graph_data(self) -> GraphData:
264
+ """Get filtered graph data based on config filters.
265
+
266
+ Returns:
267
+ GraphData with only filtered nodes and edges
268
+ """
269
+ graph_data = self.graph_data
270
+
271
+ # Apply project filter if specified
272
+ if self.config.project is not None:
273
+ filtered_nodes = {}
274
+ for key, node in graph_data.nodes.items():
275
+ # Include node if it matches the project or if it's a parent of a matching node
276
+ if node.project == self.config.project:
277
+ filtered_nodes[key] = node
278
+ else:
279
+ # Check if this node is a parent of any node in the project
280
+ for other_node in graph_data.nodes.values():
281
+ if (
282
+ other_node.project == self.config.project
283
+ and node.key in other_node.dependencies
284
+ ):
285
+ filtered_nodes[key] = node
286
+ break
287
+
288
+ # Filter edges to only include those between filtered nodes
289
+ filtered_edges = []
290
+ filtered_keys = set(filtered_nodes.keys())
291
+ for edge in graph_data.edges:
292
+ from_key_str = edge.from_key.to_string()
293
+ to_key_str = edge.to_key.to_string()
294
+ if from_key_str in filtered_keys and to_key_str in filtered_keys:
295
+ filtered_edges.append(edge)
296
+
297
+ # Create new graph data with filtered nodes and edges
298
+ graph_data = GraphData(
299
+ nodes=filtered_nodes,
300
+ edges=filtered_edges,
301
+ snapshot_version=graph_data.snapshot_version,
302
+ old_snapshot_version=graph_data.old_snapshot_version,
303
+ )
304
+
305
+ # Apply feature focus filter if specified
306
+ focus_key = self.config.get_feature_key()
307
+ if focus_key is not None:
308
+ # Use walker to extract subgraph
309
+ return self.walker.extract_subgraph(
310
+ focus_key=focus_key,
311
+ up=self.config.up,
312
+ down=self.config.down,
313
+ )
314
+
315
+ return graph_data
316
+
317
+ def render(self) -> str:
318
+ """Render the graph and return string output.
319
+
320
+ Returns:
321
+ Rendered graph as string
322
+ """
323
+ raise NotImplementedError
@@ -0,0 +1,188 @@
1
+ """Cards renderer using Rich panels for graph visualization.
2
+
3
+ Requires rich library to be installed.
4
+ """
5
+
6
+ from metaxy.graph.diff.models import NodeStatus
7
+ from metaxy.graph.diff.rendering.base import BaseRenderer
8
+
9
+
10
+ class CardsRenderer(BaseRenderer):
11
+ """Renders graph as cards with edges for terminal display.
12
+
13
+ Uses Rich panels to show features as cards/boxes with dependency information.
14
+ Supports both normal and diff rendering via node status.
15
+ """
16
+
17
+ def render(self) -> str:
18
+ """Render graph as cards.
19
+
20
+ Returns:
21
+ Rendered cards as string with ANSI color codes
22
+ """
23
+ from rich.columns import Columns
24
+ from rich.console import Console, Group
25
+ from rich.text import Text
26
+
27
+ console = Console()
28
+
29
+ # Get filtered graph data based on config
30
+ filtered_graph = self._get_filtered_graph_data()
31
+
32
+ # Build feature panels in topological order
33
+ from metaxy.graph.diff.traversal import GraphWalker
34
+
35
+ walker = GraphWalker(filtered_graph)
36
+ feature_panels = []
37
+
38
+ for node in walker.topological_sort():
39
+ panel = self._build_feature_panel(node)
40
+ feature_panels.append(panel)
41
+
42
+ # Build edges representation
43
+ edges_text = Text()
44
+ if self.config.show_snapshot_version:
45
+ snapshot_version = self._format_hash(filtered_graph.snapshot_version)
46
+ edges_text.append(
47
+ f"📊 Graph (snapshot: {snapshot_version})\n\n", style="bold"
48
+ )
49
+ else:
50
+ edges_text.append("📊 Graph\n\n", style="bold")
51
+
52
+ # Show dependency edges
53
+ edges_text.append("Dependencies:\n", style="bold cyan")
54
+ for node in walker.topological_sort():
55
+ if node.dependencies:
56
+ source_label = self._format_feature_key(node.key)
57
+ source_color = self._get_status_color(node.status)
58
+ for dep_key in node.dependencies:
59
+ dep_node = filtered_graph.get_node(dep_key)
60
+ target_label = self._format_feature_key(dep_key)
61
+ target_color = (
62
+ self._get_status_color(dep_node.status)
63
+ if dep_node
64
+ else source_color
65
+ )
66
+ edges_text.append(f" {target_label} ", style=target_color)
67
+ edges_text.append("→", style="yellow bold")
68
+ edges_text.append(f" {source_label}\n", style=source_color)
69
+
70
+ # Combine everything
71
+ output_group = Group(
72
+ edges_text,
73
+ Text("\nFeatures:", style="bold"),
74
+ Columns(feature_panels, equal=True, expand=True),
75
+ )
76
+
77
+ # Render to string
78
+ with console.capture() as capture:
79
+ console.print(output_group)
80
+ return capture.get()
81
+
82
+ def _build_feature_panel(self, node):
83
+ """Build a Rich Panel for a feature.
84
+
85
+ Args:
86
+ node: GraphNode
87
+
88
+ Returns:
89
+ Rich Panel with feature information
90
+ """
91
+ from rich.panel import Panel
92
+ from rich.text import Text
93
+
94
+ content = Text()
95
+
96
+ # Get status color
97
+ status_color = self._get_status_color(node.status)
98
+ border_color = status_color
99
+
100
+ # Feature name
101
+ content.append(self._format_feature_key(node.key), style=f"bold {status_color}")
102
+
103
+ # Add status badge for diff mode
104
+ if node.status != NodeStatus.NORMAL:
105
+ status_badge = self._get_status_badge(node.status)
106
+ content.append(f" {status_badge}")
107
+
108
+ content.append("\n")
109
+
110
+ # Versions
111
+ if self.config.show_feature_versions:
112
+ if node.status == NodeStatus.CHANGED and node.old_version is not None:
113
+ # Show version transition for changed nodes
114
+ old_v = self._format_hash(node.old_version)
115
+ new_v = self._format_hash(node.version)
116
+ content.append(f"v: {old_v} → {new_v}", style="yellow")
117
+ else:
118
+ version = self._format_hash(node.version)
119
+ content.append(f"v: {version}", style="yellow")
120
+ content.append("\n")
121
+
122
+ if self.config.show_code_versions and node.code_version is not None:
123
+ content.append(f"cv: {node.code_version}", style="dim")
124
+ content.append("\n")
125
+
126
+ # Fields
127
+ if self.config.show_fields and node.fields:
128
+ content.append("\nFields:\n", style="bold green")
129
+ for field_node in node.fields:
130
+ field_text = self._format_field_info(field_node)
131
+ content.append(f" • {field_text}\n")
132
+
133
+ return Panel(content, border_style=border_color, padding=(0, 1))
134
+
135
+ def _format_field_info(self, field_node) -> str:
136
+ """Format field information as a string.
137
+
138
+ Args:
139
+ field_node: FieldNode
140
+
141
+ Returns:
142
+ Formatted field string
143
+ """
144
+ parts = [self._format_field_key(field_node.key)]
145
+
146
+ # Show version info
147
+ if self.config.show_field_versions:
148
+ if (
149
+ field_node.status == NodeStatus.CHANGED
150
+ and field_node.old_version is not None
151
+ ):
152
+ # Show version transition for changed fields
153
+ old_v = self._format_hash(field_node.old_version)
154
+ new_v = self._format_hash(field_node.version)
155
+ parts.append(f"(v: {old_v} → {new_v})")
156
+ else:
157
+ version = self._format_hash(field_node.version)
158
+ parts.append(f"(v: {version})")
159
+
160
+ if self.config.show_code_versions and field_node.code_version is not None:
161
+ parts.append(f"(cv: {field_node.code_version})")
162
+
163
+ # Add status badge for diff mode
164
+ if field_node.status != NodeStatus.NORMAL:
165
+ status_badge = self._get_status_badge(field_node.status)
166
+ parts.append(status_badge)
167
+
168
+ return " ".join(parts)
169
+
170
+ def _get_status_badge(self, status: NodeStatus) -> str:
171
+ """Get status badge text.
172
+
173
+ Args:
174
+ status: Node status
175
+
176
+ Returns:
177
+ Status badge string
178
+ """
179
+ if status == NodeStatus.ADDED:
180
+ return "[+]"
181
+ elif status == NodeStatus.REMOVED:
182
+ return "[-]"
183
+ elif status == NodeStatus.CHANGED:
184
+ return "[~]"
185
+ elif status == NodeStatus.UNCHANGED:
186
+ return ""
187
+ else:
188
+ return ""