metaxy 0.0.1.dev3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. metaxy/__init__.py +170 -0
  2. metaxy/_packaging.py +96 -0
  3. metaxy/_testing/__init__.py +55 -0
  4. metaxy/_testing/config.py +43 -0
  5. metaxy/_testing/metaxy_project.py +780 -0
  6. metaxy/_testing/models.py +111 -0
  7. metaxy/_testing/parametric/__init__.py +13 -0
  8. metaxy/_testing/parametric/metadata.py +664 -0
  9. metaxy/_testing/pytest_helpers.py +74 -0
  10. metaxy/_testing/runbook.py +533 -0
  11. metaxy/_utils.py +35 -0
  12. metaxy/_version.py +1 -0
  13. metaxy/cli/app.py +97 -0
  14. metaxy/cli/console.py +13 -0
  15. metaxy/cli/context.py +167 -0
  16. metaxy/cli/graph.py +610 -0
  17. metaxy/cli/graph_diff.py +290 -0
  18. metaxy/cli/list.py +46 -0
  19. metaxy/cli/metadata.py +317 -0
  20. metaxy/cli/migrations.py +999 -0
  21. metaxy/cli/utils.py +268 -0
  22. metaxy/config.py +680 -0
  23. metaxy/entrypoints.py +296 -0
  24. metaxy/ext/__init__.py +1 -0
  25. metaxy/ext/dagster/__init__.py +54 -0
  26. metaxy/ext/dagster/constants.py +10 -0
  27. metaxy/ext/dagster/dagster_type.py +156 -0
  28. metaxy/ext/dagster/io_manager.py +200 -0
  29. metaxy/ext/dagster/metaxify.py +512 -0
  30. metaxy/ext/dagster/observable.py +115 -0
  31. metaxy/ext/dagster/resources.py +27 -0
  32. metaxy/ext/dagster/selection.py +73 -0
  33. metaxy/ext/dagster/table_metadata.py +417 -0
  34. metaxy/ext/dagster/utils.py +462 -0
  35. metaxy/ext/sqlalchemy/__init__.py +23 -0
  36. metaxy/ext/sqlalchemy/config.py +29 -0
  37. metaxy/ext/sqlalchemy/plugin.py +353 -0
  38. metaxy/ext/sqlmodel/__init__.py +13 -0
  39. metaxy/ext/sqlmodel/config.py +29 -0
  40. metaxy/ext/sqlmodel/plugin.py +499 -0
  41. metaxy/graph/__init__.py +29 -0
  42. metaxy/graph/describe.py +325 -0
  43. metaxy/graph/diff/__init__.py +21 -0
  44. metaxy/graph/diff/diff_models.py +446 -0
  45. metaxy/graph/diff/differ.py +769 -0
  46. metaxy/graph/diff/models.py +443 -0
  47. metaxy/graph/diff/rendering/__init__.py +18 -0
  48. metaxy/graph/diff/rendering/base.py +323 -0
  49. metaxy/graph/diff/rendering/cards.py +188 -0
  50. metaxy/graph/diff/rendering/formatter.py +805 -0
  51. metaxy/graph/diff/rendering/graphviz.py +246 -0
  52. metaxy/graph/diff/rendering/mermaid.py +326 -0
  53. metaxy/graph/diff/rendering/rich.py +169 -0
  54. metaxy/graph/diff/rendering/theme.py +48 -0
  55. metaxy/graph/diff/traversal.py +247 -0
  56. metaxy/graph/status.py +329 -0
  57. metaxy/graph/utils.py +58 -0
  58. metaxy/metadata_store/__init__.py +32 -0
  59. metaxy/metadata_store/_ducklake_support.py +419 -0
  60. metaxy/metadata_store/base.py +1792 -0
  61. metaxy/metadata_store/bigquery.py +354 -0
  62. metaxy/metadata_store/clickhouse.py +184 -0
  63. metaxy/metadata_store/delta.py +371 -0
  64. metaxy/metadata_store/duckdb.py +446 -0
  65. metaxy/metadata_store/exceptions.py +61 -0
  66. metaxy/metadata_store/ibis.py +542 -0
  67. metaxy/metadata_store/lancedb.py +391 -0
  68. metaxy/metadata_store/memory.py +292 -0
  69. metaxy/metadata_store/system/__init__.py +57 -0
  70. metaxy/metadata_store/system/events.py +264 -0
  71. metaxy/metadata_store/system/keys.py +9 -0
  72. metaxy/metadata_store/system/models.py +129 -0
  73. metaxy/metadata_store/system/storage.py +957 -0
  74. metaxy/metadata_store/types.py +10 -0
  75. metaxy/metadata_store/utils.py +104 -0
  76. metaxy/metadata_store/warnings.py +36 -0
  77. metaxy/migrations/__init__.py +32 -0
  78. metaxy/migrations/detector.py +291 -0
  79. metaxy/migrations/executor.py +516 -0
  80. metaxy/migrations/generator.py +319 -0
  81. metaxy/migrations/loader.py +231 -0
  82. metaxy/migrations/models.py +528 -0
  83. metaxy/migrations/ops.py +447 -0
  84. metaxy/models/__init__.py +0 -0
  85. metaxy/models/bases.py +12 -0
  86. metaxy/models/constants.py +139 -0
  87. metaxy/models/feature.py +1335 -0
  88. metaxy/models/feature_spec.py +338 -0
  89. metaxy/models/field.py +263 -0
  90. metaxy/models/fields_mapping.py +307 -0
  91. metaxy/models/filter_expression.py +297 -0
  92. metaxy/models/lineage.py +285 -0
  93. metaxy/models/plan.py +232 -0
  94. metaxy/models/types.py +475 -0
  95. metaxy/py.typed +0 -0
  96. metaxy/utils/__init__.py +1 -0
  97. metaxy/utils/constants.py +2 -0
  98. metaxy/utils/exceptions.py +23 -0
  99. metaxy/utils/hashing.py +230 -0
  100. metaxy/versioning/__init__.py +31 -0
  101. metaxy/versioning/engine.py +656 -0
  102. metaxy/versioning/feature_dep_transformer.py +151 -0
  103. metaxy/versioning/ibis.py +249 -0
  104. metaxy/versioning/lineage_handler.py +205 -0
  105. metaxy/versioning/polars.py +189 -0
  106. metaxy/versioning/renamed_df.py +35 -0
  107. metaxy/versioning/types.py +63 -0
  108. metaxy-0.0.1.dev3.dist-info/METADATA +96 -0
  109. metaxy-0.0.1.dev3.dist-info/RECORD +111 -0
  110. metaxy-0.0.1.dev3.dist-info/WHEEL +4 -0
  111. metaxy-0.0.1.dev3.dist-info/entry_points.txt +4 -0
@@ -0,0 +1,169 @@
1
+ """Terminal renderer using Rich Tree for hierarchical display.
2
+
3
+ Requires rich library to be installed.
4
+ """
5
+
6
+ from metaxy.graph.diff.models import NodeStatus
7
+ from metaxy.graph.diff.rendering.base import BaseRenderer
8
+
9
+
10
+ class TerminalRenderer(BaseRenderer):
11
+ """Renders graph using Rich Tree for terminal display.
12
+
13
+ Creates a hierarchical tree view with colors and icons.
14
+ Supports both normal and diff rendering via node status.
15
+ """
16
+
17
+ def render(self) -> str:
18
+ """Render graph as Rich Tree for terminal.
19
+
20
+ Returns:
21
+ Rendered tree as string with ANSI color codes
22
+ """
23
+ from rich.console import Console
24
+ from rich.tree import Tree
25
+
26
+ console = Console()
27
+
28
+ # Get filtered graph data based on config
29
+ filtered_graph = self._get_filtered_graph_data()
30
+
31
+ # Create root node
32
+ if self.config.show_snapshot_version:
33
+ snapshot_version = self._format_hash(filtered_graph.snapshot_version)
34
+ root = Tree(
35
+ f"📊 [bold]Graph[/bold] [dim](snapshot: {snapshot_version})[/dim]"
36
+ )
37
+ else:
38
+ root = Tree("📊 [bold]Graph[/bold]")
39
+
40
+ # Create walker for filtered graph and add features in topological order
41
+ from metaxy.graph.diff.traversal import GraphWalker
42
+
43
+ walker = GraphWalker(filtered_graph)
44
+ for node in walker.topological_sort():
45
+ self._render_feature_node(root, node)
46
+
47
+ # Render to string
48
+ with console.capture() as capture:
49
+ console.print(root)
50
+ return capture.get()
51
+
52
+ def _render_feature_node(self, parent, node):
53
+ """Add a feature node to the tree.
54
+
55
+ Args:
56
+ parent: Parent tree node
57
+ node: GraphNode
58
+ """
59
+ # Get status color
60
+ status_color = self._get_status_color(node.status)
61
+
62
+ # Build feature label
63
+ label_parts = [
64
+ f"[{status_color}]{self._format_feature_key(node.key)}[/{status_color}]"
65
+ ]
66
+
67
+ # Show project if configured
68
+ if self.config.show_projects and node.project:
69
+ label_parts.append(f"[dim](project: {node.project})[/dim]")
70
+
71
+ # Show version info
72
+ if self.config.show_feature_versions:
73
+ if node.status == NodeStatus.CHANGED and node.old_version is not None:
74
+ # Show version transition for changed nodes
75
+ version_transition = self._format_version_transition(
76
+ node.old_version, node.version
77
+ )
78
+ label_parts.append(version_transition)
79
+ else:
80
+ # Normal version display
81
+ version = self._format_hash(node.version)
82
+ label_parts.append(f"[yellow](v: {version})[/yellow]")
83
+
84
+ if self.config.show_code_versions and node.code_version is not None:
85
+ label_parts.append(f"[dim](cv: {node.code_version})[/dim]")
86
+
87
+ # Add status badge for diff mode
88
+ if node.status != NodeStatus.NORMAL:
89
+ status_badge = self._get_status_badge(node.status)
90
+ label_parts.append(status_badge)
91
+
92
+ label = " ".join(label_parts)
93
+ feature_branch = parent.add(label)
94
+
95
+ # Add fields
96
+ if self.config.show_fields and node.fields:
97
+ fields_branch = feature_branch.add("🔧 [green]fields[/green]")
98
+ for field_node in node.fields:
99
+ self._render_field_node(fields_branch, field_node)
100
+
101
+ # Add dependencies
102
+ if node.dependencies:
103
+ deps_branch = feature_branch.add("⬅️ [blue]depends on[/blue]")
104
+ for dep_key in node.dependencies:
105
+ dep_color = status_color # Use same color as parent for simplicity
106
+ deps_branch.add(
107
+ f"[{dep_color}]{self._format_feature_key(dep_key)}[/{dep_color}]"
108
+ )
109
+
110
+ def _render_field_node(self, parent, field_node):
111
+ """Add a field node to the tree.
112
+
113
+ Args:
114
+ parent: Parent tree node
115
+ field_node: FieldNode
116
+ """
117
+ # Get status color
118
+ status_color = self._get_status_color(field_node.status)
119
+
120
+ label_parts = [
121
+ f"[{status_color}]{self._format_field_key(field_node.key)}[/{status_color}]"
122
+ ]
123
+
124
+ # Show version info
125
+ if self.config.show_field_versions:
126
+ if (
127
+ field_node.status == NodeStatus.CHANGED
128
+ and field_node.old_version is not None
129
+ ):
130
+ # Show version transition for changed fields
131
+ version_transition = self._format_version_transition(
132
+ field_node.old_version, field_node.version
133
+ )
134
+ label_parts.append(version_transition)
135
+ else:
136
+ # Normal version display
137
+ version = self._format_hash(field_node.version)
138
+ label_parts.append(f"[yellow](v: {version})[/yellow]")
139
+
140
+ if self.config.show_code_versions and field_node.code_version is not None:
141
+ label_parts.append(f"[dim](cv: {field_node.code_version})[/dim]")
142
+
143
+ # Add status badge for diff mode
144
+ if field_node.status != NodeStatus.NORMAL:
145
+ status_badge = self._get_status_badge(field_node.status)
146
+ label_parts.append(status_badge)
147
+
148
+ label = " ".join(label_parts)
149
+ parent.add(label)
150
+
151
+ def _get_status_badge(self, status: NodeStatus) -> str:
152
+ """Get status badge text with color.
153
+
154
+ Args:
155
+ status: Node status
156
+
157
+ Returns:
158
+ Rich-formatted status badge
159
+ """
160
+ if status == NodeStatus.ADDED:
161
+ return f"[{self.theme.added_color}][+][/{self.theme.added_color}]"
162
+ elif status == NodeStatus.REMOVED:
163
+ return f"[{self.theme.removed_color}][-][/{self.theme.removed_color}]"
164
+ elif status == NodeStatus.CHANGED:
165
+ return f"[{self.theme.changed_color}][~][/{self.theme.changed_color}]"
166
+ elif status == NodeStatus.UNCHANGED:
167
+ return "" # No badge for unchanged
168
+ else:
169
+ return "" # No badge for normal
@@ -0,0 +1,48 @@
1
+ """Theme system for graph rendering."""
2
+
3
+ from pydantic import BaseModel, Field
4
+
5
+
6
+ class Theme(BaseModel):
7
+ """Color theme for graph rendering.
8
+
9
+ Unified theme for all rendering backends (terminal, Mermaid, Graphviz).
10
+ All colors use hex format (e.g., "#FF5733") for consistency.
11
+ Rich terminal supports hex via markup: [#FF0000]text[/#FF0000]
12
+ """
13
+
14
+ # Normal mode colors
15
+ feature_color: str = Field(
16
+ default="#00FFFF", description="Feature node color (cyan)"
17
+ )
18
+ field_color: str = Field(default="#5F87AF", description="Field color (steel blue)")
19
+ version_color: str = Field(
20
+ default="#FFFF00", description="Version info color (yellow)"
21
+ )
22
+ edge_color: str = Field(
23
+ default="#808080", description="Edge/dependency color (gray)"
24
+ )
25
+ snapshot_color: str = Field(
26
+ default="#FF00FF", description="Snapshot info color (magenta)"
27
+ )
28
+
29
+ # Diff mode - node/edge colors
30
+ added_color: str = Field(default="#00FF00", description="Added items (green)")
31
+ removed_color: str = Field(default="#FF0000", description="Removed items (red)")
32
+ changed_color: str = Field(default="#FFAA00", description="Changed items (orange)")
33
+ unchanged_color: str = Field(
34
+ default="#808080", description="Unchanged items (gray)"
35
+ )
36
+
37
+ # Version transition colors (for showing old→new in diffs)
38
+ old_version_color: str = Field(
39
+ default="#FF0000", description="Old version color (red)"
40
+ )
41
+ new_version_color: str = Field(
42
+ default="#00FF00", description="New version color (green)"
43
+ )
44
+
45
+ @classmethod
46
+ def default(cls) -> "Theme":
47
+ """Create default theme."""
48
+ return cls()
@@ -0,0 +1,247 @@
1
+ """Graph traversal utilities."""
2
+
3
+ from collections import deque
4
+
5
+ from metaxy.graph.diff.models import GraphData, GraphNode
6
+ from metaxy.models.types import FeatureKey
7
+
8
+
9
+ class GraphWalker:
10
+ """Traverses and filters graph data structures.
11
+
12
+ Provides various traversal strategies:
13
+ - Topological sort (dependencies first)
14
+ - BFS from starting node
15
+ - Subgraph extraction with up/down filtering
16
+ """
17
+
18
+ def __init__(self, graph_data: GraphData):
19
+ """Initialize walker with graph data.
20
+
21
+ Args:
22
+ graph_data: Graph structure to traverse
23
+ """
24
+ self.graph_data = graph_data
25
+
26
+ def topological_sort(
27
+ self, nodes_to_include: set[str] | None = None
28
+ ) -> list[GraphNode]:
29
+ """Get nodes in topological order (dependencies first).
30
+
31
+ Uses stable alphabetical ordering when multiple nodes are at the same level.
32
+ This ensures deterministic output for diff comparisons.
33
+
34
+ Args:
35
+ nodes_to_include: Optional set of feature key strings to include.
36
+ If None, includes all nodes.
37
+
38
+ Returns:
39
+ List of nodes sorted so dependencies appear before dependents
40
+ """
41
+ if nodes_to_include is None:
42
+ nodes_to_include = set(self.graph_data.nodes.keys())
43
+
44
+ visited = set()
45
+ result = []
46
+
47
+ def visit(key_str: str):
48
+ if key_str in visited or key_str not in nodes_to_include:
49
+ return
50
+ visited.add(key_str)
51
+
52
+ node = self.graph_data.nodes[key_str]
53
+
54
+ # Visit dependencies first, in sorted order for determinism
55
+ sorted_deps = sorted(
56
+ (dep_key.to_string() for dep_key in node.dependencies),
57
+ key=str.lower, # Case-insensitive sort
58
+ )
59
+ for dep_key_str in sorted_deps:
60
+ if dep_key_str in nodes_to_include:
61
+ visit(dep_key_str)
62
+
63
+ result.append(node)
64
+
65
+ # Visit all nodes in sorted order for deterministic traversal
66
+ for key_str in sorted(nodes_to_include, key=str.lower):
67
+ visit(key_str)
68
+
69
+ return result
70
+
71
+ def bfs_from(
72
+ self, start_key: FeatureKey, max_depth: int | None = None
73
+ ) -> list[GraphNode]:
74
+ """BFS traversal starting from a node.
75
+
76
+ Args:
77
+ start_key: Feature key to start from
78
+ max_depth: Maximum depth to traverse (None = unlimited)
79
+
80
+ Returns:
81
+ List of nodes in BFS order
82
+ """
83
+ start_key_str = start_key.to_string()
84
+ if start_key_str not in self.graph_data.nodes:
85
+ return []
86
+
87
+ visited = set()
88
+ result = []
89
+ queue = deque([(start_key_str, 0)]) # (key_str, depth)
90
+
91
+ while queue:
92
+ key_str, depth = queue.popleft()
93
+
94
+ if key_str in visited:
95
+ continue
96
+
97
+ if max_depth is not None and depth > max_depth:
98
+ continue
99
+
100
+ visited.add(key_str)
101
+ node = self.graph_data.nodes[key_str]
102
+ result.append(node)
103
+
104
+ # Add dependencies
105
+ for dep_key in node.dependencies:
106
+ dep_key_str = dep_key.to_string()
107
+ if dep_key_str not in visited and dep_key_str in self.graph_data.nodes:
108
+ queue.append((dep_key_str, depth + 1))
109
+
110
+ return result
111
+
112
+ def extract_subgraph(
113
+ self,
114
+ focus_key: FeatureKey,
115
+ up: int | None = None,
116
+ down: int | None = None,
117
+ ) -> GraphData:
118
+ """Extract a subgraph centered on a focus node.
119
+
120
+ Args:
121
+ focus_key: Feature to focus on
122
+ up: Number of upstream levels (dependencies) to include.
123
+ None = all, 0 = none
124
+ down: Number of downstream levels (dependents) to include.
125
+ None = all, 0 = none
126
+
127
+ Returns:
128
+ New GraphData with filtered nodes and edges
129
+
130
+ Raises:
131
+ ValueError: If focus_key not found in graph
132
+ """
133
+ focus_key_str = focus_key.to_string()
134
+ if focus_key_str not in self.graph_data.nodes:
135
+ raise ValueError(f"Feature '{focus_key_str}' not found in graph")
136
+
137
+ # Start with focus node
138
+ nodes_to_include = {focus_key_str}
139
+
140
+ # Add upstream (dependencies)
141
+ if up != 0:
142
+ max_up = None if up is None or up < 0 else up
143
+ upstream = self._get_upstream(focus_key_str, max_levels=max_up)
144
+ nodes_to_include.update(upstream)
145
+
146
+ # Add downstream (dependents)
147
+ if down != 0:
148
+ max_down = None if down is None or down < 0 else down
149
+ downstream = self._get_downstream(focus_key_str, max_levels=max_down)
150
+ nodes_to_include.update(downstream)
151
+
152
+ # Filter nodes and edges
153
+ filtered_nodes = {
154
+ k: v for k, v in self.graph_data.nodes.items() if k in nodes_to_include
155
+ }
156
+
157
+ filtered_edges = [
158
+ edge
159
+ for edge in self.graph_data.edges
160
+ if edge.from_key.to_string() in nodes_to_include
161
+ and edge.to_key.to_string() in nodes_to_include
162
+ ]
163
+
164
+ return GraphData(
165
+ nodes=filtered_nodes,
166
+ edges=filtered_edges,
167
+ snapshot_version=self.graph_data.snapshot_version,
168
+ old_snapshot_version=self.graph_data.old_snapshot_version,
169
+ )
170
+
171
+ def _get_upstream(
172
+ self, start_key_str: str, max_levels: int | None = None
173
+ ) -> set[str]:
174
+ """Get upstream features (dependencies) recursively.
175
+
176
+ Args:
177
+ start_key_str: Feature key string to start from
178
+ max_levels: Maximum levels to traverse (None = unlimited)
179
+
180
+ Returns:
181
+ Set of upstream feature key strings
182
+ """
183
+ upstream = set()
184
+
185
+ def visit(key_str: str, level: int):
186
+ if key_str not in self.graph_data.nodes:
187
+ return
188
+
189
+ node = self.graph_data.nodes[key_str]
190
+
191
+ for dep_key in node.dependencies:
192
+ dep_key_str = dep_key.to_string()
193
+ if dep_key_str not in upstream and dep_key_str in self.graph_data.nodes:
194
+ upstream.add(dep_key_str)
195
+ # Only recurse if we haven't reached max level
196
+ if max_levels is None or level + 1 < max_levels:
197
+ visit(dep_key_str, level + 1)
198
+
199
+ visit(start_key_str, 0)
200
+ return upstream
201
+
202
+ def _get_downstream(
203
+ self, start_key_str: str, max_levels: int | None = None
204
+ ) -> set[str]:
205
+ """Get downstream features (dependents) recursively.
206
+
207
+ Args:
208
+ start_key_str: Feature key string to start from
209
+ max_levels: Maximum levels to traverse (None = unlimited)
210
+
211
+ Returns:
212
+ Set of downstream feature key strings
213
+ """
214
+ # Build reverse dependency map (feature -> dependents)
215
+ dependents_map: dict[str, list[str]] = {}
216
+ for node in self.graph_data.nodes.values():
217
+ for dep_key in node.dependencies:
218
+ dep_key_str = dep_key.to_string()
219
+ if dep_key_str not in dependents_map:
220
+ dependents_map[dep_key_str] = []
221
+ dependents_map[dep_key_str].append(node.key.to_string())
222
+
223
+ downstream = set()
224
+
225
+ def visit(key_str: str, level: int):
226
+ if key_str not in dependents_map:
227
+ return
228
+
229
+ for dependent_key_str in dependents_map[key_str]:
230
+ if dependent_key_str not in downstream:
231
+ downstream.add(dependent_key_str)
232
+ # Only recurse if we haven't reached max level
233
+ if max_levels is None or level + 1 < max_levels:
234
+ visit(dependent_key_str, level + 1)
235
+
236
+ visit(start_key_str, 0)
237
+ return downstream
238
+
239
+ def get_root_nodes(self) -> list[GraphNode]:
240
+ """Get all root nodes (nodes with no dependencies).
241
+
242
+ Returns:
243
+ List of root nodes
244
+ """
245
+ return [
246
+ node for node in self.graph_data.nodes.values() if not node.dependencies
247
+ ]