metaxy 0.0.0__tar.gz

metaxy-0.0.0/PKG-INFO

@@ -0,0 +1,247 @@
+Metadata-Version: 2.3
+Name: metaxy
+Version: 0.0.0
+Summary: Add your description here
+Author: Daniel Gafni
+Author-email: Daniel Gafni <danielgafni16@gmail.com>
+Requires-Dist: cyclopts==4.0.0b1
+Requires-Dist: narwhals>=2.9.0
+Requires-Dist: polars>=1.33.1
+Requires-Dist: polars-hash>=0.5.1
+Requires-Dist: pydantic>=2.11.9
+Requires-Dist: pydantic-settings>=2.11.0
+Requires-Dist: pyyaml>=6.0.0
+Requires-Dist: tomli>=2.3.0
+Requires-Dist: rich>=13.0.0
+Requires-Dist: pygraphviz>=1.14 ; extra == 'graphviz'
+Requires-Dist: pyarrow>=18.0.0 ; extra == 'ibis'
+Requires-Dist: ibis-framework>=11.0.0 ; extra == 'ibis'
+Requires-Dist: mermaid-py>=0.8.0 ; extra == 'mermaid'
+Requires-Dist: sqlmodel>=0.0.27 ; extra == 'sqlmodel'
+Requires-Python: >=3.10
+Provides-Extra: graphviz
+Provides-Extra: ibis
+Provides-Extra: mermaid
+Provides-Extra: sqlmodel
+Description-Content-Type: text/markdown
+
+# Metaxy
+
+## Overview
+
+**Metaxy** is a declarative metadata management system for multi-modal data and machine learning pipelines. Metaxy allows statically defining graphs of features with versioned **fields** -- logical components like `audio`, `frames` for `.mp4` files and **columns** for feature metadata stored in Metaxy's metadata store. With this in place, Metaxy provides:
+
+- **Sample-level data versioning**: Track field and column lineage, compute versions as hashes of upstream versions for each sample
+- **Incremental computation**: Automatically detect which samples need recomputation when upstream fields change
+- **Migration system**: When feature code changes without changing outputs (refactoring, graph restructuring), Metaxy can reconcile metadata versions without recomputing expensive features
+- **Storage flexibility**: Pluggable backends (DuckDB, ClickHouse, PostgreSQL, SQLite, in-memory) with native SQL optimization where possible
+- **Big Metadata**: Metaxy is designed with large-scale distributed systems in mind and can handle large amounts of metadata efficiently.
+
+Metaxy is designed for production data and ML systems where data and features evolve over time, and you need to track what changed, why, and whether expensive recomputation is actually necessary.
+
+## Data Versioning
+
+To demonstrate how Metaxy handles data versioning, let's consider a video processing pipeline:
+
+```py
+from metaxy import (
+    Feature,
+    FeatureDep,
+    FeatureKey,
+    FeatureSpec,
+    FieldDep,
+    FieldKey,
+    FieldSpec,
+)
+
+
+class Video(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "video"]),
+        deps=None,  # Root feature
+        fields=[
+            FieldSpec(
+                key=FieldKey(["audio"]),
+                code_version=1,
+            ),
+            FieldSpec(
+                key=FieldKey(["frames"]),
+                code_version=1,
+            ),
+        ],
+    ),
+):
+    """Video metadata feature (root)."""
+
+    frames: int
+    duration: float
+    size: int
+
+
+class Crop(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "crop"]),
+        deps=[FeatureDep(key=Video.spec.key)],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["audio"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["audio"])],
+                    )
+                ],
+            ),
+            FieldSpec(
+                key=FieldKey(["frames"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["frames"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass  # omit columns for the sake of simplicity
+
+
+class FaceDetection(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "face_detection"]),
+        deps=[
+            FeatureDep(
+                key=Crop.spec.key,
+            )
+        ],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["faces"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Crop.spec.key,
+                        fields=[FieldKey(["frames"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass
+
+
+class SpeechToText(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["overview", "stt"]),
+        deps=[
+            FeatureDep(
+                key=Video.spec.key,
+            )
+        ],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["transcription"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["audio"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass
+```
+
+When provided with this Python module, `metaxy graph render --format mermaid` (that's handy, right?) produces the following graph:
+
+```mermaid
+---
+title: Feature Graph
+---
+flowchart TB
+    %% Snapshot version: 8468950d
+    %%{init: {'flowchart': {'htmlLabels': true, 'curve': 'basis'}, 'themeVariables': {'fontSize': '14px'}}}%%
+        example_video["<div style="text-align:left"><b>example/video</b><br/><small>(v: bc9ca835)</small><br/><font
+color="#999">---</font><br/>• audio <small>(v: 22742381)</small><br/>• frames <small>(v: 794116a9)</small></div>"]
+        example_crop["<div style="text-align:left"><b>example/crop</b><br/><small>(v: 3ac04df8)</small><br/><font
+color="#999">---</font><br/>• audio <small>(v: 76c8bdc9)</small><br/>• frames <small>(v: abc79017)</small></div>"]
+        example_face_detection["<div style="text-align:left"><b>example/face_detection</b><br/><small>(v: 1ac83b07)</small><br/><font
+color="#999">---</font><br/>• faces <small>(v: 2d75f0bd)</small></div>"]
+        example_stt["<div style="text-align:left"><b>example/stt</b><br/><small>(v: c83a754a)</small><br/><font
+color="#999">---</font><br/>• transcription <small>(v: ac412b3c)</small></div>"]
+        example_video --> example_crop
+        example_crop --> example_face_detection
+        example_video --> example_stt
+```
+
+Now imagine the `audio` logical field (don't mix up with metadata columns!) of the very first `Video` feature has been changed. Perhaps it has been cleaned or denoised.
+
+```diff
+         key=FeatureKey(["example", "video"]),
+         deps=None,  # Root feature
+         fields=[
+             FieldSpec(
+                 key=FieldKey(["audio"]),
+-                code_version=1,
++                code_version=2,
+             ),
+```
+
+In this case we'd typically want to recompute the downstream `Crop`, `SpeechToText`  and `Embeddings` features, but not the `FaceDetection` feature, since it only depends on `frames` and not on `audio`.
+
+`metaxy graph diff` reveals exactly that:
+
+```mermaid
+---
+title: Merged Graph Diff
+---
+flowchart TB
+    %%{init: {'flowchart': {'htmlLabels': true, 'curve': 'basis'}, 'themeVariables': {'fontSize': '14px'}}}%%
+
+    example_video["<div style="text-align:left"><b>example/video</b><br/><font color="#CC0000">bc9ca8</font> → <font
+color="#00AA00">6db302</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">audio</font> (<font
+color="#CC0000">227423</font> → <font color="#00AA00">09c839</font>)<br/>- frames (794116)</div>"]
+    style example_video stroke:#FFA500,stroke-width:3px
+    example_crop["<div style="text-align:left"><b>example/crop</b><br/><font color="#CC0000">3ac04d</font> → <font
+color="#00AA00">54dc7f</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">audio</font> (<font
+color="#CC0000">76c8bd</font> → <font color="#00AA00">f3130c</font>)<br/>- frames (abc790)</div>"]
+    style example_crop stroke:#FFA500,stroke-width:3px
+    example_face_detection["<div style="text-align:left"><b>example/face_detection</b><br/>1ac83b<br/><font
+color="#999">---</font><br/>- faces (2d75f0)</div>"]
+    example_stt["<div style="text-align:left"><b>example/stt</b><br/><font color="#CC0000">c83a75</font> → <font
+color="#00AA00">066d34</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">transcription</font> (<font
+color="#CC0000">ac412b</font> → <font color="#00AA00">058410</font>)</div>"]
+    style example_stt stroke:#FFA500,stroke-width:3px
+
+    example_video --> example_crop
+    example_crop --> example_face_detection
+    example_video --> example_stt
+```
+
+The versions of `audio` fields through the graph as well as the whole `FaceDetection` feature stayed the same!
+
+We can use Metaxy's static graph analysis to identify which features need to be recomputed when a new version of a feature is introduced. In addition to feature and field level versions, Metaxy can also compute a sample-level version (may be different for each sample in the one million dataset you have) ahead of computations through the whole graph. This enables exciting features such as processing cost prediction and automatic migrations for metadata.
+
+## Development
+
+Setting up the environment:
+
+```shell
+uv sync --all-extras
+uv run prek install
+```
+
+## Examples
+
+See [examples](examples/README.md).

metaxy-0.0.0/README.md

@@ -0,0 +1,220 @@
+# Metaxy
+
+## Overview
+
+**Metaxy** is a declarative metadata management system for multi-modal data and machine learning pipelines. Metaxy allows statically defining graphs of features with versioned **fields** -- logical components like `audio`, `frames` for `.mp4` files and **columns** for feature metadata stored in Metaxy's metadata store. With this in place, Metaxy provides:
+
+- **Sample-level data versioning**: Track field and column lineage, compute versions as hashes of upstream versions for each sample
+- **Incremental computation**: Automatically detect which samples need recomputation when upstream fields change
+- **Migration system**: When feature code changes without changing outputs (refactoring, graph restructuring), Metaxy can reconcile metadata versions without recomputing expensive features
+- **Storage flexibility**: Pluggable backends (DuckDB, ClickHouse, PostgreSQL, SQLite, in-memory) with native SQL optimization where possible
+- **Big Metadata**: Metaxy is designed with large-scale distributed systems in mind and can handle large amounts of metadata efficiently.
+
+Metaxy is designed for production data and ML systems where data and features evolve over time, and you need to track what changed, why, and whether expensive recomputation is actually necessary.
+
+## Data Versioning
+
+To demonstrate how Metaxy handles data versioning, let's consider a video processing pipeline:
+
+```py
+from metaxy import (
+    Feature,
+    FeatureDep,
+    FeatureKey,
+    FeatureSpec,
+    FieldDep,
+    FieldKey,
+    FieldSpec,
+)
+
+
+class Video(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "video"]),
+        deps=None,  # Root feature
+        fields=[
+            FieldSpec(
+                key=FieldKey(["audio"]),
+                code_version=1,
+            ),
+            FieldSpec(
+                key=FieldKey(["frames"]),
+                code_version=1,
+            ),
+        ],
+    ),
+):
+    """Video metadata feature (root)."""
+
+    frames: int
+    duration: float
+    size: int
+
+
+class Crop(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "crop"]),
+        deps=[FeatureDep(key=Video.spec.key)],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["audio"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["audio"])],
+                    )
+                ],
+            ),
+            FieldSpec(
+                key=FieldKey(["frames"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["frames"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass  # omit columns for the sake of simplicity
+
+
+class FaceDetection(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["example", "face_detection"]),
+        deps=[
+            FeatureDep(
+                key=Crop.spec.key,
+            )
+        ],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["faces"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Crop.spec.key,
+                        fields=[FieldKey(["frames"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass
+
+
+class SpeechToText(
+    Feature,
+    spec=FeatureSpec(
+        key=FeatureKey(["overview", "stt"]),
+        deps=[
+            FeatureDep(
+                key=Video.spec.key,
+            )
+        ],
+        fields=[
+            FieldSpec(
+                key=FieldKey(["transcription"]),
+                code_version=1,
+                deps=[
+                    FieldDep(
+                        feature_key=Video.spec.key,
+                        fields=[FieldKey(["audio"])],
+                    )
+                ],
+            ),
+        ],
+    ),
+):
+    pass
+```
+
+When provided with this Python module, `metaxy graph render --format mermaid` (that's handy, right?) produces the following graph:
+
+```mermaid
+---
+title: Feature Graph
+---
+flowchart TB
+    %% Snapshot version: 8468950d
+    %%{init: {'flowchart': {'htmlLabels': true, 'curve': 'basis'}, 'themeVariables': {'fontSize': '14px'}}}%%
+        example_video["<div style="text-align:left"><b>example/video</b><br/><small>(v: bc9ca835)</small><br/><font
+color="#999">---</font><br/>• audio <small>(v: 22742381)</small><br/>• frames <small>(v: 794116a9)</small></div>"]
+        example_crop["<div style="text-align:left"><b>example/crop</b><br/><small>(v: 3ac04df8)</small><br/><font
+color="#999">---</font><br/>• audio <small>(v: 76c8bdc9)</small><br/>• frames <small>(v: abc79017)</small></div>"]
+        example_face_detection["<div style="text-align:left"><b>example/face_detection</b><br/><small>(v: 1ac83b07)</small><br/><font
+color="#999">---</font><br/>• faces <small>(v: 2d75f0bd)</small></div>"]
+        example_stt["<div style="text-align:left"><b>example/stt</b><br/><small>(v: c83a754a)</small><br/><font
+color="#999">---</font><br/>• transcription <small>(v: ac412b3c)</small></div>"]
+        example_video --> example_crop
+        example_crop --> example_face_detection
+        example_video --> example_stt
+```
+
+Now imagine the `audio` logical field (don't mix up with metadata columns!) of the very first `Video` feature has been changed. Perhaps it has been cleaned or denoised.
+
+```diff
+         key=FeatureKey(["example", "video"]),
+         deps=None,  # Root feature
+         fields=[
+             FieldSpec(
+                 key=FieldKey(["audio"]),
+-                code_version=1,
++                code_version=2,
+             ),
+```
+
+In this case we'd typically want to recompute the downstream `Crop`, `SpeechToText`  and `Embeddings` features, but not the `FaceDetection` feature, since it only depends on `frames` and not on `audio`.
+
+`metaxy graph diff` reveals exactly that:
+
+```mermaid
+---
+title: Merged Graph Diff
+---
+flowchart TB
+    %%{init: {'flowchart': {'htmlLabels': true, 'curve': 'basis'}, 'themeVariables': {'fontSize': '14px'}}}%%
+
+    example_video["<div style="text-align:left"><b>example/video</b><br/><font color="#CC0000">bc9ca8</font> → <font
+color="#00AA00">6db302</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">audio</font> (<font
+color="#CC0000">227423</font> → <font color="#00AA00">09c839</font>)<br/>- frames (794116)</div>"]
+    style example_video stroke:#FFA500,stroke-width:3px
+    example_crop["<div style="text-align:left"><b>example/crop</b><br/><font color="#CC0000">3ac04d</font> → <font
+color="#00AA00">54dc7f</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">audio</font> (<font
+color="#CC0000">76c8bd</font> → <font color="#00AA00">f3130c</font>)<br/>- frames (abc790)</div>"]
+    style example_crop stroke:#FFA500,stroke-width:3px
+    example_face_detection["<div style="text-align:left"><b>example/face_detection</b><br/>1ac83b<br/><font
+color="#999">---</font><br/>- faces (2d75f0)</div>"]
+    example_stt["<div style="text-align:left"><b>example/stt</b><br/><font color="#CC0000">c83a75</font> → <font
+color="#00AA00">066d34</font><br/><font color="#999">---</font><br/>- <font color="#FFAA00">transcription</font> (<font
+color="#CC0000">ac412b</font> → <font color="#00AA00">058410</font>)</div>"]
+    style example_stt stroke:#FFA500,stroke-width:3px
+
+    example_video --> example_crop
+    example_crop --> example_face_detection
+    example_video --> example_stt
+```
+
+The versions of `audio` fields through the graph as well as the whole `FaceDetection` feature stayed the same!
+
+We can use Metaxy's static graph analysis to identify which features need to be recomputed when a new version of a feature is introduced. In addition to feature and field level versions, Metaxy can also compute a sample-level version (may be different for each sample in the one million dataset you have) ahead of computations through the whole graph. This enables exciting features such as processing cost prediction and automatic migrations for metadata.
+
+## Development
+
+Setting up the environment:
+
+```shell
+uv sync --all-extras
+uv run prek install
+```
+
+## Examples
+
+See [examples](examples/README.md).

metaxy-0.0.0/pyproject.toml

@@ -0,0 +1,112 @@
+[project]
+name = "metaxy"
+version = "0.0.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "Daniel Gafni", email = "danielgafni16@gmail.com" }
+]
+requires-python = ">=3.10"
+dependencies = [
+    "cyclopts==4.0.0b1",
+    "narwhals>=2.9.0",
+    "polars>=1.33.1",
+    "polars-hash>=0.5.1",
+    "pydantic>=2.11.9",
+    "pydantic-settings>=2.11.0",
+    "pyyaml>=6.0.0",
+    "tomli>=2.3.0",
+    "rich>=13.0.0",
+]
+
+[project.optional-dependencies]
+ibis = [
+    "pyarrow>=18.0.0",
+    "ibis-framework>=11.0.0",
+]
+mermaid = [
+    "mermaid-py>=0.8.0",
+]
+graphviz = [
+    "pygraphviz>=1.14",
+]
+sqlmodel = [
+    "sqlmodel>=0.0.27",
+]
+
+[project.scripts]
+metaxy = "metaxy.cli.app:main"
+
+[tool.pyrefly]
+project-includes = ["src", "tests"]
+# search-path = ["src"]
+project-excludes = [
+    "**/node_modules",
+    "**/__pycache__",
+    "**/*venv/**/*",
+]
+
+[build-system]
+requires = ["uv_build>=0.8.14,<0.9.0"]
+build-backend = "uv_build"
+
+[dependency-groups]
+dev = [
+    "examples",
+    "ipython>=8.37.0",
+    "duckdb>=1.4.1",
+    "pyrefly>=0.34.0",
+    "pytest>=8.4.2",
+    "pytest-cases>=3.9.1",
+    "ruff>=0.13.1",
+    "syrupy>=5.0.0",
+    "ibis-framework[duckdb,sqlite,clickhouse]>=11.0.0",
+    "basedpyright>=1.32.1",
+    "vulture>=2.14",
+    "prek>=0.2.11",
+]
+
+[tool.ruff]
+target-version = "py310"
+
+[tool.ruff.lint]
+extend-select = [
+    "I",
+    "TID252",
+    "UP"
+]
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Ban certain modules from being imported at module level, instead requiring
+# that they're imported lazily (e.g., within a function definition).
+banned-module-level-imports = ["ibis", "duckdb"]
+
+[tool.uv.workspace]
+members = [
+    "examples",
+]
+
+[tool.uv.sources]
+examples = { workspace = true }
+
+[tool.vulture]
+exclude = [
+    "src/metaxy/_testing.py",
+    "src/metaxy/cli/",
+    "examples/**/feature*.py",
+]
+min_confidence = 60
+
+[tool.pyright]
+reportAny = "none"
+reportExplicitAny = "none"
+include = [
+    "src/metaxy",
+    "tests",
+    "examples",
+]
+exclude = [
+    "**/node_modules",
+    "**/__pycache__",
+    "**/*venv/**/*",
+]

metaxy-0.0.0/src/metaxy/__init__.py

@@ -0,0 +1,61 @@
+from metaxy.config import MetaxyConfig, StoreConfig
+from metaxy.entrypoints import (
+    load_features,
+    load_module_entrypoint,
+    load_package_entrypoints,
+)
+from metaxy.metadata_store import (
+    InMemoryMetadataStore,
+    MetadataStore,
+)
+from metaxy.migrations import (
+    BaseOperation,
+    CustomMigration,
+    DataVersionReconciliation,
+    DiffMigration,
+    FullGraphMigration,
+    MetadataBackfill,
+    Migration,
+    MigrationExecutor,
+    MigrationResult,
+    SystemTableStorage,
+    detect_migration,
+)
+from metaxy.models.feature import Feature, FeatureGraph, get_feature_by_key, graph
+from metaxy.models.feature_spec import FeatureDep, FeatureSpec
+from metaxy.models.field import FieldDep, FieldSpec, SpecialFieldDep
+from metaxy.models.types import FeatureDepMetadata, FeatureKey, FieldKey
+
+__all__ = [
+    "Feature",
+    "FeatureGraph",
+    "graph",
+    "get_feature_by_key",
+    "FeatureDep",
+    "FeatureDepMetadata",
+    "FeatureSpec",
+    "FieldDep",
+    "FieldSpec",
+    "SpecialFieldDep",
+    "FeatureKey",
+    "FieldKey",
+    "MetadataStore",
+    "InMemoryMetadataStore",
+    "load_features",
+    "load_config_entrypoints",
+    "load_module_entrypoint",
+    "load_package_entrypoints",
+    "Migration",
+    "DiffMigration",
+    "FullGraphMigration",
+    "CustomMigration",
+    "MigrationResult",
+    "MigrationExecutor",
+    "SystemTableStorage",
+    "BaseOperation",
+    "DataVersionReconciliation",
+    "MetadataBackfill",
+    "detect_migration",
+    "MetaxyConfig",
+    "StoreConfig",
+]