metaxy 0.0.1.dev3__py3-none-any.whl

metaxy/__init__.py

@@ -0,0 +1,170 @@
+from pathlib import Path
+
+from metaxy._version import __version__
+from metaxy.config import MetaxyConfig, StoreConfig
+from metaxy.entrypoints import (
+    load_features,
+    load_module_entrypoint,
+    load_package_entrypoints,
+)
+from metaxy.metadata_store import AccessMode, InMemoryMetadataStore, MetadataStore
+from metaxy.migrations import (
+    BaseOperation,
+    DataVersionReconciliation,
+    DiffMigration,
+    FullGraphMigration,
+    MetadataBackfill,
+    Migration,
+    MigrationExecutor,
+    MigrationResult,
+    SystemTableStorage,
+    detect_diff_migration,
+)
+from metaxy.models.feature import (
+    BaseFeature,
+    FeatureGraph,
+    current_graph,
+    get_feature_by_key,
+    graph,
+)
+from metaxy.models.feature_spec import (
+    FeatureDep,
+    FeatureSpec,
+    FeatureSpecWithIDColumns,
+    IDColumns,
+)
+from metaxy.models.field import (
+    FieldDep,
+    FieldSpec,
+    SpecialFieldDep,
+)
+from metaxy.models.fields_mapping import (
+    AllFieldsMapping,
+    DefaultFieldsMapping,
+    FieldsMapping,
+    FieldsMappingType,
+)
+from metaxy.models.lineage import LineageRelationship
+from metaxy.models.types import (
+    CoercibleToFeatureKey,
+    CoercibleToFieldKey,
+    FeatureDepMetadata,
+    FeatureKey,
+    FieldKey,
+    ValidatedFeatureKey,
+    ValidatedFeatureKeyAdapter,
+    ValidatedFeatureKeySequence,
+    ValidatedFeatureKeySequenceAdapter,
+    ValidatedFieldKey,
+    ValidatedFieldKeyAdapter,
+    ValidatedFieldKeySequence,
+    ValidatedFieldKeySequenceAdapter,
+)
+from metaxy.versioning.types import HashAlgorithm
+
+
+def coerce_to_feature_key(value: CoercibleToFeatureKey) -> FeatureKey:
+    """Coerce a value to a [`FeatureKey`][metaxy.FeatureKey].
+
+    Accepts:
+
+    - slashed `str`: `"a/b/c"`
+    - `Sequence[str]`: `["a", "b", "c"]`
+    - `FeatureKey`: pass through
+    - `type[BaseFeature]`: extracts `.spec().key`
+
+    Args:
+        value: Value to coerce to `FeatureKey`
+
+    Returns:
+        The coerced `FeatureKey`
+
+    Raises:
+        ValidationError: If the value cannot be coerced to a `FeatureKey`
+    """
+    return ValidatedFeatureKeyAdapter.validate_python(value)
+
+
+def init_metaxy(
+    config_file: Path | None = None, search_parents: bool = True
+) -> MetaxyConfig:
+    """Main user-facing initialization function for Metaxy. It loads the configuration and features.
+
+    Features are [discovered](../../guide/learn/feature-discovery.md) from installed Python packages metadata.
+
+    Args:
+        config_file (Path | None, optional): Path to the configuration file.
+
+            Will be auto-discovered in current or parent directories if not provided.
+
+            !!! tip
+                `METAXY_CONFIG` environment variable can be used to set this parameter
+
+        search_parents (bool, optional): Whether to search parent directories for configuration files. Defaults to True.
+
+    Returns:
+        MetaxyConfig: The initialized Metaxy configuration.
+    """
+    cfg = MetaxyConfig.load(
+        config_file=config_file,
+        search_parents=search_parents,
+    )
+    load_features(cfg.entrypoints)
+    return cfg
+
+
+__all__ = [
+    "BaseFeature",
+    "FeatureGraph",
+    "graph",
+    "FeatureSpec",
+    "get_feature_by_key",
+    "FeatureDep",
+    "FeatureDepMetadata",
+    "FeatureSpec",
+    "FeatureSpecWithIDColumns",
+    "AllFieldsMapping",
+    "DefaultFieldsMapping",
+    "FieldsMapping",
+    "FieldsMappingType",
+    "FieldDep",
+    "FieldSpec",
+    "SpecialFieldDep",
+    "FeatureKey",
+    "FieldKey",
+    "CoercibleToFeatureKey",
+    "CoercibleToFieldKey",
+    "coerce_to_feature_key",
+    "ValidatedFeatureKey",
+    "ValidatedFieldKey",
+    "ValidatedFeatureKeySequence",
+    "ValidatedFieldKeySequence",
+    "MetadataStore",
+    "InMemoryMetadataStore",
+    "load_features",
+    "load_module_entrypoint",
+    "load_package_entrypoints",
+    "Migration",
+    "DiffMigration",
+    "FullGraphMigration",
+    "MigrationResult",
+    "MigrationExecutor",
+    "SystemTableStorage",
+    "BaseOperation",
+    "DataVersionReconciliation",
+    "MetadataBackfill",
+    "detect_diff_migration",
+    "MetaxyConfig",
+    "StoreConfig",
+    "init_metaxy",
+    "IDColumns",
+    "HashAlgorithm",
+    "LineageRelationship",
+    "AccessMode",
+    "current_graph",
+    "ValidatedFeatureKeyAdapter",
+    "ValidatedFieldKeyAdapter",
+    "ValidatedFeatureKeySequenceAdapter",
+    "ValidatedFieldKeySequenceAdapter",
+    "__version__",
+]

metaxy/_packaging.py

@@ -0,0 +1,96 @@
+"""Packaging utilities for Metaxy project detection.
+
+This module provides functionality to detect which Metaxy project a Feature belongs to
+based on package metadata entry points.
+
+Entry point format in pyproject.toml:
+    [project.entry-points."metaxy.project"]
+    my-project = "my_package.entrypoint"
+
+The entry point name is the project name, and the value is the entrypoint module.
+"""
+
+from functools import lru_cache
+
+try:
+    from importlib.metadata import entry_points
+except ImportError:
+    from importlib_metadata import entry_points  # type: ignore
+
+
+def detect_project_from_entrypoints(module_name: str) -> str | None:
+    """Detect project name from metaxy.project entry points.
+
+    Checks if the module was loaded via a declared metaxy entry point.
+
+    Entry point format:
+        [project.entry-points."metaxy.project"]
+        my-project = "my_package:init"
+
+    The entry point should point to a function, but the function is never called.
+    Metaxy only uses the module path portion for project detection.
+
+    Args:
+        module_name: Fully qualified module name (e.g., "my_package.features")
+
+    Returns:
+        Project name if found, None otherwise
+    """
+    # Use cached entry points to avoid repeated queries
+    all_entrypoints = get_all_project_entrypoints()
+
+    # Check if our module matches any entry point
+    for project_name, ep_module in all_entrypoints.items():
+        if module_name == ep_module or module_name.startswith(ep_module + "."):
+            # The entry point name is the project name
+            return project_name
+
+    return None
+
+
+@lru_cache(maxsize=1)
+def get_all_project_entrypoints() -> dict[str, str]:
+    """Get all declared metaxy.project entry points.
+
+    This function is cached since entry points don't change during program execution.
+
+    Returns:
+        Dictionary mapping project names to entrypoint module paths (without :function)
+
+    Raises:
+        ValueError: If a package declares multiple entry points (only one per package is allowed)
+    """
+    result: dict[str, str] = {}
+
+    # Track which top-level package each entry point belongs to
+    package_to_projects: dict[str, list[str]] = {}
+
+    # Query the metaxy.project group directly - no deprecation warning
+    group_eps = entry_points(group="metaxy.project")
+
+    # Map project name (entry point name) to entrypoint module (entry point value)
+    for ep in group_eps:
+        # Extract module path from entry point value (before ':' if present)
+        ep_module = ep.value.split(":")[0] if ":" in ep.value else ep.value
+
+        # Get the top-level package name (first component of module path)
+        top_level_package = ep_module.split(".")[0]
+
+        # Track this project for this package
+        if top_level_package not in package_to_projects:
+            package_to_projects[top_level_package] = []
+        package_to_projects[top_level_package].append(ep.name)
+
+        result[ep.name] = ep_module
+
+    # Validate that each package only declares one entry point
+    for package, projects in package_to_projects.items():
+        if len(projects) > 1:
+            # Format the entry points list
+            entries = ", ".join(f"'{p}'" for p in projects)
+            raise ValueError(
+                f"Found multiple entries in `metaxy.project` entrypoints group: {entries}. "
+                f"The key should be the Metaxy project name, thus only one entry is allowed."
+            )
+
+    return result

metaxy/_testing/__init__.py

@@ -0,0 +1,55 @@
+"""Testing infrastructure for Metaxy examples and runbooks.
+
+This is a private module (_testing) containing testing utilities organized into:
+- runbook: Runbook system for testing and documenting examples
+- metaxy_project: Project helpers for creating and managing temporary Metaxy projects
+- pytest_helpers: Testing helpers for pytest tests
+- models: Testing-specific model implementations
+"""
+
+# Runbook system
+# Metaxy project helpers
+from metaxy._testing.metaxy_project import (
+    ExternalMetaxyProject,
+    HashAlgorithmCases,
+    MetaxyProject,
+    TempFeatureModule,
+    TempMetaxyProject,
+    assert_all_results_equal,
+)
+from metaxy._testing.models import SampleFeature, SampleFeatureSpec
+from metaxy._testing.pytest_helpers import add_metaxy_provenance_column
+from metaxy._testing.runbook import (
+    ApplyPatchStep,
+    AssertOutputStep,
+    BaseStep,
+    Runbook,
+    RunbookRunner,
+    RunCommandStep,
+    Scenario,
+    StepType,
+)
+
+__all__ = [
+    # Runbook system
+    "Runbook",
+    "Scenario",
+    "BaseStep",
+    "RunCommandStep",
+    "ApplyPatchStep",
+    "AssertOutputStep",
+    "StepType",
+    "RunbookRunner",
+    # Metaxy project helpers
+    "TempFeatureModule",
+    "HashAlgorithmCases",
+    "MetaxyProject",
+    "ExternalMetaxyProject",
+    "TempMetaxyProject",
+    "assert_all_results_equal",
+    # Pytest helpers
+    "add_metaxy_provenance_column",
+    # Testing models
+    "SampleFeatureSpec",
+    "SampleFeature",
+]

metaxy/_testing/config.py

@@ -0,0 +1,43 @@
+"""Test configuration classes for documentation generation testing."""
+
+from pydantic import Field as PydanticField
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class SamplePluginConfig(BaseSettings):
+    """Sample plugin configuration for doc generation testing.
+
+    This is a minimal config class used to test documentation generation
+    without depending on the full Metaxy config structure.
+    """
+
+    model_config = SettingsConfigDict(
+        env_prefix="SAMPLE_PLUGIN_",
+        env_nested_delimiter="__",
+        frozen=True,
+    )
+
+    enable: bool = PydanticField(
+        default=False,
+        description="Whether to enable the test plugin",
+    )
+
+    name: str = PydanticField(
+        default="test",
+        description="Name of the test plugin",
+    )
+
+    port: int = PydanticField(
+        default=8080,
+        description="Port number for the test service",
+    )
+
+    debug: bool = PydanticField(
+        default=False,
+        description="Enable debug mode",
+    )
+
+    optional_setting: str | None = PydanticField(
+        default=None,
+        description="Optional configuration setting",
+    )