flowyml 1.1.0__py3-none-any.whl

flowyml/assets/featureset.py

@@ -0,0 +1,301 @@
+"""FeatureSet asset for feature engineering outputs."""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+from flowyml.assets.base import Asset, AssetMetadata
+
+
+@dataclass
+class FeatureSetMetadata(AssetMetadata):
+    """Metadata specific to FeatureSets."""
+
+    feature_names: list[str] = field(default_factory=list)
+    num_features: int = 0
+    num_samples: int = 0
+    feature_types: dict[str, str] = field(default_factory=dict)
+    statistics: dict[str, dict[str, float]] = field(default_factory=dict)
+    transformations: list[str] = field(default_factory=list)
+    source_dataset: str | None = None
+
+
+class FeatureSet(Asset):
+    """Asset representing a set of engineered features.
+
+    FeatureSets are created through feature engineering pipelines and contain
+    transformed data ready for model training.
+
+    Example:
+        ```python
+        from flowyml import FeatureSet
+
+        # Create a feature set
+        features = FeatureSet.create(
+            data=feature_matrix,
+            feature_names=["age_scaled", "income_log", "category_encoded"],
+            num_samples=10000,
+            transformations=["StandardScaler", "LogTransform", "OneHotEncoder"],
+            source_dataset="customers_v1",
+        )
+
+        # Access feature information
+        print(features.num_features)  # 3
+        print(features.feature_names)
+        print(features.statistics)
+        ```
+    """
+
+    def __init__(
+        self,
+        name: str,
+        data: Any = None,
+        feature_names: list[str] | None = None,
+        num_samples: int = 0,
+        transformations: list[str] | None = None,
+        source_dataset: str | None = None,
+        **kwargs,
+    ):
+        """Initialize a FeatureSet.
+
+        Args:
+            name: Name of the feature set
+            data: The feature matrix (DataFrame, array, etc.)
+            feature_names: List of feature names
+            num_samples: Number of samples in the feature set
+            transformations: List of transformations applied
+            source_dataset: Name of source dataset
+            **kwargs: Additional metadata
+        """
+        # Initialize base asset
+        metadata = FeatureSetMetadata(
+            name=name,
+            type="featureset",
+            feature_names=feature_names or [],
+            num_features=len(feature_names) if feature_names else 0,
+            num_samples=num_samples,
+            transformations=transformations or [],
+            source_dataset=source_dataset,
+            **kwargs,
+        )
+
+        super().__init__(name=name, type="featureset", metadata=metadata)
+        self._data = data
+
+        # Extract feature metadata if data provided
+        if data is not None:
+            self._extract_feature_metadata()
+
+    def _extract_feature_metadata(self) -> None:
+        """Extract feature metadata from data."""
+        try:
+            import pandas as pd
+
+            if isinstance(self._data, pd.DataFrame):
+                # Update feature names from DataFrame
+                if not self.metadata.feature_names:
+                    self.metadata.feature_names = self._data.columns.tolist()
+                    self.metadata.num_features = len(self._data.columns)
+
+                # Extract feature types
+                self.metadata.feature_types = {col: str(dtype) for col, dtype in self._data.dtypes.items()}
+
+                # Update num_samples
+                if not self.metadata.num_samples:
+                    self.metadata.num_samples = len(self._data)
+
+                # Calculate statistics for numerical columns
+                numerical_cols = self._data.select_dtypes(include=["number"]).columns
+                for col in numerical_cols:
+                    self.metadata.statistics[col] = {
+                        "min": float(self._data[col].min()),
+                        "max": float(self._data[col].max()),
+                        "mean": float(self._data[col].mean()),
+                        "std": float(self._data[col].std()),
+                        "missing": int(self._data[col].isna().sum()),
+                    }
+
+        except ImportError:
+            pass
+
+        try:
+            import numpy as np
+
+            if isinstance(self._data, np.ndarray):
+                # Update dimensions
+                if self._data.ndim >= 2:
+                    self.metadata.num_samples = self._data.shape[0]
+                    self.metadata.num_features = self._data.shape[1]
+
+                # Calculate statistics if numerical
+                if np.issubdtype(self._data.dtype, np.number):
+                    for i in range(min(self.metadata.num_features, 100)):  # Limit to 100 features
+                        feature_name = (
+                            self.metadata.feature_names[i] if i < len(self.metadata.feature_names) else f"feature_{i}"
+                        )
+                        self.metadata.statistics[feature_name] = {
+                            "min": float(np.min(self._data[:, i])),
+                            "max": float(np.max(self._data[:, i])),
+                            "mean": float(np.mean(self._data[:, i])),
+                            "std": float(np.std(self._data[:, i])),
+                        }
+        except ImportError:
+            pass
+
+    @property
+    def data(self) -> Any:
+        """Get the feature data."""
+        return self._data
+
+    @property
+    def feature_names(self) -> list[str]:
+        """Get feature names."""
+        return self.metadata.feature_names
+
+    @property
+    def num_features(self) -> int:
+        """Get number of features."""
+        return self.metadata.num_features
+
+    @property
+    def num_samples(self) -> int:
+        """Get number of samples."""
+        return self.metadata.num_samples
+
+    @property
+    def feature_types(self) -> dict[str, str]:
+        """Get feature types."""
+        return self.metadata.feature_types
+
+    @property
+    def statistics(self) -> dict[str, dict[str, float]]:
+        """Get feature statistics."""
+        return self.metadata.statistics
+
+    @property
+    def transformations(self) -> list[str]:
+        """Get list of transformations applied."""
+        return self.metadata.transformations
+
+    @property
+    def source_dataset(self) -> str | None:
+        """Get source dataset name."""
+        return self.metadata.source_dataset
+
+    @classmethod
+    def create(
+        cls,
+        data: Any,
+        name: str | None = None,
+        feature_names: list[str] | None = None,
+        transformations: list[str] | None = None,
+        source_dataset: str | None = None,
+        **kwargs,
+    ) -> "FeatureSet":
+        """Factory method to create a FeatureSet.
+
+        Args:
+            data: The feature matrix
+            name: Name of the feature set (auto-generated if not provided)
+            feature_names: List of feature names
+            transformations: List of transformations applied
+            source_dataset: Name of source dataset
+            **kwargs: Additional metadata
+
+        Returns:
+            New FeatureSet instance
+        """
+        if name is None:
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            name = f"features_{timestamp}"
+
+        return cls(
+            name=name,
+            data=data,
+            feature_names=feature_names,
+            transformations=transformations,
+            source_dataset=source_dataset,
+            **kwargs,
+        )
+
+    def select_features(self, feature_names: list[str]) -> "FeatureSet":
+        """Select a subset of features.
+
+        Args:
+            feature_names: List of feature names to select
+
+        Returns:
+            New FeatureSet with selected features
+        """
+        if self._data is None:
+            raise ValueError("Cannot select features from FeatureSet without data")
+
+        try:
+            import pandas as pd
+
+            if isinstance(self._data, pd.DataFrame):
+                selected_data = self._data[feature_names]
+                return FeatureSet.create(
+                    data=selected_data,
+                    name=f"{self.name}_selected",
+                    feature_names=feature_names,
+                    transformations=self.transformations,
+                    source_dataset=self.name,
+                )
+        except ImportError:
+            pass
+
+        try:
+            import numpy as np
+
+            if isinstance(self._data, np.ndarray):
+                # Map feature names to indices
+                indices = [self.feature_names.index(fn) for fn in feature_names if fn in self.feature_names]
+                selected_data = self._data[:, indices]
+                return FeatureSet.create(
+                    data=selected_data,
+                    name=f"{self.name}_selected",
+                    feature_names=feature_names,
+                    transformations=self.transformations,
+                    source_dataset=self.name,
+                )
+        except ImportError:
+            pass
+
+        raise TypeError("Unsupported data type for feature selection")
+
+    def get_feature_statistics(self, feature_name: str) -> dict[str, float] | None:
+        """Get statistics for a specific feature.
+
+        Args:
+            feature_name: Name of the feature
+
+        Returns:
+            Dictionary of statistics or None if not found
+        """
+        return self.statistics.get(feature_name)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert FeatureSet to dictionary.
+
+        Returns:
+            Dictionary representation (excluding data)
+        """
+        return {
+            "id": self.id,
+            "name": self.name,
+            "type": self.type,
+            "feature_names": self.feature_names,
+            "num_features": self.num_features,
+            "num_samples": self.num_samples,
+            "feature_types": self.feature_types,
+            "statistics": self.statistics,
+            "transformations": self.transformations,
+            "source_dataset": self.source_dataset,
+            "created_at": self.created_at.isoformat(),
+            "tags": self.tags,
+            "properties": self.properties,
+        }
+
+    def __repr__(self) -> str:
+        return f"FeatureSet(name='{self.name}', num_features={self.num_features}, num_samples={self.num_samples})"

flowyml/assets/metrics.py

@@ -0,0 +1,104 @@
+"""Metrics Asset - Represents experiment metrics and evaluation results."""
+
+from typing import Any
+from flowyml.assets.base import Asset
+
+
+class Metrics(Asset):
+    """Metrics asset for experiment tracking.
+
+    Example:
+        >>> metrics = Metrics(name="training_metrics", data={"accuracy": 0.95, "loss": 0.05, "f1_score": 0.93})
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: str | None = None,
+        data: dict[str, Any] | None = None,
+        parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
+        **metrics,
+    ):
+        # Merge data and kwargs metrics
+        all_metrics = data or {}
+        all_metrics.update(metrics)
+
+        super().__init__(
+            name=name,
+            version=version,
+            data=all_metrics,
+            parent=parent,
+            tags=tags,
+            properties=properties,
+        )
+
+        # Store metrics in properties for easy access
+        self.metadata.properties.update(all_metrics)
+
+    def get_metric(self, name: str, default: Any = None) -> Any:
+        """Get a specific metric value."""
+        if self.data and name in self.data:
+            return self.data[name]
+        return self.metadata.properties.get(name, default)
+
+    def add_metric(self, name: str, value: Any) -> None:
+        """Add a new metric."""
+        if self.data is None:
+            self.data = {}
+        self.data[name] = value
+        self.metadata.properties[name] = value
+
+    def get_all_metrics(self) -> dict[str, Any]:
+        """Get all metrics."""
+        return self.data or {}
+
+    def compare_with(self, other: "Metrics") -> dict[str, dict[str, Any]]:
+        """Compare metrics with another Metrics asset.
+
+        Returns:
+            Dictionary with comparison results
+        """
+        self_metrics = self.get_all_metrics()
+        other_metrics = other.get_all_metrics()
+
+        comparison = {}
+        all_keys = set(self_metrics.keys()) | set(other_metrics.keys())
+
+        for key in all_keys:
+            self_val = self_metrics.get(key)
+            other_val = other_metrics.get(key)
+
+            comparison[key] = {
+                "self": self_val,
+                "other": other_val,
+                "diff": self_val - other_val
+                if (
+                    self_val is not None
+                    and other_val is not None
+                    and isinstance(self_val, (int, float))
+                    and isinstance(other_val, (int, float))
+                )
+                else None,
+            }
+
+        return comparison
+
+    @classmethod
+    def create(
+        cls,
+        name: str | None = None,
+        parent: Asset | None = None,
+        **metrics,
+    ) -> "Metrics":
+        """Factory method to create metrics.
+
+        Example:
+            >>> metrics = Metrics.create(accuracy=0.95, loss=0.05, training_time="2h 15m")
+        """
+        return cls(
+            name=name or "metrics",
+            data=metrics,
+            parent=parent,
+        )

flowyml/assets/model.py

@@ -0,0 +1,82 @@
+"""Model Asset - Represents ML models with metadata and lineage."""
+
+from typing import Any
+from flowyml.assets.base import Asset
+
+
+class Model(Asset):
+    """Model asset with training metadata and lineage.
+
+    Example:
+        >>> model = Model(
+        ...     name="resnet50_v1",
+        ...     version="v1.0.0",
+        ...     data=trained_model,
+        ...     architecture="resnet50",
+        ...     framework="pytorch",
+        ...     properties={"params": 25_557_032},
+        ... )
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: str | None = None,
+        data: Any = None,
+        architecture: str | None = None,
+        framework: str | None = None,
+        input_shape: tuple | None = None,
+        output_shape: tuple | None = None,
+        trained_on: Asset | None = None,
+        parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            data=data,
+            parent=parent,
+            tags=tags,
+            properties=properties,
+        )
+
+        self.architecture = architecture
+        self.framework = framework
+        self.input_shape = input_shape
+        self.output_shape = output_shape
+
+        # Track training dataset
+        if trained_on:
+            self.parents.append(trained_on)
+            trained_on.children.append(self)
+
+        # Add model-specific properties
+        if architecture:
+            self.metadata.properties["architecture"] = architecture
+        if framework:
+            self.metadata.properties["framework"] = framework
+        if input_shape:
+            self.metadata.properties["input_shape"] = input_shape
+        if output_shape:
+            self.metadata.properties["output_shape"] = output_shape
+
+    def get_training_datasets(self):
+        """Get all datasets this model was trained on."""
+        from flowyml.assets.dataset import Dataset
+
+        return [p for p in self.parents if isinstance(p, Dataset)]
+
+    def get_parameters_count(self) -> int | None:
+        """Get number of model parameters if available."""
+        return self.metadata.properties.get("params") or self.metadata.properties.get("parameters")
+
+    def get_architecture_info(self) -> dict[str, Any]:
+        """Get architecture information."""
+        return {
+            "architecture": self.architecture,
+            "framework": self.framework,
+            "input_shape": self.input_shape,
+            "output_shape": self.output_shape,
+            "parameters": self.get_parameters_count(),
+        }

flowyml/assets/registry.py

@@ -0,0 +1,157 @@
+"""Asset Registry - Central registry for all pipeline assets."""
+
+import json
+from pathlib import Path
+from datetime import datetime
+
+from flowyml.assets.base import Asset
+
+
+class AssetRegistry:
+    """Central registry for managing and querying assets."""
+
+    def __init__(self, registry_dir: str = ".flowyml/assets"):
+        self.registry_dir = Path(registry_dir)
+        self.registry_dir.mkdir(parents=True, exist_ok=True)
+
+        self.assets: dict[str, Asset] = {}  # id -> asset
+        self.assets_by_name: dict[str, list[Asset]] = {}  # name -> [assets]
+        self.assets_by_type: dict[str, set[Asset]] = {}  # type -> {assets}
+
+        self._load_registry()
+
+    def register(self, asset: Asset) -> None:
+        """Register an asset."""
+        self.assets[asset.id] = asset
+
+        # Index by name
+        if asset.name not in self.assets_by_name:
+            self.assets_by_name[asset.name] = []
+        self.assets_by_name[asset.name].append(asset)
+
+        # Index by type
+        asset_type = asset.metadata.asset_type
+        if asset_type not in self.assets_by_type:
+            self.assets_by_type[asset_type] = set()
+        self.assets_by_type[asset_type].add(asset)
+
+        self._save_registry()
+
+    def get(self, asset_id: str) -> Asset | None:
+        """Get asset by ID."""
+        return self.assets.get(asset_id)
+
+    def get_by_name(
+        self,
+        name: str,
+        version: str | None = None,
+    ) -> Asset | None:
+        """Get asset by name and optionally version.
+
+        Returns the latest version if version not specified.
+        """
+        assets = self.assets_by_name.get(name, [])
+
+        if not assets:
+            return None
+
+        if version:
+            for asset in assets:
+                if asset.version == version:
+                    return asset
+            return None
+
+        # Return latest version
+        return max(assets, key=lambda a: a.metadata.created_at)
+
+    def list_by_type(self, asset_type: str) -> list[Asset]:
+        """List all assets of a specific type."""
+        return list(self.assets_by_type.get(asset_type, set()))
+
+    def list_all(self) -> list[Asset]:
+        """List all registered assets."""
+        return list(self.assets.values())
+
+    def search(
+        self,
+        name: str | None = None,
+        asset_type: str | None = None,
+        tags: dict[str, str] | None = None,
+        created_after: datetime | None = None,
+    ) -> list[Asset]:
+        """Search for assets matching criteria.
+
+        Args:
+            name: Filter by name (substring match)
+            asset_type: Filter by asset type
+            tags: Filter by tags (must match all)
+            created_after: Filter by creation date
+
+        Returns:
+            List of matching assets
+        """
+        results = list(self.assets.values())
+
+        if name:
+            results = [a for a in results if name.lower() in a.name.lower()]
+
+        if asset_type:
+            results = [a for a in results if a.metadata.asset_type == asset_type]
+
+        if tags:
+            results = [a for a in results if all(a.metadata.tags.get(k) == v for k, v in tags.items())]
+
+        if created_after:
+            results = [a for a in results if a.metadata.created_at > created_after]
+
+        return results
+
+    def get_lineage_graph(self, asset_id: str) -> dict:
+        """Get full lineage graph for an asset."""
+        asset = self.get(asset_id)
+        if not asset:
+            return {}
+        return asset.get_lineage()
+
+    def _save_registry(self) -> None:
+        """Save registry metadata to disk."""
+        registry_file = self.registry_dir / "registry.json"
+
+        data = {
+            "assets": {asset_id: asset.to_dict() for asset_id, asset in self.assets.items()},
+        }
+
+        with open(registry_file, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def _load_registry(self) -> None:
+        """Load registry metadata from disk."""
+        registry_file = self.registry_dir / "registry.json"
+
+        if not registry_file.exists():
+            return
+
+        try:
+            with open(registry_file) as f:
+                json.load(f)
+
+            # Note: This is simplified - in production, we'd need to
+            # deserialize the actual asset objects with their data
+            # For now, we just load the metadata
+
+        except Exception:
+            pass
+
+    def clear(self) -> None:
+        """Clear the registry."""
+        self.assets.clear()
+        self.assets_by_name.clear()
+        self.assets_by_type.clear()
+        self._save_registry()
+
+    def stats(self) -> dict[str, int]:
+        """Get registry statistics."""
+        return {
+            "total_assets": len(self.assets),
+            "by_type": {asset_type: len(assets) for asset_type, assets in self.assets_by_type.items()},
+        }