flowyml 1.1.0__py3-none-any.whl

flowyml/__init__.py

@@ -0,0 +1,207 @@
+"""🌊 flowyml - Next-Generation ML Pipeline Framework.
+
+flowyml is a developer-first ML pipeline orchestration framework that combines
+the simplicity of Metaflow with the power of ZenML and the elegance of
+asset-centric design.
+"""
+
+__version__ = "0.1.0"
+__author__ = "flowyml Team"
+
+# Core imports
+from flowyml.core.context import Context, context
+from flowyml.core.step import step, Step
+from flowyml.core.pipeline import Pipeline
+from flowyml.core.executor import Executor, LocalExecutor
+from flowyml.core.cache import CacheStrategy
+from flowyml.core.conditional import Condition, ConditionalBranch, Switch, when, unless
+from flowyml.core.parallel import ParallelExecutor, DataParallelExecutor, BatchExecutor, parallel_map
+from flowyml.core.error_handling import (
+    CircuitBreaker,
+    ExponentialBackoff,
+    RetryConfig,
+    FallbackHandler,
+    retry,
+    on_failure,
+)
+
+# Asset imports
+from flowyml.assets.base import Asset
+from flowyml.assets.dataset import Dataset
+from flowyml.assets.model import Model
+from flowyml.assets.metrics import Metrics
+from flowyml.assets.artifact import Artifact
+from flowyml.assets.featureset import FeatureSet
+from flowyml.assets.report import Report
+from flowyml.assets.registry import AssetRegistry
+
+# Stack imports
+from flowyml.stacks.base import Stack
+from flowyml.stacks.local import LocalStack
+
+# Tracking imports
+from flowyml.tracking.experiment import Experiment
+from flowyml.tracking.runs import Run
+
+# Registry imports
+from flowyml.registry.model_registry import ModelRegistry, ModelVersion, ModelStage
+
+# Storage imports (for advanced usage)
+from flowyml.storage import (
+    ArtifactStore,
+    LocalArtifactStore,
+    MetadataStore,
+    SQLiteMetadataStore,
+    materializer_registry,
+)
+
+# Monitoring & Integrations
+from flowyml.monitoring.llm import trace_llm, tracer
+from flowyml.monitoring.data import detect_drift, compute_stats
+from flowyml.monitoring.notifications import (
+    NotificationManager,
+    configure_notifications,
+    get_notifier,
+    ConsoleNotifier,
+    SlackNotifier,
+    EmailNotifier,
+)
+from flowyml.integrations.keras import FlowymlKerasCallback
+
+# Advanced Features
+from flowyml.core.scheduler import PipelineScheduler
+from flowyml.core.approval import approval, ApprovalStep
+from flowyml.core.checkpoint import PipelineCheckpoint, checkpoint_enabled_pipeline
+from flowyml.core.templates import create_from_template, list_templates, TEMPLATES
+from flowyml.tracking.leaderboard import ModelLeaderboard, compare_runs
+from flowyml.core.versioning import VersionedPipeline, PipelineVersion
+from flowyml.core.project import Project, ProjectManager
+from flowyml.core.advanced_cache import (
+    ContentBasedCache,
+    SharedCache,
+    SmartCache,
+    memoize,
+)
+from flowyml.utils.debug import (
+    StepDebugger,
+    PipelineDebugger,
+    debug_step,
+    trace_step,
+    profile_step,
+    inspect_step,
+)
+from flowyml.utils.performance import (
+    LazyValue,
+    lazy_property,
+    IncrementalComputation,
+    GPUResourceManager,
+    optimize_dataframe,
+    batch_iterator,
+)
+from flowyml.registry.pipeline_registry import pipeline_registry, register_pipeline
+
+__all__ = [
+    # Core
+    "Context",
+    "context",
+    "step",
+    "Step",
+    "Pipeline",
+    "Executor",
+    "LocalExecutor",
+    "CacheStrategy",
+    # Conditional & Control Flow
+    "Condition",
+    "ConditionalBranch",
+    "Switch",
+    "when",
+    "unless",
+    # Parallel Execution
+    "ParallelExecutor",
+    "DataParallelExecutor",
+    "BatchExecutor",
+    "parallel_map",
+    # Error Handling
+    "CircuitBreaker",
+    "ExponentialBackoff",
+    "RetryConfig",
+    "FallbackHandler",
+    "retry",
+    "on_failure",
+    # Assets
+    "Asset",
+    "Dataset",
+    "Model",
+    "Metrics",
+    "Artifact",
+    "FeatureSet",
+    "Report",
+    "AssetRegistry",
+    # Stacks
+    "Stack",
+    "LocalStack",
+    # Tracking
+    "Experiment",
+    "Run",
+    # Registry
+    "ModelRegistry",
+    "ModelVersion",
+    "ModelStage",
+    # Storage
+    "ArtifactStore",
+    "LocalArtifactStore",
+    "MetadataStore",
+    "SQLiteMetadataStore",
+    "materializer_registry",
+    # Monitoring & Integrations
+    "trace_llm",
+    "tracer",
+    "detect_drift",
+    "compute_stats",
+    "FlowymlKerasCallback",
+    # Advanced Features
+    "PipelineScheduler",
+    "approval",
+    "ApprovalStep",
+    "PipelineCheckpoint",
+    "checkpoint_enabled_pipeline",
+    "create_from_template",
+    "list_templates",
+    "TEMPLATES",
+    "ModelLeaderboard",
+    "compare_runs",
+    "NotificationManager",
+    "configure_notifications",
+    "get_notifier",
+    "ConsoleNotifier",
+    "SlackNotifier",
+    "EmailNotifier",
+    # Versioning & Projects
+    "VersionedPipeline",
+    "PipelineVersion",
+    "Project",
+    "ProjectManager",
+    # Advanced Caching
+    "ContentBasedCache",
+    "SharedCache",
+    "SmartCache",
+    "memoize",
+    # Debugging
+    "StepDebugger",
+    "PipelineDebugger",
+    "debug_step",
+    "trace_step",
+    "profile_step",
+    "inspect_step",
+    # Performance
+    "LazyValue",
+    "lazy_property",
+    "ParallelExecutor",
+    "IncrementalComputation",
+    "GPUResourceManager",
+    "optimize_dataframe",
+    "batch_iterator",
+    # Registry
+    "pipeline_registry",
+    "register_pipeline",
+]

flowyml/assets/__init__.py

@@ -0,0 +1,22 @@
+"""Asset-centric design for ML pipelines."""
+
+from flowyml.assets.base import Asset, AssetMetadata
+from flowyml.assets.dataset import Dataset
+from flowyml.assets.model import Model
+from flowyml.assets.metrics import Metrics
+from flowyml.assets.artifact import Artifact
+from flowyml.assets.featureset import FeatureSet
+from flowyml.assets.report import Report
+from flowyml.assets.registry import AssetRegistry
+
+__all__ = [
+    "Asset",
+    "AssetMetadata",
+    "Dataset",
+    "Model",
+    "Metrics",
+    "Artifact",
+    "FeatureSet",
+    "Report",
+    "AssetRegistry",
+]

flowyml/assets/artifact.py

@@ -0,0 +1,40 @@
+"""Artifact Asset - Represents generic artifacts (configs, checkpoints, etc)."""
+
+from typing import Any
+from flowyml.assets.base import Asset
+
+
+class Artifact(Asset):
+    """Generic artifact asset for configs, checkpoints, reports, etc.
+
+    Example:
+        >>> config = Artifact(name="training_config", artifact_type="config", data={"lr": 0.001, "epochs": 10})
+    """
+
+    def __init__(
+        self,
+        name: str,
+        artifact_type: str = "generic",
+        version: str | None = None,
+        data: Any = None,
+        file_path: str | None = None,
+        parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            data=data,
+            parent=parent,
+            tags=tags,
+            properties=properties,
+        )
+
+        self.artifact_type = artifact_type
+        self.file_path = file_path
+
+        # Add artifact-specific properties
+        self.metadata.properties["artifact_type"] = artifact_type
+        if file_path:
+            self.metadata.properties["file_path"] = file_path

flowyml/assets/base.py

@@ -0,0 +1,209 @@
+"""Base Asset - Foundation for all ML assets in flowyml."""
+
+import hashlib
+import json
+from typing import Any, Optional
+from datetime import datetime
+from dataclasses import dataclass, field
+from uuid import uuid4
+
+
+@dataclass
+class AssetMetadata:
+    """Metadata for an asset."""
+
+    asset_id: str
+    name: str
+    version: str
+    asset_type: str
+    created_at: datetime
+    created_by: str
+    parent_ids: list[str] = field(default_factory=list)
+    tags: dict[str, str] = field(default_factory=dict)
+    properties: dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "asset_id": self.asset_id,
+            "name": self.name,
+            "version": self.version,
+            "asset_type": self.asset_type,
+            "created_at": self.created_at.isoformat(),
+            "created_by": self.created_by,
+            "parent_ids": self.parent_ids,
+            "tags": self.tags,
+            "properties": self.properties,
+        }
+
+
+class Asset:
+    """Base class for all ML assets (datasets, models, features, etc).
+
+    Assets are first-class objects in flowyml pipelines with full lineage tracking.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: str | None = None,
+        data: Any = None,
+        parent: Optional["Asset"] = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
+    ):
+        self.name = name
+        self.version = version or "v1.0.0"
+        self.data = data
+        self.asset_id = str(uuid4())
+
+        # Metadata
+        self.metadata = AssetMetadata(
+            asset_id=self.asset_id,
+            name=name,
+            version=self.version,
+            asset_type=self.__class__.__name__,
+            created_at=datetime.now(),
+            created_by="flowyml",
+            parent_ids=[parent.asset_id] if parent else [],
+            tags=tags or {},
+            properties=properties or {},
+        )
+
+        # Lineage tracking
+        self.parents: list[Asset] = [parent] if parent else []
+        self.children: list[Asset] = []
+
+        if parent:
+            parent.children.append(self)
+
+    @classmethod
+    def create(
+        cls,
+        data: Any,
+        name: str | None = None,
+        version: str | None = None,
+        parent: Optional["Asset"] = None,
+        **kwargs: Any,
+    ) -> "Asset":
+        """Factory method to create an asset.
+
+        Args:
+            data: The actual data/object
+            name: Asset name
+            version: Asset version
+            parent: Parent asset for lineage
+            **kwargs: Additional metadata
+
+        Returns:
+            New asset instance
+        """
+        asset_name = name or f"{cls.__name__}_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+
+        # Extract tags and properties if passed explicitly
+        tags = kwargs.pop("tags", {})
+        props = kwargs.pop("properties", {})
+        # Merge remaining kwargs into properties
+        props.update(kwargs)
+
+        return cls(
+            name=asset_name,
+            version=version,
+            data=data,
+            parent=parent,
+            tags=tags,
+            properties=props,
+        )
+
+    def get_hash(self) -> str:
+        """Generate hash of asset for caching/versioning."""
+        content = json.dumps(
+            {
+                "name": self.name,
+                "version": self.version,
+                "type": self.metadata.asset_type,
+                "created_at": self.metadata.created_at.isoformat(),
+            },
+            sort_keys=True,
+        )
+        return hashlib.sha256(content.encode()).hexdigest()[:16]
+
+    def get_lineage(self, depth: int = -1) -> dict[str, Any]:
+        """Get asset lineage.
+
+        Args:
+            depth: How many levels to traverse (-1 for all)
+
+        Returns:
+            Lineage tree as nested dict
+        """
+        lineage = {
+            "asset": {
+                "asset_id": self.asset_id,
+                "name": self.name,
+                "type": self.metadata.asset_type,
+                "version": self.version,
+            },
+            "parents": [],
+            "children": [],
+        }
+
+        if depth != 0:
+            next_depth = depth - 1 if depth > 0 else -1
+            lineage["parents"] = [p.get_lineage(next_depth) for p in self.parents]
+            lineage["children"] = [c.get_lineage(next_depth) for c in self.children]
+
+        return lineage
+
+    def get_all_ancestors(self) -> set["Asset"]:
+        """Get all ancestor assets."""
+        ancestors = set()
+
+        def traverse(asset) -> None:
+            for parent in asset.parents:
+                if parent not in ancestors:
+                    ancestors.add(parent)
+                    traverse(parent)
+
+        traverse(self)
+        return ancestors
+
+    def get_all_descendants(self) -> set["Asset"]:
+        """Get all descendant assets."""
+        descendants = set()
+
+        def traverse(asset) -> None:
+            for child in asset.children:
+                if child not in descendants:
+                    descendants.add(child)
+                    traverse(child)
+
+        traverse(self)
+        return descendants
+
+    def add_tag(self, key: str, value: str) -> None:
+        """Add a tag to the asset."""
+        self.metadata.tags[key] = value
+
+    def add_property(self, key: str, value: Any) -> None:
+        """Add a property to the asset."""
+        self.metadata.properties[key] = value
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert asset to dictionary."""
+        return {
+            "metadata": self.metadata.to_dict(),
+            "lineage": {
+                "parents": [p.asset_id for p in self.parents],
+                "children": [c.asset_id for c in self.children],
+            },
+        }
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}(name='{self.name}', version='{self.version}')"
+
+    def __hash__(self):
+        return hash(self.asset_id)
+
+    def __eq__(self, other):
+        return isinstance(other, Asset) and self.asset_id == other.asset_id

flowyml/assets/dataset.py

@@ -0,0 +1,100 @@
+"""Dataset Asset - Represents ML datasets with schema validation."""
+
+from typing import Any
+from flowyml.assets.base import Asset
+
+
+class Dataset(Asset):
+    """Dataset asset with schema and lineage tracking.
+
+    Example:
+        >>> raw_data = Dataset(
+        ...     name="imagenet_train",
+        ...     version="v2.0",
+        ...     data=train_dataset,
+        ...     properties={"size": "150GB", "samples": 1_281_167},
+        ... )
+    """
+
+    def __init__(
+        self,
+        name: str,
+        version: str | None = None,
+        data: Any = None,
+        schema: Any | None = None,
+        location: str | None = None,
+        parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
+    ):
+        super().__init__(
+            name=name,
+            version=version,
+            data=data,
+            parent=parent,
+            tags=tags,
+            properties=properties,
+        )
+
+        self.schema = schema
+        self.location = location
+
+        # Add dataset-specific properties
+        if schema:
+            self.metadata.properties["schema"] = str(schema)
+        if location:
+            self.metadata.properties["location"] = location
+
+    @property
+    def size(self) -> int | None:
+        """Get dataset size if available."""
+        return self.metadata.properties.get("size")
+
+    @property
+    def num_samples(self) -> int | None:
+        """Get number of samples if available."""
+        return self.metadata.properties.get("samples") or self.metadata.properties.get("num_samples")
+
+    def validate_schema(self) -> bool:
+        """Validate data against schema (placeholder)."""
+        if self.schema is None or self.data is None:
+            return True
+        # Schema validation would go here
+        return True
+
+    def split(self, train_ratio: float = 0.8, name_prefix: str | None = None) -> tuple["Dataset", "Dataset"]:
+        """Split dataset into train/test.
+
+        Args:
+            train_ratio: Ratio for training split
+            name_prefix: Prefix for split dataset names
+
+        Returns:
+            Tuple of (train_dataset, test_dataset)
+        """
+        prefix = name_prefix or self.name
+
+        # Placeholder - actual splitting logic would depend on data type
+        _ = train_ratio  # Unused in placeholder
+        train_data = self.data  # Would actually split the data
+        test_data = self.data
+
+        train_dataset = Dataset(
+            name=f"{prefix}_train",
+            version=self.version,
+            data=train_data,
+            schema=self.schema,
+            parent=self,
+            tags={**self.metadata.tags, "split": "train"},
+        )
+
+        test_dataset = Dataset(
+            name=f"{prefix}_test",
+            version=self.version,
+            data=test_data,
+            schema=self.schema,
+            parent=self,
+            tags={**self.metadata.tags, "split": "test"},
+        )
+
+        return train_dataset, test_dataset