flowyml 1.3.0__tar.gz → 1.4.0__tar.gz

{flowyml-1.3.0 → flowyml-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowyml
-Version: 1.3.0
+Version: 1.4.0
 Summary: Next-Generation ML Pipeline Framework
 License: Apache-2.0
 License-File: LICENSE
@@ -36,11 +36,13 @@ Requires-Dist: httpx (>=0.24,<0.28)
 Requires-Dist: loguru (>=0.7.3,<0.8.0)
 Requires-Dist: numpy (>=1.20.0)
 Requires-Dist: pandas (>=1.3.0)
+Requires-Dist: psycopg2-binary (>=2.9.0)
 Requires-Dist: pydantic (>=2.0.0)
 Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
 Requires-Dist: pytz (>=2024.1,<2025.0)
 Requires-Dist: pyyaml (>=6.0)
 Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
+Requires-Dist: sqlalchemy (>=2.0.0)
 Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: toml (>=0.10.2)
 Requires-Dist: torch (>=2.0.0) ; extra == "pytorch" or extra == "all"

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/observability.py

@@ -1,6 +1,6 @@
 """Observability hooks for monitoring and metrics collection."""
 
-from typing import Protocol, Any, Optional, TYPE_CHECKING
+from typing import Protocol, Any, TYPE_CHECKING
 from datetime import datetime
 from dataclasses import dataclass, field
 
@@ -31,8 +31,8 @@ class PipelineMetricEvent(MetricEvent):
 
     pipeline_name: str = ""
     run_id: str = ""
-    duration_seconds: Optional[float] = None
-    success: Optional[bool] = None
+    duration_seconds: float | None = None
+    success: bool | None = None
 
     def to_dict(self) -> dict[str, Any]:
         base = super().to_dict()
@@ -54,8 +54,8 @@ class StepMetricEvent(MetricEvent):
     step_name: str = ""
     pipeline_name: str = ""
     run_id: str = ""
-    duration_seconds: Optional[float] = None
-    success: Optional[bool] = None
+    duration_seconds: float | None = None
+    success: bool | None = None
     cached: bool = False
 
     def to_dict(self) -> dict[str, Any]:
@@ -196,7 +196,7 @@ class PrometheusMetricsCollector:
 
 
 # Global metrics collector
-_metrics_collector: Optional[MetricsCollector] = None
+_metrics_collector: MetricsCollector | None = None
 
 
 def set_metrics_collector(collector: MetricsCollector) -> None:
@@ -205,6 +205,6 @@ def set_metrics_collector(collector: MetricsCollector) -> None:
     _metrics_collector = collector
 
 
-def get_metrics_collector() -> Optional[MetricsCollector]:
+def get_metrics_collector() -> MetricsCollector | None:
     """Get global metrics collector."""
     return _metrics_collector

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/resources.py

@@ -5,7 +5,7 @@ including CPU, memory, GPU, storage, and node affinity requirements.
 """
 
 from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any
 import re
 
 
@@ -25,7 +25,7 @@ class GPUConfig:
 
     gpu_type: str
     count: int = 1
-    memory: Optional[str] = None
+    memory: str | None = None
 
     def __post_init__(self):
         """Validate GPU configuration."""
@@ -215,11 +215,11 @@ class ResourceRequirements:
         ... )
     """
 
-    cpu: Optional[str] = None
-    memory: Optional[str] = None
-    storage: Optional[str] = None
-    gpu: Optional[GPUConfig] = None
-    node_affinity: Optional[NodeAffinity] = None
+    cpu: str | None = None
+    memory: str | None = None
+    storage: str | None = None
+    gpu: GPUConfig | None = None
+    node_affinity: NodeAffinity | None = None
 
     def __post_init__(self):
         """Validate resource specifications."""
@@ -408,11 +408,11 @@ class ResourceRequirements:
 
 
 def resources(
-    cpu: Optional[str] = None,
-    memory: Optional[str] = None,
-    storage: Optional[str] = None,
-    gpu: Optional[GPUConfig] = None,
-    node_affinity: Optional[NodeAffinity] = None,
+    cpu: str | None = None,
+    memory: str | None = None,
+    storage: str | None = None,
+    gpu: GPUConfig | None = None,
+    node_affinity: NodeAffinity | None = None,
 ) -> ResourceRequirements:
     """Create a ResourceRequirements object with validation.
 

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/retry_policy.py

@@ -1,7 +1,7 @@
 """Retry policies for orchestrators."""
 
 from dataclasses import dataclass
-from typing import Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING
 from flowyml.core.error_handling import RetryConfig, ExponentialBackoff, execute_with_retry
 
 if TYPE_CHECKING:
@@ -61,7 +61,7 @@ def with_retry(orchestrator_method):
             ...
     """
 
-    def wrapper(self, pipeline: "Pipeline", *args, retry_policy: Optional[OrchestratorRetryPolicy] = None, **kwargs):
+    def wrapper(self, pipeline: "Pipeline", *args, retry_policy: OrchestratorRetryPolicy | None = None, **kwargs):
         if retry_policy is None:
             # No retry policy, execute normally
             return orchestrator_method(self, pipeline, *args, **kwargs)

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/scheduler.py

@@ -10,7 +10,7 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 
 from flowyml.core.scheduler_config import SchedulerConfig
 
@@ -77,10 +77,10 @@ class ScheduleExecution:
 
     schedule_name: str
     started_at: datetime
-    completed_at: Optional[datetime] = None
+    completed_at: datetime | None = None
     success: bool = False
-    error: Optional[str] = None
-    duration_seconds: Optional[float] = None
+    error: str | None = None
+    duration_seconds: float | None = None
 
 
 class SchedulerMetrics:
@@ -121,7 +121,7 @@ class SchedulerMetrics:
 class SchedulerPersistence:
     """Persist schedules to SQLite database."""
 
-    def __init__(self, db_path: Optional[str] = None):
+    def __init__(self, db_path: str | None = None):
         self.db_path = db_path or str(Path.cwd() / ".flowyml_scheduler.db")
         self._init_db()
 
@@ -230,7 +230,7 @@ class SchedulerPersistence:
 class DistributedLock:
     """Distributed lock for coordinating multiple scheduler instances."""
 
-    def __init__(self, backend: str = "file", redis_url: Optional[str] = None):
+    def __init__(self, backend: str = "file", redis_url: str | None = None):
         self.backend = backend
         self.redis_url = redis_url
         self._redis = None
@@ -286,9 +286,9 @@ class PipelineScheduler:
 
     def __init__(
         self,
-        config: Optional[SchedulerConfig] = None,
-        on_success: Optional[Callable] = None,
-        on_failure: Optional[Callable] = None,
+        config: SchedulerConfig | None = None,
+        on_success: Callable | None = None,
+        on_failure: Callable | None = None,
     ):
         self.config = config or SchedulerConfig.from_env()
         self.schedules: dict[str, Schedule] = {}

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/scheduler_config.py

@@ -1,7 +1,6 @@
 """Scheduler configuration."""
 
 import os
-from typing import Optional
 from pydantic import BaseModel
 
 
@@ -9,10 +8,10 @@ class SchedulerConfig(BaseModel):
     """Scheduler configuration."""
 
     persist_schedules: bool = True
-    db_path: Optional[str] = None
+    db_path: str | None = None
     distributed: bool = False
     lock_backend: str = "file"  # "file", "redis"
-    redis_url: Optional[str] = None
+    redis_url: str | None = None
     check_interval_seconds: int = 10
     max_concurrent_runs: int = 5
     timezone: str = "UTC"

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/core/submission_result.py

@@ -1,6 +1,6 @@
 """Submission result for async pipeline execution."""
 
-from typing import Any, Optional
+from typing import Any
 from collections.abc import Callable
 
 
@@ -15,8 +15,8 @@ class SubmissionResult:
     def __init__(
         self,
         job_id: str,
-        wait_for_completion: Optional[Callable[[], None]] = None,
-        metadata: Optional[dict[str, Any]] = None,
+        wait_for_completion: Callable[[], None] | None = None,
+        metadata: dict[str, Any] | None = None,
     ):
         """Initialize a submission result.
 
@@ -29,7 +29,7 @@ class SubmissionResult:
         self.wait_for_completion = wait_for_completion
         self.metadata = metadata or {}
 
-    def wait(self, timeout: Optional[int] = None) -> None:
+    def wait(self, timeout: int | None = None) -> None:
         """Wait for the pipeline run to complete.
 
         Args:

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/stacks/bridge.py

@@ -6,7 +6,7 @@ frameworks (ZenML, Airflow, Prefect, etc.) using rule-based adaptation.
 
 import inspect
 from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any
 import logging
 
 from flowyml.stacks.components import (
@@ -25,8 +25,8 @@ class AdaptationRule:
     """Rule for adapting an external component."""
 
     # Matching criteria
-    source_type: Optional[str] = None  # e.g., "zenml.orchestrators.base.BaseOrchestrator"
-    name_pattern: Optional[str] = None  # e.g., ".*Orchestrator"
+    source_type: str | None = None  # e.g., "zenml.orchestrators.base.BaseOrchestrator"
+    name_pattern: str | None = None  # e.g., ".*Orchestrator"
     has_methods: list[str] = field(default_factory=list)  # e.g., ["run", "prepare_pipeline"]
 
     # Adaptation logic
@@ -49,7 +49,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        config: Optional[dict[str, Any]] = None,
+        config: dict[str, Any] | None = None,
     ) -> type[StackComponent]:
         """Dynamically create a wrapper class based on rules.
 
@@ -75,7 +75,7 @@ class GenericBridge:
         else:
             return self._create_generic_wrapper(external_class, name, component_type, rule)
 
-    def _find_matching_rule(self, external_class: Any) -> Optional[AdaptationRule]:
+    def _find_matching_rule(self, external_class: Any) -> AdaptationRule | None:
         """Find the first rule that matches the external class."""
         for rule in self.rules:
             # Check source type
@@ -120,7 +120,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[Orchestrator]:
         """Create a wrapper for an Orchestrator."""
 
@@ -174,7 +174,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[ArtifactStore]:
         """Create a wrapper for an Artifact Store."""
 
@@ -216,7 +216,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[ContainerRegistry]:
         """Create a wrapper for a Container Registry."""
 
@@ -259,7 +259,7 @@ class GenericBridge:
         external_class: Any,
         name: str,
         comp_type: ComponentType,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[StackComponent]:
         """Create a generic wrapper."""
 

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/stacks/plugins.py

@@ -14,7 +14,7 @@ import inspect
 import subprocess
 import sys
 from dataclasses import dataclass, field
-from typing import Any, Optional, Protocol, runtime_checkable
+from typing import Any, Protocol, runtime_checkable
 
 from flowyml.stacks.components import (
     StackComponent,
@@ -48,7 +48,7 @@ class PluginBridge(Protocol):
         self,
         component_class: Any,
         name: str,
-        config: Optional[dict[str, Any]] = None,
+        config: dict[str, Any] | None = None,
     ) -> type[StackComponent]:
         """Wrap an external component class into a flowyml component."""
         ...

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/stacks/registry.py

@@ -214,6 +214,27 @@ def get_active_stack() -> Stack | None:
     Returns:
         Active stack or None
     """
+    # Check for remote execution mode in global config
+    from flowyml.utils.config import get_config
+
+    config = get_config()
+
+    if config.execution_mode == "remote" and config.remote_server_url:
+        # Dynamically create a remote logging stack
+        from flowyml.core.executor import LocalExecutor
+        from flowyml.storage.remote import RemoteMetadataStore, RemoteArtifactStore
+        from flowyml.stacks.base import Stack
+        import os
+
+        api_token = os.getenv("FLOWYML_API_TOKEN")
+
+        return Stack(
+            name="remote_logging",
+            executor=LocalExecutor(),
+            metadata_store=RemoteMetadataStore(api_url=config.remote_server_url, api_token=api_token),
+            artifact_store=RemoteArtifactStore(api_url=config.remote_server_url, api_token=api_token),
+        )
+
     return get_registry().get_active_stack()
 
 

{flowyml-1.3.0 → flowyml-1.4.0}/flowyml/storage/materializers/base.py

@@ -99,6 +99,27 @@ class MaterializerRegistry:
 
         return None
 
+    def get_materializer_by_type_name(self, type_name: str) -> BaseMaterializer | None:
+        """Get materializer by type name string.
+
+        Args:
+            type_name: Full type name (e.g. 'pandas.core.frame.DataFrame')
+
+        Returns:
+            Materializer instance or None
+        """
+        # Check if we have a direct mapping
+        if type_name in self._materializers:
+            return self._materializers[type_name]()
+
+        # Try to find by just class name if full path fails (less reliable but useful fallback)
+        simple_name = type_name.split(".")[-1]
+        for key, materializer_cls in self._materializers.items():
+            if key.split(".")[-1] == simple_name:
+                return materializer_cls()
+
+        return None
+
     def list_materializers(self) -> dict[str, type[BaseMaterializer]]:
         """List all registered materializers.
 
@@ -131,3 +152,15 @@ def get_materializer(obj: Any) -> BaseMaterializer | None:
         Materializer instance or None
     """
     return materializer_registry.get_materializer(obj)
+
+
+def get_materializer_by_type_name(type_name: str) -> BaseMaterializer | None:
+    """Get materializer by type name string from global registry.
+
+    Args:
+        type_name: Full type name (e.g. 'pandas.core.frame.DataFrame')
+
+    Returns:
+        Materializer instance or None
+    """
+    return materializer_registry.get_materializer_by_type_name(type_name)

flowyml-1.4.0/flowyml/storage/metadata.py

@@ -0,0 +1,53 @@
+"""Metadata storage backends for flowyml."""
+
+from abc import ABC, abstractmethod
+
+
+class MetadataStore(ABC):
+    """Base class for metadata storage backends."""
+
+    @abstractmethod
+    def save_run(self, run_id: str, metadata: dict) -> None:
+        """Save run metadata."""
+        pass
+
+    @abstractmethod
+    def load_run(self, run_id: str) -> dict | None:
+        """Load run metadata."""
+        pass
+
+    @abstractmethod
+    def list_runs(self, limit: int | None = None) -> list[dict]:
+        """List all runs."""
+        pass
+
+    @abstractmethod
+    def list_pipelines(self) -> list[str]:
+        """List all unique pipeline names."""
+        pass
+
+    @abstractmethod
+    def save_artifact(self, artifact_id: str, metadata: dict) -> None:
+        """Save artifact metadata."""
+        pass
+
+    @abstractmethod
+    def load_artifact(self, artifact_id: str) -> dict | None:
+        """Load artifact metadata."""
+        pass
+
+    @abstractmethod
+    def list_assets(self, limit: int | None = None, **filters) -> list[dict]:
+        """List assets with optional filters."""
+        pass
+
+    @abstractmethod
+    def query(self, **filters) -> list[dict]:
+        """Query runs with filters."""
+        pass
+
+
+from flowyml.storage.sql import SQLMetadataStore  # noqa: E402
+
+# Alias for backward compatibility
+SQLiteMetadataStore = SQLMetadataStore