flowyml 1.3.0__tar.gz → 1.5.0__tar.gz

{flowyml-1.3.0 → flowyml-1.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowyml
-Version: 1.3.0
+Version: 1.5.0
 Summary: Next-Generation ML Pipeline Framework
 License: Apache-2.0
 License-File: LICENSE
@@ -36,11 +36,13 @@ Requires-Dist: httpx (>=0.24,<0.28)
 Requires-Dist: loguru (>=0.7.3,<0.8.0)
 Requires-Dist: numpy (>=1.20.0)
 Requires-Dist: pandas (>=1.3.0)
+Requires-Dist: psycopg2-binary (>=2.9.0)
 Requires-Dist: pydantic (>=2.0.0)
 Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
 Requires-Dist: pytz (>=2024.1,<2025.0)
 Requires-Dist: pyyaml (>=6.0)
 Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
+Requires-Dist: sqlalchemy (>=2.0.0)
 Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: toml (>=0.10.2)
 Requires-Dist: torch (>=2.0.0) ; extra == "pytorch" or extra == "all"

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/execution_status.py

@@ -25,6 +25,7 @@ class ExecutionStatus(str, Enum):
     # Intermediate states
     STOPPING = "stopping"
     CANCELLING = "cancelling"
+    DEAD = "dead"
 
     @property
     def is_finished(self) -> bool:

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/executor.py

@@ -7,6 +7,133 @@ from typing import Any
 from dataclasses import dataclass
 from datetime import datetime
 
+import threading
+import ctypes
+import requests
+import os
+import inspect
+
+
+class StopExecutionError(Exception):
+    """Exception raised when execution is stopped externally."""
+
+    pass
+
+
+# Alias for backwards compatibility
+StopExecution = StopExecutionError
+
+
+def _async_raise(tid, exctype):
+    """Raises an exception in the threads with id tid"""
+    if not inspect.isclass(exctype):
+        raise TypeError("Only types can be raised (not instances)")
+    res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), ctypes.py_object(exctype))
+    if res == 0:
+        raise ValueError("invalid thread id")
+    if res != 1:
+        # """if it returns a number greater than one, you're in trouble,
+        # and you should call it again with exc=NULL to revert the effect"""
+        ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(tid), None)
+        raise SystemError("PyThreadState_SetAsyncExc failed")
+
+
+class LogCapture:
+    """Context manager to capture stdout/stderr for streaming to the server."""
+
+    def __init__(self):
+        self._buffer = []
+        self._lock = threading.Lock()
+
+    def write(self, text):
+        if text.strip():
+            with self._lock:
+                self._buffer.append(text)
+
+    def flush(self):
+        pass
+
+    def get_and_clear(self) -> list[str]:
+        with self._lock:
+            lines = self._buffer[:]
+            self._buffer.clear()
+            return lines
+
+
+class MonitorThread(threading.Thread):
+    """Background thread that sends heartbeats and flushes logs to the server."""
+
+    def __init__(
+        self,
+        run_id: str,
+        step_name: str,
+        target_tid: int,
+        log_capture: LogCapture | None = None,
+        interval: int = 5,
+    ):
+        super().__init__()
+        self.run_id = run_id
+        self.step_name = step_name
+        self.target_tid = target_tid
+        self.log_capture = log_capture
+        self.interval = interval
+        self._stop_event = threading.Event()
+        self.api_url = os.getenv("FLOWYML_SERVER_URL", "http://localhost:8000")
+
+    def stop(self):
+        self._stop_event.set()
+
+    def _flush_logs(self):
+        """Send captured logs to the server."""
+        if not self.log_capture:
+            return
+
+        lines = self.log_capture.get_and_clear()
+        if not lines:
+            return
+
+        content = "".join(lines)
+        with contextlib.suppress(Exception):
+            requests.post(
+                f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/logs",
+                json={
+                    "content": content,
+                    "level": "INFO",
+                    "timestamp": datetime.now().isoformat(),
+                },
+                timeout=2,
+            )
+
+    def run(self):
+        while not self._stop_event.is_set():
+            try:
+                # Send heartbeat
+                response = requests.post(
+                    f"{self.api_url}/api/runs/{self.run_id}/steps/{self.step_name}/heartbeat",
+                    json={"step_name": self.step_name, "status": "running"},
+                    timeout=2,
+                )
+                if response.status_code == 200:
+                    data = response.json()
+                    if data.get("action") == "stop":
+                        print(f"Received stop signal for step {self.step_name}")
+                        _async_raise(self.target_tid, StopExecution)
+                        break
+            except Exception:
+                pass  # Ignore heartbeat failures
+
+            # Flush logs
+            self._flush_logs()
+
+            self._stop_event.wait(self.interval)
+
+        # Final log flush
+        self._flush_logs()
+
+
+# Keep HeartbeatThread as an alias for backwards compatibility
+HeartbeatThread = MonitorThread
+
 
 @dataclass
 class ExecutionResult:
@@ -103,8 +230,6 @@ class LocalExecutor(Executor):
                 # or just pass what we can.
                 # A simple approach: pass nothing if it takes no args, or kwargs if it does.
                 # But inspect is safer.
-                import inspect
-
                 sig = inspect.signature(step.condition)
                 kwargs = {**inputs, **context_params}
 
@@ -157,7 +282,54 @@ class LocalExecutor(Executor):
                 kwargs = {**inputs, **context_params}
 
                 # Execute step
-                result = step.func(**kwargs)
+                monitor_thread = None
+                log_capture = None
+                original_stdout = None
+                original_stderr = None
+                try:
+                    # Start monitoring thread with log capture if run_id is present
+                    if run_id:
+                        import sys
+
+                        log_capture = LogCapture()
+                        original_stdout = sys.stdout
+                        original_stderr = sys.stderr
+                        sys.stdout = log_capture
+                        sys.stderr = log_capture
+
+                        monitor_thread = MonitorThread(
+                            run_id=run_id,
+                            step_name=step.name,
+                            target_tid=threading.get_ident(),
+                            log_capture=log_capture,
+                        )
+                        monitor_thread.start()
+
+                    result = step.func(**kwargs)
+                except StopExecution:
+                    duration = time.time() - start_time
+                    return ExecutionResult(
+                        step_name=step.name,
+                        success=False,
+                        error="Execution stopped by user",
+                        duration_seconds=duration,
+                        retries=retries,
+                    )
+                finally:
+                    # Restore stdout/stderr
+                    if original_stdout:
+                        import sys
+
+                        sys.stdout = original_stdout
+                    if original_stderr:
+                        import sys
+
+                        sys.stderr = original_stderr
+
+                    # Stop monitor thread
+                    if monitor_thread:
+                        monitor_thread.stop()
+                        monitor_thread.join()
 
                 # Materialize output if artifact store is available
                 artifact_uri = None

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/observability.py

@@ -1,6 +1,6 @@
 """Observability hooks for monitoring and metrics collection."""
 
-from typing import Protocol, Any, Optional, TYPE_CHECKING
+from typing import Protocol, Any, TYPE_CHECKING
 from datetime import datetime
 from dataclasses import dataclass, field
 
@@ -31,8 +31,8 @@ class PipelineMetricEvent(MetricEvent):
 
     pipeline_name: str = ""
     run_id: str = ""
-    duration_seconds: Optional[float] = None
-    success: Optional[bool] = None
+    duration_seconds: float | None = None
+    success: bool | None = None
 
     def to_dict(self) -> dict[str, Any]:
         base = super().to_dict()
@@ -54,8 +54,8 @@ class StepMetricEvent(MetricEvent):
     step_name: str = ""
     pipeline_name: str = ""
     run_id: str = ""
-    duration_seconds: Optional[float] = None
-    success: Optional[bool] = None
+    duration_seconds: float | None = None
+    success: bool | None = None
     cached: bool = False
 
     def to_dict(self) -> dict[str, Any]:
@@ -196,7 +196,7 @@ class PrometheusMetricsCollector:
 
 
 # Global metrics collector
-_metrics_collector: Optional[MetricsCollector] = None
+_metrics_collector: MetricsCollector | None = None
 
 
 def set_metrics_collector(collector: MetricsCollector) -> None:
@@ -205,6 +205,6 @@ def set_metrics_collector(collector: MetricsCollector) -> None:
     _metrics_collector = collector
 
 
-def get_metrics_collector() -> Optional[MetricsCollector]:
+def get_metrics_collector() -> MetricsCollector | None:
     """Get global metrics collector."""
     return _metrics_collector

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/resources.py

@@ -5,7 +5,7 @@ including CPU, memory, GPU, storage, and node affinity requirements.
 """
 
 from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any
 import re
 
 
@@ -25,7 +25,7 @@ class GPUConfig:
 
     gpu_type: str
     count: int = 1
-    memory: Optional[str] = None
+    memory: str | None = None
 
     def __post_init__(self):
         """Validate GPU configuration."""
@@ -215,11 +215,11 @@ class ResourceRequirements:
         ... )
     """
 
-    cpu: Optional[str] = None
-    memory: Optional[str] = None
-    storage: Optional[str] = None
-    gpu: Optional[GPUConfig] = None
-    node_affinity: Optional[NodeAffinity] = None
+    cpu: str | None = None
+    memory: str | None = None
+    storage: str | None = None
+    gpu: GPUConfig | None = None
+    node_affinity: NodeAffinity | None = None
 
     def __post_init__(self):
         """Validate resource specifications."""
@@ -408,11 +408,11 @@ class ResourceRequirements:
 
 
 def resources(
-    cpu: Optional[str] = None,
-    memory: Optional[str] = None,
-    storage: Optional[str] = None,
-    gpu: Optional[GPUConfig] = None,
-    node_affinity: Optional[NodeAffinity] = None,
+    cpu: str | None = None,
+    memory: str | None = None,
+    storage: str | None = None,
+    gpu: GPUConfig | None = None,
+    node_affinity: NodeAffinity | None = None,
 ) -> ResourceRequirements:
     """Create a ResourceRequirements object with validation.
 

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/retry_policy.py

@@ -1,7 +1,7 @@
 """Retry policies for orchestrators."""
 
 from dataclasses import dataclass
-from typing import Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING
 from flowyml.core.error_handling import RetryConfig, ExponentialBackoff, execute_with_retry
 
 if TYPE_CHECKING:
@@ -61,7 +61,7 @@ def with_retry(orchestrator_method):
             ...
     """
 
-    def wrapper(self, pipeline: "Pipeline", *args, retry_policy: Optional[OrchestratorRetryPolicy] = None, **kwargs):
+    def wrapper(self, pipeline: "Pipeline", *args, retry_policy: OrchestratorRetryPolicy | None = None, **kwargs):
         if retry_policy is None:
             # No retry policy, execute normally
             return orchestrator_method(self, pipeline, *args, **kwargs)

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/scheduler.py

@@ -10,7 +10,7 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Any, Optional
+from typing import Any
 
 from flowyml.core.scheduler_config import SchedulerConfig
 
@@ -77,10 +77,10 @@ class ScheduleExecution:
 
     schedule_name: str
     started_at: datetime
-    completed_at: Optional[datetime] = None
+    completed_at: datetime | None = None
     success: bool = False
-    error: Optional[str] = None
-    duration_seconds: Optional[float] = None
+    error: str | None = None
+    duration_seconds: float | None = None
 
 
 class SchedulerMetrics:
@@ -121,7 +121,7 @@ class SchedulerMetrics:
 class SchedulerPersistence:
     """Persist schedules to SQLite database."""
 
-    def __init__(self, db_path: Optional[str] = None):
+    def __init__(self, db_path: str | None = None):
         self.db_path = db_path or str(Path.cwd() / ".flowyml_scheduler.db")
         self._init_db()
 
@@ -230,7 +230,7 @@ class SchedulerPersistence:
 class DistributedLock:
     """Distributed lock for coordinating multiple scheduler instances."""
 
-    def __init__(self, backend: str = "file", redis_url: Optional[str] = None):
+    def __init__(self, backend: str = "file", redis_url: str | None = None):
         self.backend = backend
         self.redis_url = redis_url
         self._redis = None
@@ -286,9 +286,9 @@ class PipelineScheduler:
 
     def __init__(
         self,
-        config: Optional[SchedulerConfig] = None,
-        on_success: Optional[Callable] = None,
-        on_failure: Optional[Callable] = None,
+        config: SchedulerConfig | None = None,
+        on_success: Callable | None = None,
+        on_failure: Callable | None = None,
     ):
         self.config = config or SchedulerConfig.from_env()
         self.schedules: dict[str, Schedule] = {}

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/scheduler_config.py

@@ -1,7 +1,6 @@
 """Scheduler configuration."""
 
 import os
-from typing import Optional
 from pydantic import BaseModel
 
 
@@ -9,10 +8,10 @@ class SchedulerConfig(BaseModel):
     """Scheduler configuration."""
 
     persist_schedules: bool = True
-    db_path: Optional[str] = None
+    db_path: str | None = None
     distributed: bool = False
     lock_backend: str = "file"  # "file", "redis"
-    redis_url: Optional[str] = None
+    redis_url: str | None = None
     check_interval_seconds: int = 10
     max_concurrent_runs: int = 5
     timezone: str = "UTC"

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/core/submission_result.py

@@ -1,6 +1,6 @@
 """Submission result for async pipeline execution."""
 
-from typing import Any, Optional
+from typing import Any
 from collections.abc import Callable
 
 
@@ -15,8 +15,8 @@ class SubmissionResult:
     def __init__(
         self,
         job_id: str,
-        wait_for_completion: Optional[Callable[[], None]] = None,
-        metadata: Optional[dict[str, Any]] = None,
+        wait_for_completion: Callable[[], None] | None = None,
+        metadata: dict[str, Any] | None = None,
     ):
         """Initialize a submission result.
 
@@ -29,7 +29,7 @@ class SubmissionResult:
         self.wait_for_completion = wait_for_completion
         self.metadata = metadata or {}
 
-    def wait(self, timeout: Optional[int] = None) -> None:
+    def wait(self, timeout: int | None = None) -> None:
         """Wait for the pipeline run to complete.
 
         Args:

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/stacks/bridge.py

@@ -6,7 +6,7 @@ frameworks (ZenML, Airflow, Prefect, etc.) using rule-based adaptation.
 
 import inspect
 from dataclasses import dataclass, field
-from typing import Any, Optional
+from typing import Any
 import logging
 
 from flowyml.stacks.components import (
@@ -25,8 +25,8 @@ class AdaptationRule:
     """Rule for adapting an external component."""
 
     # Matching criteria
-    source_type: Optional[str] = None  # e.g., "zenml.orchestrators.base.BaseOrchestrator"
-    name_pattern: Optional[str] = None  # e.g., ".*Orchestrator"
+    source_type: str | None = None  # e.g., "zenml.orchestrators.base.BaseOrchestrator"
+    name_pattern: str | None = None  # e.g., ".*Orchestrator"
     has_methods: list[str] = field(default_factory=list)  # e.g., ["run", "prepare_pipeline"]
 
     # Adaptation logic
@@ -49,7 +49,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        config: Optional[dict[str, Any]] = None,
+        config: dict[str, Any] | None = None,
     ) -> type[StackComponent]:
         """Dynamically create a wrapper class based on rules.
 
@@ -75,7 +75,7 @@ class GenericBridge:
         else:
             return self._create_generic_wrapper(external_class, name, component_type, rule)
 
-    def _find_matching_rule(self, external_class: Any) -> Optional[AdaptationRule]:
+    def _find_matching_rule(self, external_class: Any) -> AdaptationRule | None:
         """Find the first rule that matches the external class."""
         for rule in self.rules:
             # Check source type
@@ -120,7 +120,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[Orchestrator]:
         """Create a wrapper for an Orchestrator."""
 
@@ -174,7 +174,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[ArtifactStore]:
         """Create a wrapper for an Artifact Store."""
 
@@ -216,7 +216,7 @@ class GenericBridge:
         self,
         external_class: Any,
         name: str,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[ContainerRegistry]:
         """Create a wrapper for a Container Registry."""
 
@@ -259,7 +259,7 @@ class GenericBridge:
         external_class: Any,
         name: str,
         comp_type: ComponentType,
-        rule: Optional[AdaptationRule],
+        rule: AdaptationRule | None,
     ) -> type[StackComponent]:
         """Create a generic wrapper."""
 

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/stacks/plugins.py

@@ -14,7 +14,7 @@ import inspect
 import subprocess
 import sys
 from dataclasses import dataclass, field
-from typing import Any, Optional, Protocol, runtime_checkable
+from typing import Any, Protocol, runtime_checkable
 
 from flowyml.stacks.components import (
     StackComponent,
@@ -48,7 +48,7 @@ class PluginBridge(Protocol):
         self,
         component_class: Any,
         name: str,
-        config: Optional[dict[str, Any]] = None,
+        config: dict[str, Any] | None = None,
     ) -> type[StackComponent]:
         """Wrap an external component class into a flowyml component."""
         ...

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/stacks/registry.py

@@ -214,6 +214,27 @@ def get_active_stack() -> Stack | None:
     Returns:
         Active stack or None
     """
+    # Check for remote execution mode in global config
+    from flowyml.utils.config import get_config
+
+    config = get_config()
+
+    if config.execution_mode == "remote" and config.remote_server_url:
+        # Dynamically create a remote logging stack
+        from flowyml.core.executor import LocalExecutor
+        from flowyml.storage.remote import RemoteMetadataStore, RemoteArtifactStore
+        from flowyml.stacks.base import Stack
+        import os
+
+        api_token = os.getenv("FLOWYML_API_TOKEN")
+
+        return Stack(
+            name="remote_logging",
+            executor=LocalExecutor(),
+            metadata_store=RemoteMetadataStore(api_url=config.remote_server_url, api_token=api_token),
+            artifact_store=RemoteArtifactStore(api_url=config.remote_server_url, api_token=api_token),
+        )
+
     return get_registry().get_active_stack()
 
 

{flowyml-1.3.0 → flowyml-1.5.0}/flowyml/storage/materializers/base.py

@@ -99,6 +99,27 @@ class MaterializerRegistry:
 
         return None
 
+    def get_materializer_by_type_name(self, type_name: str) -> BaseMaterializer | None:
+        """Get materializer by type name string.
+
+        Args:
+            type_name: Full type name (e.g. 'pandas.core.frame.DataFrame')
+
+        Returns:
+            Materializer instance or None
+        """
+        # Check if we have a direct mapping
+        if type_name in self._materializers:
+            return self._materializers[type_name]()
+
+        # Try to find by just class name if full path fails (less reliable but useful fallback)
+        simple_name = type_name.split(".")[-1]
+        for key, materializer_cls in self._materializers.items():
+            if key.split(".")[-1] == simple_name:
+                return materializer_cls()
+
+        return None
+
     def list_materializers(self) -> dict[str, type[BaseMaterializer]]:
         """List all registered materializers.
 
@@ -131,3 +152,15 @@ def get_materializer(obj: Any) -> BaseMaterializer | None:
         Materializer instance or None
     """
     return materializer_registry.get_materializer(obj)
+
+
+def get_materializer_by_type_name(type_name: str) -> BaseMaterializer | None:
+    """Get materializer by type name string from global registry.
+
+    Args:
+        type_name: Full type name (e.g. 'pandas.core.frame.DataFrame')
+
+    Returns:
+        Materializer instance or None
+    """
+    return materializer_registry.get_materializer_by_type_name(type_name)

flowyml-1.5.0/flowyml/storage/metadata.py

@@ -0,0 +1,53 @@
+"""Metadata storage backends for flowyml."""
+
+from abc import ABC, abstractmethod
+
+
+class MetadataStore(ABC):
+    """Base class for metadata storage backends."""
+
+    @abstractmethod
+    def save_run(self, run_id: str, metadata: dict) -> None:
+        """Save run metadata."""
+        pass
+
+    @abstractmethod
+    def load_run(self, run_id: str) -> dict | None:
+        """Load run metadata."""
+        pass
+
+    @abstractmethod
+    def list_runs(self, limit: int | None = None) -> list[dict]:
+        """List all runs."""
+        pass
+
+    @abstractmethod
+    def list_pipelines(self) -> list[str]:
+        """List all unique pipeline names."""
+        pass
+
+    @abstractmethod
+    def save_artifact(self, artifact_id: str, metadata: dict) -> None:
+        """Save artifact metadata."""
+        pass
+
+    @abstractmethod
+    def load_artifact(self, artifact_id: str) -> dict | None:
+        """Load artifact metadata."""
+        pass
+
+    @abstractmethod
+    def list_assets(self, limit: int | None = None, **filters) -> list[dict]:
+        """List assets with optional filters."""
+        pass
+
+    @abstractmethod
+    def query(self, **filters) -> list[dict]:
+        """Query runs with filters."""
+        pass
+
+
+from flowyml.storage.sql import SQLMetadataStore  # noqa: E402
+
+# Alias for backward compatibility
+SQLiteMetadataStore = SQLMetadataStore