flowyml 1.2.0__py3-none-any.whl → 1.4.0__py3-none-any.whl

flowyml/__init__.py

@@ -38,6 +38,7 @@ from flowyml.assets.registry import AssetRegistry
 # Stack imports
 from flowyml.stacks.base import Stack
 from flowyml.stacks.local import LocalStack
+from flowyml.stacks.components import ResourceConfig, DockerConfig
 
 # Tracking imports
 from flowyml.tracking.experiment import Experiment
@@ -140,6 +141,8 @@ __all__ = [
     # Stacks
     "Stack",
     "LocalStack",
+    "ResourceConfig",
+    "DockerConfig",
     # Tracking
     "Experiment",
     "Run",

flowyml/assets/base.py

@@ -77,6 +77,16 @@ class Asset:
         if parent:
             parent.children.append(self)
 
+    @property
+    def properties(self) -> dict[str, Any]:
+        """Expose mutable properties stored in metadata."""
+        return self.metadata.properties
+
+    @property
+    def tags(self) -> dict[str, str]:
+        """Expose mutable tags stored in metadata."""
+        return self.metadata.tags
+
     @classmethod
     def create(
         cls,

flowyml/assets/metrics.py

@@ -89,7 +89,10 @@ class Metrics(Asset):
     def create(
         cls,
         name: str | None = None,
+        version: str | None = None,
         parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
         **metrics,
     ) -> "Metrics":
         """Factory method to create metrics.
@@ -99,6 +102,9 @@ class Metrics(Asset):
         """
         return cls(
             name=name or "metrics",
+            version=version,
             data=metrics,
             parent=parent,
+            tags=tags,
+            properties=properties,
         )

flowyml/cli/main.py

@@ -48,9 +48,11 @@ def init(name: str, template: str, directory: str) -> None:
 @click.option("--stack", default="local", help="Stack to use for execution")
 @click.option("--context", "-c", multiple=True, help="Context parameters (key=value)")
 @click.option("--debug", is_flag=True, help="Enable debug mode")
-def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
+@click.option("--retry", type=int, help="Number of retries for the pipeline")
+def run(pipeline_name: str, stack: str, context: tuple, debug: bool, retry: int | None) -> None:
     """Run a pipeline."""
     from flowyml.cli.run import run_pipeline
+    from flowyml.core.retry_policy import OrchestratorRetryPolicy
 
     # Parse context parameters
     ctx_params = {}
@@ -60,8 +62,13 @@ def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
 
     click.echo(f"Running pipeline '{pipeline_name}' on stack '{stack}'...")
 
+    kwargs = {}
+    if retry:
+        kwargs["retry_policy"] = OrchestratorRetryPolicy(max_attempts=retry)
+        click.echo(f"  Retry policy enabled: max_attempts={retry}")
+
     try:
-        result = run_pipeline(pipeline_name, stack, ctx_params, debug)
+        result = run_pipeline(pipeline_name, stack, ctx_params, debug, **kwargs)
         click.echo("✓ Pipeline completed successfully")
         click.echo(f"  Run ID: {result.get('run_id', 'N/A')}")
         click.echo(f"  Duration: {result.get('duration', 'N/A')}")
@@ -70,6 +77,105 @@ def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
         raise click.Abort()
 
 
+@cli.group()
+def schedule() -> None:
+    """Schedule management commands."""
+    pass
+
+
+@schedule.command("create")
+@click.argument("pipeline_name")
+@click.argument("schedule_type", type=click.Choice(["cron", "interval", "daily", "hourly"]))
+@click.argument("value")
+@click.option("--stack", default="local", help="Stack to use for execution")
+def create_schedule(pipeline_name: str, schedule_type: str, value: str, stack: str) -> None:
+    """Create a new schedule for a pipeline.
+
+    VALUE format depends on SCHEDULE_TYPE:
+    - cron: "*/5 * * * *"
+    - interval: seconds (e.g. 60)
+    - daily: "HH:MM" (e.g. 14:30)
+    - hourly: minute (e.g. 30)
+    """
+    from flowyml.core.scheduler import PipelineScheduler
+    from flowyml.cli.run import run_pipeline
+
+    # We need a callable for the scheduler.
+    # Since CLI is stateless, we wrap the run_pipeline command.
+    # Note: In a real distributed system, this would submit to a scheduler service.
+    # Here we are just registering it in the local scheduler DB.
+
+    # For now, we'll just use the scheduler API to register the definition
+    scheduler = PipelineScheduler()
+
+    # Define a wrapper that runs the pipeline via CLI logic
+    def job_func():
+        run_pipeline(pipeline_name, stack, {}, False)
+
+    try:
+        if schedule_type == "cron":
+            scheduler.schedule_cron(pipeline_name, job_func, value)
+        elif schedule_type == "interval":
+            scheduler.schedule_interval(pipeline_name, job_func, seconds=int(value))
+        elif schedule_type == "daily":
+            if ":" in value:
+                h, m = map(int, value.split(":"))
+                scheduler.schedule_daily(pipeline_name, job_func, hour=h, minute=m)
+            else:
+                raise ValueError("Daily value must be HH:MM")
+        elif schedule_type == "hourly":
+            scheduler.schedule_hourly(pipeline_name, job_func, minute=int(value))
+
+        click.echo(f"✓ Schedule created for '{pipeline_name}' ({schedule_type}={value})")
+        click.echo("  Note: Ensure the scheduler service is running to execute this schedule.")
+    except Exception as e:
+        click.echo(f"✗ Error creating schedule: {e}", err=True)
+
+
+@schedule.command("list")
+def list_schedules() -> None:
+    """List all active schedules."""
+    from flowyml.core.scheduler import PipelineScheduler
+
+    scheduler = PipelineScheduler()
+    jobs = scheduler.get_jobs()
+
+    if not jobs:
+        click.echo("No active schedules found.")
+        return
+
+    click.echo(f"Found {len(jobs)} schedules:\n")
+    for job in jobs:
+        click.echo(f"  {job.id} - {job.name}")
+        click.echo(f"    Next run: {job.next_run_time}")
+        click.echo()
+
+
+@schedule.command("start")
+def start_scheduler() -> None:
+    """Start the scheduler service (blocking)."""
+    from flowyml.core.scheduler import PipelineScheduler
+    import time
+
+    click.echo("🚀 Starting Scheduler Service...")
+    scheduler = PipelineScheduler()
+
+    try:
+        # In a real app, this would load definitions from DB and register them
+        # For now, it just runs the scheduler loop for existing in-memory jobs
+        # (which might be empty if we restarted).
+        # To make this persistent, we'd need to serialize job definitions to DB.
+        # The current Scheduler implementation supports SQLite persistence for job state,
+        # but we need to re-register jobs on startup.
+
+        click.echo("  Scheduler running. Press Ctrl+C to stop.")
+        while True:
+            scheduler.run_pending()
+            time.sleep(1)
+    except KeyboardInterrupt:
+        click.echo("\n🛑 Scheduler stopped.")
+
+
 @cli.group()
 def ui() -> None:
     """UI server commands."""

flowyml/cli/run.py

@@ -6,7 +6,13 @@ from pathlib import Path
 from typing import Any
 
 
-def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any], debug: bool) -> dict[str, Any]:
+def run_pipeline(
+    pipeline_name: str,
+    stack: str,
+    context_params: dict[str, Any],
+    debug: bool,
+    **kwargs,
+) -> dict[str, Any]:
     """Run a pipeline by name.
 
     Args:
@@ -14,6 +20,7 @@ def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any],
         stack: Stack to use for execution
         context_params: Context parameters to override
         debug: Enable debug mode
+        **kwargs: Additional arguments passed to pipeline.run
 
     Returns:
         Dictionary with run results
@@ -65,7 +72,7 @@ def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any],
         pipeline.set_stack(stack)
 
     # Run pipeline
-    result = pipeline.run(debug=debug)
+    result = pipeline.run(debug=debug, **kwargs)
 
     return {
         "run_id": result.run_id,

flowyml/core/execution_status.py

@@ -0,0 +1,52 @@
+"""Execution status tracking for pipeline runs."""
+
+from enum import Enum
+
+
+class ExecutionStatus(str, Enum):
+    """Status of a pipeline or step execution."""
+
+    # Pre-execution states
+    INITIALIZING = "initializing"
+    PROVISIONING = "provisioning"
+
+    # Active execution states
+    RUNNING = "running"
+
+    # Terminal success states
+    COMPLETED = "completed"
+    CACHED = "cached"
+
+    # Terminal failure states
+    FAILED = "failed"
+    STOPPED = "stopped"
+    CANCELLED = "cancelled"
+
+    # Intermediate states
+    STOPPING = "stopping"
+    CANCELLING = "cancelling"
+
+    @property
+    def is_finished(self) -> bool:
+        """Check if execution is in a terminal state."""
+        return self in {
+            ExecutionStatus.COMPLETED,
+            ExecutionStatus.CACHED,
+            ExecutionStatus.FAILED,
+            ExecutionStatus.STOPPED,
+            ExecutionStatus.CANCELLED,
+        }
+
+    @property
+    def is_successful(self) -> bool:
+        """Check if execution completed successfully."""
+        return self in {ExecutionStatus.COMPLETED, ExecutionStatus.CACHED}
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if execution failed."""
+        return self in {
+            ExecutionStatus.FAILED,
+            ExecutionStatus.STOPPED,
+            ExecutionStatus.CANCELLED,
+        }

flowyml/core/hooks.py

@@ -0,0 +1,106 @@
+"""Lifecycle hooks for pipelines and steps."""
+
+from typing import Any, TYPE_CHECKING
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+if TYPE_CHECKING:
+    from flowyml.core.pipeline import Pipeline, PipelineResult
+    from flowyml.core.step import Step
+    from flowyml.core.executor import ExecutionResult
+
+
+@dataclass
+class HookRegistry:
+    """Registry for pipeline and step lifecycle hooks."""
+
+    # Pipeline-level hooks
+    on_pipeline_start: list[Callable[["Pipeline"], None]] = field(default_factory=list)
+    on_pipeline_end: list[Callable[["Pipeline", "PipelineResult"], None]] = field(default_factory=list)
+
+    # Step-level hooks
+    on_step_start: list[Callable[["Step", dict[str, Any]], None]] = field(default_factory=list)
+    on_step_end: list[Callable[["Step", "ExecutionResult"], None]] = field(default_factory=list)
+
+    def register_pipeline_start_hook(self, hook: Callable[["Pipeline"], None]) -> None:
+        """Register a hook to run at pipeline start."""
+        self.on_pipeline_start.append(hook)
+
+    def register_pipeline_end_hook(self, hook: Callable[["Pipeline", "PipelineResult"], None]) -> None:
+        """Register a hook to run at pipeline end."""
+        self.on_pipeline_end.append(hook)
+
+    def register_step_start_hook(self, hook: Callable[["Step", dict[str, Any]], None]) -> None:
+        """Register a hook to run before step execution."""
+        self.on_step_start.append(hook)
+
+    def register_step_end_hook(self, hook: Callable[["Step", "ExecutionResult"], None]) -> None:
+        """Register a hook to run after step execution."""
+        self.on_step_end.append(hook)
+
+    def run_pipeline_start_hooks(self, pipeline: "Pipeline") -> None:
+        """Execute all pipeline start hooks."""
+        for hook in self.on_pipeline_start:
+            try:
+                hook(pipeline)
+            except Exception as e:
+                print(f"Warning: Pipeline start hook failed: {e}")
+
+    def run_pipeline_end_hooks(self, pipeline: "Pipeline", result: "PipelineResult") -> None:
+        """Execute all pipeline end hooks."""
+        for hook in self.on_pipeline_end:
+            try:
+                hook(pipeline, result)
+            except Exception as e:
+                print(f"Warning: Pipeline end hook failed: {e}")
+
+    def run_step_start_hooks(self, step: "Step", inputs: dict[str, Any]) -> None:
+        """Execute all step start hooks."""
+        for hook in self.on_step_start:
+            try:
+                hook(step, inputs)
+            except Exception as e:
+                print(f"Warning: Step start hook failed: {e}")
+
+    def run_step_end_hooks(self, step: "Step", result: "ExecutionResult") -> None:
+        """Execute all step end hooks."""
+        for hook in self.on_step_end:
+            try:
+                hook(step, result)
+            except Exception as e:
+                print(f"Warning: Step end hook failed: {e}")
+
+
+# Global hook registry
+_global_hooks = HookRegistry()
+
+
+def get_global_hooks() -> HookRegistry:
+    """Get the global hook registry."""
+    return _global_hooks
+
+
+def on_pipeline_start(func: Callable[["Pipeline"], None]) -> Callable[["Pipeline"], None]:
+    """Decorator to register a pipeline start hook."""
+    _global_hooks.register_pipeline_start_hook(func)
+    return func
+
+
+def on_pipeline_end(
+    func: Callable[["Pipeline", "PipelineResult"], None],
+) -> Callable[["Pipeline", "PipelineResult"], None]:
+    """Decorator to register a pipeline end hook."""
+    _global_hooks.register_pipeline_end_hook(func)
+    return func
+
+
+def on_step_start(func: Callable[["Step", dict[str, Any]], None]) -> Callable[["Step", dict[str, Any]], None]:
+    """Decorator to register a step start hook."""
+    _global_hooks.register_step_start_hook(func)
+    return func
+
+
+def on_step_end(func: Callable[["Step", "ExecutionResult"], None]) -> Callable[["Step", "ExecutionResult"], None]:
+    """Decorator to register a step end hook."""
+    _global_hooks.register_step_end_hook(func)
+    return func

flowyml/core/observability.py

@@ -0,0 +1,210 @@
+"""Observability hooks for monitoring and metrics collection."""
+
+from typing import Protocol, Any, TYPE_CHECKING
+from datetime import datetime
+from dataclasses import dataclass, field
+
+if TYPE_CHECKING:
+    from flowyml.core.pipeline import Pipeline, PipelineResult
+    from flowyml.core.step import Step
+    from flowyml.core.executor import ExecutionResult
+
+
+@dataclass
+class MetricEvent:
+    """Base metric event."""
+
+    timestamp: datetime = field(default_factory=datetime.now)
+    tags: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "timestamp": self.timestamp.isoformat(),
+            "tags": self.tags,
+        }
+
+
+@dataclass
+class PipelineMetricEvent(MetricEvent):
+    """Pipeline-level metric event."""
+
+    pipeline_name: str = ""
+    run_id: str = ""
+    duration_seconds: float | None = None
+    success: bool | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        base = super().to_dict()
+        base.update(
+            {
+                "pipeline_name": self.pipeline_name,
+                "run_id": self.run_id,
+                "duration_seconds": self.duration_seconds,
+                "success": self.success,
+            },
+        )
+        return base
+
+
+@dataclass
+class StepMetricEvent(MetricEvent):
+    """Step-level metric event."""
+
+    step_name: str = ""
+    pipeline_name: str = ""
+    run_id: str = ""
+    duration_seconds: float | None = None
+    success: bool | None = None
+    cached: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        base = super().to_dict()
+        base.update(
+            {
+                "step_name": self.step_name,
+                "pipeline_name": self.pipeline_name,
+                "run_id": self.run_id,
+                "duration_seconds": self.duration_seconds,
+                "success": self.success,
+                "cached": self.cached,
+            },
+        )
+        return base
+
+
+class MetricsCollector(Protocol):
+    """Protocol for metrics collectors."""
+
+    def record_pipeline_start(self, pipeline: "Pipeline", run_id: str) -> None:
+        """Record pipeline start."""
+        ...
+
+    def record_pipeline_end(self, pipeline: "Pipeline", result: "PipelineResult") -> None:
+        """Record pipeline completion."""
+        ...
+
+    def record_step_start(self, step: "Step", pipeline_name: str, run_id: str) -> None:
+        """Record step start."""
+        ...
+
+    def record_step_end(self, step: "Step", result: "ExecutionResult", pipeline_name: str, run_id: str) -> None:
+        """Record step completion."""
+        ...
+
+
+class ConsoleMetricsCollector:
+    """Simple console metrics collector for debugging."""
+
+    def record_pipeline_start(self, pipeline: "Pipeline", run_id: str) -> None:
+        event = PipelineMetricEvent(
+            pipeline_name=pipeline.name,
+            run_id=run_id,
+        )
+        print(f"📊 Pipeline Started: {event.to_dict()}")
+
+    def record_pipeline_end(self, pipeline: "Pipeline", result: "PipelineResult") -> None:
+        event = PipelineMetricEvent(
+            pipeline_name=pipeline.name,
+            run_id=result.run_id,
+            duration_seconds=result.duration_seconds,
+            success=result.success,
+        )
+        print(f"📊 Pipeline Ended: {event.to_dict()}")
+
+    def record_step_start(self, step: "Step", pipeline_name: str, run_id: str) -> None:
+        event = StepMetricEvent(
+            step_name=step.name,
+            pipeline_name=pipeline_name,
+            run_id=run_id,
+        )
+        print(f"📊 Step Started: {event.to_dict()}")
+
+    def record_step_end(self, step: "Step", result: "ExecutionResult", pipeline_name: str, run_id: str) -> None:
+        event = StepMetricEvent(
+            step_name=step.name,
+            pipeline_name=pipeline_name,
+            run_id=run_id,
+            duration_seconds=getattr(result, "duration_seconds", None),
+            success=result.success,
+            cached=getattr(result, "cached", False),
+        )
+        print(f"📊 Step Ended: {event.to_dict()}")
+
+
+class PrometheusMetricsCollector:
+    """Prometheus metrics collector (requires prometheus_client)."""
+
+    def __init__(self):
+        try:
+            from prometheus_client import Counter, Histogram
+
+            self.pipeline_starts = Counter(
+                "flowyml_pipeline_starts_total",
+                "Total pipeline starts",
+                ["pipeline_name"],
+            )
+            self.pipeline_completions = Counter(
+                "flowyml_pipeline_completions_total",
+                "Total pipeline completions",
+                ["pipeline_name", "status"],
+            )
+            self.pipeline_duration = Histogram(
+                "flowyml_pipeline_duration_seconds",
+                "Pipeline duration in seconds",
+                ["pipeline_name"],
+            )
+            self.step_duration = Histogram(
+                "flowyml_step_duration_seconds",
+                "Step duration in seconds",
+                ["pipeline_name", "step_name"],
+            )
+            self.step_cache_hits = Counter(
+                "flowyml_step_cache_hits_total",
+                "Total step cache hits",
+                ["pipeline_name", "step_name"],
+            )
+        except ImportError:
+            raise ImportError("prometheus_client required for PrometheusMetricsCollector")
+
+    def record_pipeline_start(self, pipeline: "Pipeline", run_id: str) -> None:
+        self.pipeline_starts.labels(pipeline_name=pipeline.name).inc()
+
+    def record_pipeline_end(self, pipeline: "Pipeline", result: "PipelineResult") -> None:
+        status = "success" if result.success else "failure"
+        self.pipeline_completions.labels(pipeline_name=pipeline.name, status=status).inc()
+
+        if result.duration_seconds:
+            self.pipeline_duration.labels(pipeline_name=pipeline.name).observe(result.duration_seconds)
+
+    def record_step_start(self, step: "Step", pipeline_name: str, run_id: str) -> None:
+        pass  # No-op for Prometheus (only track completion)
+
+    def record_step_end(self, step: "Step", result: "ExecutionResult", pipeline_name: str, run_id: str) -> None:
+        duration = getattr(result, "duration_seconds", None)
+        if duration:
+            self.step_duration.labels(
+                pipeline_name=pipeline_name,
+                step_name=step.name,
+            ).observe(duration)
+
+        if getattr(result, "cached", False):
+            self.step_cache_hits.labels(
+                pipeline_name=pipeline_name,
+                step_name=step.name,
+            ).inc()
+
+
+# Global metrics collector
+_metrics_collector: MetricsCollector | None = None
+
+
+def set_metrics_collector(collector: MetricsCollector) -> None:
+    """Set global metrics collector."""
+    global _metrics_collector
+    _metrics_collector = collector
+
+
+def get_metrics_collector() -> MetricsCollector | None:
+    """Get global metrics collector."""
+    return _metrics_collector