flowyml 1.2.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

flowyml/core/pipeline.py

@@ -19,11 +19,15 @@ class PipelineResult:
         self.run_id = run_id
         self.pipeline_name = pipeline_name
         self.success = False
+        self.state = "pending"
         self.step_results: dict[str, ExecutionResult] = {}
         self.outputs: dict[str, Any] = {}
         self.start_time = datetime.now()
         self.end_time: datetime | None = None
         self.duration_seconds: float = 0.0
+        self.resource_config: Any | None = None
+        self.docker_config: Any | None = None
+        self.remote_job_id: str | None = None
 
     def add_step_result(self, result: ExecutionResult) -> None:
         """Add result from a step execution."""
@@ -37,9 +41,21 @@ class PipelineResult:
     def finalize(self, success: bool) -> None:
         """Mark pipeline as complete."""
         self.success = success
+        self.state = "completed" if success else "failed"
         self.end_time = datetime.now()
         self.duration_seconds = (self.end_time - self.start_time).total_seconds()
 
+    def attach_configs(self, resource_config: Any | None, docker_config: Any | None) -> None:
+        """Store execution configs for downstream inspection."""
+        self.resource_config = resource_config
+        self.docker_config = docker_config
+
+    def mark_submitted(self, job_id: str) -> None:
+        """Mark result as remotely submitted."""
+        self.success = True
+        self.state = "submitted"
+        self.remote_job_id = job_id
+
     def __getitem__(self, key: str) -> Any:
         """Allow dict-style access to outputs."""
         return self.outputs.get(key)
@@ -50,9 +66,17 @@ class PipelineResult:
             "run_id": self.run_id,
             "pipeline_name": self.pipeline_name,
             "success": self.success,
+            "state": self.state,
             "start_time": self.start_time.isoformat(),
             "end_time": self.end_time.isoformat() if self.end_time else None,
             "duration_seconds": self.duration_seconds,
+            "resource_config": self.resource_config.to_dict()
+            if hasattr(self.resource_config, "to_dict")
+            else self.resource_config,
+            "docker_config": self.docker_config.to_dict()
+            if hasattr(self.docker_config, "to_dict")
+            else self.docker_config,
+            "remote_job_id": self.remote_job_id,
             "steps": {
                 name: {
                     "success": result.success,
@@ -67,10 +91,19 @@ class PipelineResult:
 
     def summary(self) -> str:
         """Generate execution summary."""
+        if self.state == "submitted":
+            status_line = f"Status: ⏳ SUBMITTED (job: {self.remote_job_id})"
+        elif self.success:
+            status_line = "Status: ✓ SUCCESS"
+        elif self.state == "failed":
+            status_line = "Status: ✗ FAILED"
+        else:
+            status_line = f"Status: {self.state.upper()}"
+
         lines = [
             f"Pipeline: {self.pipeline_name}",
             f"Run ID: {self.run_id}",
-            f"Status: {'✓ SUCCESS' if self.success else '✗ FAILED'}",
+            status_line,
             f"Duration: {self.duration_seconds:.2f}s",
             "",
             "Steps:",
@@ -127,7 +160,9 @@ class Pipeline:
         self.name = name
         self.context = context or Context()
         self.enable_cache = enable_cache
-        self.stack = stack  # Store stack instance
+        self.stack = None  # Will be assigned via _apply_stack
+        self._stack_locked = stack is not None
+        self._provided_executor = executor
 
         self.steps: list[Step] = []
         self.dag = DAG()
@@ -146,15 +181,14 @@ class Pipeline:
         self.runs_dir.mkdir(parents=True, exist_ok=True)
 
         # Initialize components from stack or defaults
-        if self.stack:
-            self.executor = executor or self.stack.executor
-            self.metadata_store = self.stack.metadata_store
-        else:
-            self.executor = executor or LocalExecutor()
-            # Metadata store for UI integration
-            from flowyml.storage.metadata import SQLiteMetadataStore
+        self.executor = executor or LocalExecutor()
+        # Metadata store for UI integration
+        from flowyml.storage.metadata import SQLiteMetadataStore
 
-            self.metadata_store = SQLiteMetadataStore()
+        self.metadata_store = SQLiteMetadataStore()
+
+        if stack:
+            self._apply_stack(stack, locked=True)
 
         # Handle Project Attachment
         if project:
@@ -179,6 +213,18 @@ class Pipeline:
         self._built = False
         self.step_groups: list[Any] = []  # Will hold StepGroup objects
 
+    def _apply_stack(self, stack: Any | None, locked: bool) -> None:
+        """Attach a stack and update executors/metadata."""
+        if not stack:
+            return
+        self.stack = stack
+        self._stack_locked = locked
+        if self._provided_executor:
+            self.executor = self._provided_executor
+        else:
+            self.executor = stack.executor
+        self.metadata_store = stack.metadata_store
+
     def add_step(self, step: Step) -> "Pipeline":
         """Add a step to the pipeline.
 
@@ -234,6 +280,7 @@ class Pipeline:
         resources: Any | None = None,  # ResourceConfig
         docker_config: Any | None = None,  # DockerConfig
         context: dict[str, Any] | None = None,  # Context vars override
+        **kwargs,
     ) -> PipelineResult:
         """Execute the pipeline.
 
@@ -244,25 +291,34 @@ class Pipeline:
             resources: Resource configuration for execution
             docker_config: Docker configuration for containerized execution
             context: Context variables override
+            **kwargs: Additional arguments passed to the orchestrator
 
         Returns:
             PipelineResult with outputs and execution info
         """
         import uuid
+        from flowyml.core.orchestrator import LocalOrchestrator
 
         run_id = str(uuid.uuid4())
 
-        # Use provided stack or instance stack
+        # Determine stack for this run
         if stack is not None:
-            self.stack = stack
-            # Update components from new stack
-            self.executor = self.stack.executor
-            self.metadata_store = self.stack.metadata_store
-
-        # Determine artifact store
-        artifact_store = None
-        if self.stack:
-            artifact_store = self.stack.artifact_store
+            self._apply_stack(stack, locked=True)
+        elif not self._stack_locked:
+            active_stack = None
+            try:
+                from flowyml.stacks.registry import get_active_stack
+            except ImportError:
+                get_active_stack = None
+            if get_active_stack:
+                active_stack = get_active_stack()
+            if active_stack:
+                self._apply_stack(active_stack, locked=False)
+
+        # Determine orchestrator
+        orchestrator = getattr(self.stack, "orchestrator", None) if self.stack else None
+        if orchestrator is None:
+            orchestrator = LocalOrchestrator()
 
         # Update context with provided values
         if context:
@@ -272,218 +328,30 @@ class Pipeline:
         if not self._built:
             self.build()
 
-        # Initialize result
-        result = PipelineResult(run_id, self.name)
-        step_outputs = inputs or {}
-
-        # Map step names to step objects for easier lookup
-        self.steps_dict = {step.name: step for step in self.steps}
-        if debug:
-            pass
-        else:
-            # Always print the run URL for better UX
-            pass
-
-        # Get execution units (individual steps or groups)
-        from flowyml.core.step_grouping import get_execution_units
-
-        execution_units = get_execution_units(self.dag, self.steps)
-
-        # Execute steps/groups in order
-        for unit in execution_units:
-            # Check if unit is a group or individual step
-            from flowyml.core.step_grouping import StepGroup
-
-            if isinstance(unit, StepGroup):
-                # Execute entire group
-                if debug:
-                    pass
-
-                # Get context parameters (use first step's function as representative)
-                first_step = unit.steps[0]
-                context_params = self.context.inject_params(first_step.func)
-
-                # Execute the group
-                group_results = self.executor.execute_step_group(
-                    step_group=unit,
-                    inputs=step_outputs,
-                    context_params=context_params,
-                    cache_store=self.cache_store,
-                    artifact_store=artifact_store,
-                    run_id=run_id,
-                    project_name=self.name,
-                )
-
-                # Process each step result
-                for step_result in group_results:
-                    result.add_step_result(step_result)
-
-                    if debug:
-                        pass
-
-                    # Handle failure
-                    if not step_result.success and not step_result.skipped:
-                        result.finalize(success=False)
-                        self._save_run(result)
-                        return result
-
-                    # Store outputs for next steps/groups
-                    if step_result.output is not None:
-                        # Find step definition to get output names
-                        step_def = next((s for s in self.steps if s.name == step_result.step_name), None)
-                        if step_def:
-                            if len(step_def.outputs) == 1:
-                                step_outputs[step_def.outputs[0]] = step_result.output
-                                result.outputs[step_def.outputs[0]] = step_result.output
-                            elif isinstance(step_result.output, (list, tuple)) and len(step_result.output) == len(
-                                step_def.outputs,
-                            ):
-                                for name, val in zip(step_def.outputs, step_result.output, strict=False):
-                                    step_outputs[name] = val
-                                    result.outputs[name] = val
-                            elif isinstance(step_result.output, dict):
-                                for name in step_def.outputs:
-                                    if name in step_result.output:
-                                        step_outputs[name] = step_result.output[name]
-                                        result.outputs[name] = step_result.output[name]
-                            else:
-                                if step_def.outputs:
-                                    step_outputs[step_def.outputs[0]] = step_result.output
-                                    result.outputs[step_def.outputs[0]] = step_result.output
-
-            else:
-                # Execute single ungrouped step
-                step = unit
-
-                if debug:
-                    pass
-
-                # Prepare step inputs
-                step_inputs = {}
-
-                # Get function signature to map inputs to parameters
-                import inspect
-
-                sig = inspect.signature(step.func)
-                params = list(sig.parameters.values())
-
-                # Filter out self/cls
-                params = [p for p in params if p.name not in ("self", "cls")]
-
-                # Strategy:
-                # 1. Map inputs to parameters
-                #    - If input name matches param name, use it
-                #    - If not, use positional mapping (input[i] -> param[i])
-
-                # Track which parameters have been assigned
-                assigned_params = set()
-
-                if step.inputs:
-                    for i, input_name in enumerate(step.inputs):
-                        if input_name not in step_outputs:
-                            continue
-
-                        val = step_outputs[input_name]
-
-                        # Check if input name matches a parameter
-                        param_match = next((p for p in params if p.name == input_name), None)
-
-                        if param_match:
-                            step_inputs[param_match.name] = val
-                            assigned_params.add(param_match.name)
-                        elif i < len(params):
-                            # Positional fallback
-                            # Only if this parameter hasn't been assigned yet
-                            target_param = params[i]
-                            if target_param.name not in assigned_params:
-                                step_inputs[target_param.name] = val
-                                assigned_params.add(target_param.name)
-
-                # Auto-map parameters from available outputs if they match function signature
-                # This allows passing inputs to run() without declaring them as asset dependencies
-                for param in params:
-                    if param.name in step_outputs and param.name not in step_inputs:
-                        step_inputs[param.name] = step_outputs[param.name]
-                        assigned_params.add(param.name)
-
-                # Validate context parameters
-                # Exclude parameters that are already provided in step_inputs
-                exclude_params = list(step.inputs) + list(step_inputs.keys())
-                missing_params = self.context.validate_for_step(step.func, exclude=exclude_params)
-                if missing_params:
-                    if debug:
-                        pass
-
-                    error_msg = f"Missing required parameters: {missing_params}"
-                    step_result = ExecutionResult(
-                        step_name=step.name,
-                        success=False,
-                        error=error_msg,
-                    )
-                    result.add_step_result(step_result)
-                    result.finalize(success=False)
-                    self._save_run(result)  # Save run before returning
-                    self._save_pipeline_definition()  # Save definition even on failure
-                    print("DEBUG: Pipeline failed at step execution")
-                    return result
-
-                # Get context parameters for this step
-                context_params = self.context.inject_params(step.func)
-
-                # Execute step
-                step_result = self.executor.execute_step(
-                    step,
-                    step_inputs,
-                    context_params,
-                    self.cache_store,
-                    artifact_store=artifact_store,
-                    run_id=run_id,
-                    project_name=self.name,
-                )
-
-                result.add_step_result(step_result)
-
-                if debug:
-                    pass
-
-                # Handle failure
-                if not step_result.success:
-                    if debug and not step_result.error:
-                        pass
-                    result.finalize(success=False)
-                    self._save_run(result)
-                    self._save_pipeline_definition()  # Save definition even on failure
-                    print("DEBUG: Pipeline failed at step execution")
-                    return result
-
-                # Store outputs for next steps
-                if step_result.output is not None:
-                    if len(step.outputs) == 1:
-                        step_outputs[step.outputs[0]] = step_result.output
-                        result.outputs[step.outputs[0]] = step_result.output
-                    elif isinstance(step_result.output, (list, tuple)) and len(step_result.output) == len(step.outputs):
-                        for name, val in zip(step.outputs, step_result.output, strict=False):
-                            step_outputs[name] = val
-                            result.outputs[name] = val
-                    elif isinstance(step_result.output, dict):
-                        for name in step.outputs:
-                            if name in step_result.output:
-                                step_outputs[name] = step_result.output[name]
-                                result.outputs[name] = step_result.output[name]
-                    else:
-                        # Fallback: assign to first output if available
-                        if step.outputs:
-                            step_outputs[step.outputs[0]] = step_result.output
-                            result.outputs[step.outputs[0]] = step_result.output
-
-        # Success!
-        result.finalize(success=True)
+        resource_config = self._coerce_resource_config(resources)
+        docker_cfg = self._coerce_docker_config(docker_config)
+
+        # Run the pipeline via orchestrator
+        result = orchestrator.run_pipeline(
+            self,
+            run_id=run_id,
+            resources=resource_config,
+            docker_config=docker_cfg,
+            inputs=inputs,
+            context=context,
+            **kwargs,
+        )
 
-        if debug:
-            pass
+        # If result is just a job ID (remote execution), wrap it in a basic result
+        if isinstance(result, str):
+            # Create a submitted result wrapper
+            wrapper = PipelineResult(run_id, self.name)
+            wrapper.attach_configs(resource_config, docker_cfg)
+            wrapper.mark_submitted(result)
+            self._save_run(wrapper)
+            self._save_pipeline_definition()
+            return wrapper
 
-        self._save_run(result)
-        self._save_pipeline_definition()  # Save pipeline structure for scheduling
         return result
 
     def to_definition(self) -> dict:
@@ -527,6 +395,36 @@ class Pipeline:
             # Don't fail the run if definition saving fails
             print(f"Warning: Failed to save pipeline definition: {e}")
 
+    def _coerce_resource_config(self, resources: Any | None):
+        """Convert resources input to ResourceConfig if necessary."""
+        if resources is None:
+            return None
+        try:
+            from flowyml.stacks.components import ResourceConfig
+        except Exception:
+            return resources
+
+        if isinstance(resources, ResourceConfig):
+            return resources
+        if isinstance(resources, dict):
+            return ResourceConfig(**resources)
+        return resources
+
+    def _coerce_docker_config(self, docker_config: Any | None):
+        """Convert docker input to DockerConfig if necessary."""
+        if docker_config is None:
+            return None
+        try:
+            from flowyml.stacks.components import DockerConfig
+        except Exception:
+            return docker_config
+
+        if isinstance(docker_config, DockerConfig):
+            return docker_config
+        if isinstance(docker_config, dict):
+            return DockerConfig(**docker_config)
+        return docker_config
+
     def _save_run(self, result: PipelineResult) -> None:
         """Save run results to disk and metadata database."""
         # Save to JSON file
@@ -576,7 +474,7 @@ class Pipeline:
         metadata = {
             "run_id": result.run_id,
             "pipeline_name": result.pipeline_name,
-            "status": "completed" if result.success else "failed",
+            "status": result.state,
             "start_time": result.start_time.isoformat(),
             "end_time": result.end_time.isoformat() if result.end_time else None,
             "duration": result.duration_seconds,
@@ -584,6 +482,13 @@ class Pipeline:
             "context": self.context._params if hasattr(self.context, "_params") else {},
             "steps": steps_metadata,
             "dag": dag_data,
+            "resources": result.resource_config.to_dict()
+            if hasattr(result.resource_config, "to_dict")
+            else result.resource_config,
+            "docker": result.docker_config.to_dict()
+            if hasattr(result.docker_config, "to_dict")
+            else result.docker_config,
+            "remote_job_id": result.remote_job_id,
         }
         self.metadata_store.save_run(result.run_id, metadata)
 
@@ -762,3 +667,60 @@ class Pipeline:
 
     def __repr__(self) -> str:
         return f"Pipeline(name='{self.name}', steps={len(self.steps)})"
+
+    def schedule(
+        self,
+        schedule_type: str,
+        value: str | int,
+        **kwargs,
+    ) -> Any:
+        """Schedule this pipeline to run automatically.
+
+        Args:
+            schedule_type: Type of schedule ('cron', 'interval', 'daily', 'hourly')
+            value: Schedule value (cron expression, seconds, 'HH:MM', or minute)
+            **kwargs: Additional arguments for scheduler
+
+        Returns:
+            Schedule object
+        """
+        from flowyml.core.scheduler import PipelineScheduler
+
+        scheduler = PipelineScheduler()
+
+        if schedule_type == "cron":
+            return scheduler.schedule_cron(self.name, self.run, str(value), **kwargs)
+        elif schedule_type == "interval":
+            return scheduler.schedule_interval(self.name, self.run, seconds=int(value), **kwargs)
+        elif schedule_type == "daily":
+            if isinstance(value, str) and ":" in value:
+                h, m = map(int, value.split(":"))
+                return scheduler.schedule_daily(self.name, self.run, hour=h, minute=m, **kwargs)
+            else:
+                raise ValueError("Daily schedule value must be 'HH:MM'")
+        elif schedule_type == "hourly":
+            return scheduler.schedule_hourly(self.name, self.run, minute=int(value), **kwargs)
+        else:
+            raise ValueError(f"Unknown schedule type: {schedule_type}")
+
+    def check_cache(self) -> dict[str, Any] | None:
+        """Check if a successful run of this pipeline already exists.
+
+        Returns:
+            Metadata of the last successful run, or None if not found.
+        """
+        # Query metadata store for successful runs of this pipeline
+        try:
+            runs = self.metadata_store.query(
+                pipeline_name=self.name,
+                status="completed",
+            )
+
+            if runs:
+                # Return the most recent one (query returns ordered by created_at DESC)
+                return runs[0]
+        except Exception as e:
+            # Don't fail if metadata store is not available or errors
+            print(f"Warning: Failed to check cache: {e}")
+
+        return None

flowyml/core/project.py

@@ -4,6 +4,7 @@ import json
 from pathlib import Path
 from typing import Any, TYPE_CHECKING
 from datetime import datetime
+from flowyml.utils.config import get_config
 
 if TYPE_CHECKING:
     from flowyml.core.pipeline import Pipeline
@@ -186,6 +187,36 @@ class Project:
         with open(output_file, "w") as f:
             json.dump(export_data, f, indent=2)
 
+    def log_model_metrics(
+        self,
+        model_name: str,
+        metrics: dict[str, float],
+        run_id: str | None = None,
+        environment: str | None = None,
+        tags: dict | None = None,
+    ) -> None:
+        """Log production metrics scoped to this project."""
+        self.metadata_store.log_model_metrics(
+            project=self.name,
+            model_name=model_name,
+            metrics=metrics,
+            run_id=run_id,
+            environment=environment,
+            tags=tags,
+        )
+
+    def list_model_metrics(
+        self,
+        model_name: str | None = None,
+        limit: int = 100,
+    ) -> list[dict]:
+        """List production metrics for this project."""
+        return self.metadata_store.list_model_metrics(
+            project=self.name,
+            model_name=model_name,
+            limit=limit,
+        )
+
     def __repr__(self) -> str:
         return f"Project(name='{self.name}', pipelines={len(self.metadata['pipelines'])})"
 
@@ -205,8 +236,9 @@ class ProjectManager:
         >>> project = manager.get_project("recommendation_system")
     """
 
-    def __init__(self, projects_dir: str = ".flowyml/projects"):
-        self.projects_dir = Path(projects_dir)
+    def __init__(self, projects_dir: str | None = None):
+        config = get_config()
+        self.projects_dir = Path(projects_dir or config.projects_dir)
         self.projects_dir.mkdir(parents=True, exist_ok=True)
 
     def create_project(self, name: str, description: str = "") -> Project: