flowyml 1.2.0__tar.gz → 1.4.0__tar.gz

{flowyml-1.2.0 → flowyml-1.4.0}/LICENSE

@@ -2,7 +2,7 @@ Apache License
 Version 2.0, January 2004
 http://www.apache.org/licenses/
 
-Copyright 2024 Flowy Team
+Copyright 2024 UnicoLab.ai
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{flowyml-1.2.0 → flowyml-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowyml
-Version: 1.2.0
+Version: 1.4.0
 Summary: Next-Generation ML Pipeline Framework
 License: Apache-2.0
 License-File: LICENSE
@@ -19,24 +19,30 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Provides-Extra: all
+Provides-Extra: aws
+Provides-Extra: azure
 Provides-Extra: gcp
 Provides-Extra: pytorch
 Provides-Extra: sklearn
 Provides-Extra: tensorflow
 Provides-Extra: ui
 Requires-Dist: click (>=8.0.0)
+Requires-Dist: cloudpickle (>=2.0.0)
 Requires-Dist: croniter (>=2.0.1,<3.0.0)
 Requires-Dist: fastapi (>=0.122.0,<0.123.0) ; extra == "ui"
 Requires-Dist: google-cloud-aiplatform (>=1.35.0) ; extra == "gcp" or extra == "all"
 Requires-Dist: google-cloud-storage (>=2.10.0) ; extra == "gcp" or extra == "all"
+Requires-Dist: httpx (>=0.24,<0.28)
 Requires-Dist: loguru (>=0.7.3,<0.8.0)
 Requires-Dist: numpy (>=1.20.0)
 Requires-Dist: pandas (>=1.3.0)
+Requires-Dist: psycopg2-binary (>=2.9.0)
 Requires-Dist: pydantic (>=2.0.0)
 Requires-Dist: python-multipart (>=0.0.6) ; extra == "ui" or extra == "all"
 Requires-Dist: pytz (>=2024.1,<2025.0)
 Requires-Dist: pyyaml (>=6.0)
 Requires-Dist: scikit-learn (>=1.0.0) ; extra == "sklearn" or extra == "all"
+Requires-Dist: sqlalchemy (>=2.0.0)
 Requires-Dist: tensorflow (>=2.12.0) ; extra == "tensorflow" or extra == "all"
 Requires-Dist: toml (>=0.10.2)
 Requires-Dist: torch (>=2.0.0) ; extra == "pytorch" or extra == "all"
@@ -304,7 +310,41 @@ Ready for the enterprise. Run locally per project or deploy as a centralized ent
 - **Cloud Providers**: AWS, GCP, Azure (via plugins).
 - **Tools**: MLflow, Weights & Biases, Great Expectations.
 
-### 20. 📂 Project-Based Organization
+### 20. 📊 Automatic Training History Tracking
+FlowyML automatically captures and visualizes your model training progress with zero manual intervention.
+
+```python
+from flowyml.integrations.keras import FlowymlKerasCallback
+
+# Just add the callback - that's it!
+callback = FlowymlKerasCallback(
+    experiment_name="my-experiment",
+    project="my-project"
+)
+
+model.fit(
+    x_train, y_train,
+    validation_data=(x_val, y_val),
+    epochs=50,
+    callbacks=[callback]  # Auto-tracks all metrics!
+)
+```
+
+**What gets captured automatically:**
+- ✅ Loss (train & validation) per epoch
+- ✅ Accuracy (train & validation) per epoch
+- ✅ All custom metrics (F1, precision, recall, etc.)
+- ✅ Model architecture and parameters
+- ✅ Interactive charts in the UI
+
+**View beautiful training graphs in the UI:**
+1. Navigate to your project's Structure tab
+2. Click on any model artifact
+3. See interactive loss/accuracy charts over epochs!
+
+No external tools needed - all visualization built into FlowyML.
+
+### 21. 📂 Project-Based Organization
 Built-in multi-tenancy for managing multiple teams and initiatives.
 
 ```python
@@ -318,7 +358,7 @@ runs = project.list_runs()
 stats = project.get_stats()
 ```
 
-### 21. 📝 Pipeline Templates
+### 22. 📝 Pipeline Templates
 Stop reinventing the wheel. Use pre-built templates for common ML patterns.
 
 ```python
@@ -382,7 +422,7 @@ print(f"Run ID: {result.run_id}")
 print(f"Model Score: {result.outputs['model'].score}")
 ```
 
-### 16. 🌐 Pipeline Versioning
+### 23. 🌐 Pipeline Versioning
 Git-like versioning for pipelines. Track changes, compare, rollback.
 
 ```python

{flowyml-1.2.0 → flowyml-1.4.0}/README.md

@@ -255,7 +255,41 @@ Ready for the enterprise. Run locally per project or deploy as a centralized ent
 - **Cloud Providers**: AWS, GCP, Azure (via plugins).
 - **Tools**: MLflow, Weights & Biases, Great Expectations.
 
-### 20. 📂 Project-Based Organization
+### 20. 📊 Automatic Training History Tracking
+FlowyML automatically captures and visualizes your model training progress with zero manual intervention.
+
+```python
+from flowyml.integrations.keras import FlowymlKerasCallback
+
+# Just add the callback - that's it!
+callback = FlowymlKerasCallback(
+    experiment_name="my-experiment",
+    project="my-project"
+)
+
+model.fit(
+    x_train, y_train,
+    validation_data=(x_val, y_val),
+    epochs=50,
+    callbacks=[callback]  # Auto-tracks all metrics!
+)
+```
+
+**What gets captured automatically:**
+- ✅ Loss (train & validation) per epoch
+- ✅ Accuracy (train & validation) per epoch
+- ✅ All custom metrics (F1, precision, recall, etc.)
+- ✅ Model architecture and parameters
+- ✅ Interactive charts in the UI
+
+**View beautiful training graphs in the UI:**
+1. Navigate to your project's Structure tab
+2. Click on any model artifact
+3. See interactive loss/accuracy charts over epochs!
+
+No external tools needed - all visualization built into FlowyML.
+
+### 21. 📂 Project-Based Organization
 Built-in multi-tenancy for managing multiple teams and initiatives.
 
 ```python
@@ -269,7 +303,7 @@ runs = project.list_runs()
 stats = project.get_stats()
 ```
 
-### 21. 📝 Pipeline Templates
+### 22. 📝 Pipeline Templates
 Stop reinventing the wheel. Use pre-built templates for common ML patterns.
 
 ```python
@@ -333,7 +367,7 @@ print(f"Run ID: {result.run_id}")
 print(f"Model Score: {result.outputs['model'].score}")
 ```
 
-### 16. 🌐 Pipeline Versioning
+### 23. 🌐 Pipeline Versioning
 Git-like versioning for pipelines. Track changes, compare, rollback.
 
 ```python

{flowyml-1.2.0 → flowyml-1.4.0}/flowyml/__init__.py

@@ -38,6 +38,7 @@ from flowyml.assets.registry import AssetRegistry
 # Stack imports
 from flowyml.stacks.base import Stack
 from flowyml.stacks.local import LocalStack
+from flowyml.stacks.components import ResourceConfig, DockerConfig
 
 # Tracking imports
 from flowyml.tracking.experiment import Experiment
@@ -140,6 +141,8 @@ __all__ = [
     # Stacks
     "Stack",
     "LocalStack",
+    "ResourceConfig",
+    "DockerConfig",
     # Tracking
     "Experiment",
     "Run",

{flowyml-1.2.0 → flowyml-1.4.0}/flowyml/assets/base.py

@@ -77,6 +77,16 @@ class Asset:
         if parent:
             parent.children.append(self)
 
+    @property
+    def properties(self) -> dict[str, Any]:
+        """Expose mutable properties stored in metadata."""
+        return self.metadata.properties
+
+    @property
+    def tags(self) -> dict[str, str]:
+        """Expose mutable tags stored in metadata."""
+        return self.metadata.tags
+
     @classmethod
     def create(
         cls,

{flowyml-1.2.0 → flowyml-1.4.0}/flowyml/assets/metrics.py

@@ -89,7 +89,10 @@ class Metrics(Asset):
     def create(
         cls,
         name: str | None = None,
+        version: str | None = None,
         parent: Asset | None = None,
+        tags: dict[str, str] | None = None,
+        properties: dict[str, Any] | None = None,
         **metrics,
     ) -> "Metrics":
         """Factory method to create metrics.
@@ -99,6 +102,9 @@ class Metrics(Asset):
         """
         return cls(
             name=name or "metrics",
+            version=version,
             data=metrics,
             parent=parent,
+            tags=tags,
+            properties=properties,
         )

{flowyml-1.2.0 → flowyml-1.4.0}/flowyml/cli/main.py

@@ -48,9 +48,11 @@ def init(name: str, template: str, directory: str) -> None:
 @click.option("--stack", default="local", help="Stack to use for execution")
 @click.option("--context", "-c", multiple=True, help="Context parameters (key=value)")
 @click.option("--debug", is_flag=True, help="Enable debug mode")
-def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
+@click.option("--retry", type=int, help="Number of retries for the pipeline")
+def run(pipeline_name: str, stack: str, context: tuple, debug: bool, retry: int | None) -> None:
     """Run a pipeline."""
     from flowyml.cli.run import run_pipeline
+    from flowyml.core.retry_policy import OrchestratorRetryPolicy
 
     # Parse context parameters
     ctx_params = {}
@@ -60,8 +62,13 @@ def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
 
     click.echo(f"Running pipeline '{pipeline_name}' on stack '{stack}'...")
 
+    kwargs = {}
+    if retry:
+        kwargs["retry_policy"] = OrchestratorRetryPolicy(max_attempts=retry)
+        click.echo(f"  Retry policy enabled: max_attempts={retry}")
+
     try:
-        result = run_pipeline(pipeline_name, stack, ctx_params, debug)
+        result = run_pipeline(pipeline_name, stack, ctx_params, debug, **kwargs)
         click.echo("✓ Pipeline completed successfully")
         click.echo(f"  Run ID: {result.get('run_id', 'N/A')}")
         click.echo(f"  Duration: {result.get('duration', 'N/A')}")
@@ -70,6 +77,105 @@ def run(pipeline_name: str, stack: str, context: tuple, debug: bool) -> None:
         raise click.Abort()
 
 
+@cli.group()
+def schedule() -> None:
+    """Schedule management commands."""
+    pass
+
+
+@schedule.command("create")
+@click.argument("pipeline_name")
+@click.argument("schedule_type", type=click.Choice(["cron", "interval", "daily", "hourly"]))
+@click.argument("value")
+@click.option("--stack", default="local", help="Stack to use for execution")
+def create_schedule(pipeline_name: str, schedule_type: str, value: str, stack: str) -> None:
+    """Create a new schedule for a pipeline.
+
+    VALUE format depends on SCHEDULE_TYPE:
+    - cron: "*/5 * * * *"
+    - interval: seconds (e.g. 60)
+    - daily: "HH:MM" (e.g. 14:30)
+    - hourly: minute (e.g. 30)
+    """
+    from flowyml.core.scheduler import PipelineScheduler
+    from flowyml.cli.run import run_pipeline
+
+    # We need a callable for the scheduler.
+    # Since CLI is stateless, we wrap the run_pipeline command.
+    # Note: In a real distributed system, this would submit to a scheduler service.
+    # Here we are just registering it in the local scheduler DB.
+
+    # For now, we'll just use the scheduler API to register the definition
+    scheduler = PipelineScheduler()
+
+    # Define a wrapper that runs the pipeline via CLI logic
+    def job_func():
+        run_pipeline(pipeline_name, stack, {}, False)
+
+    try:
+        if schedule_type == "cron":
+            scheduler.schedule_cron(pipeline_name, job_func, value)
+        elif schedule_type == "interval":
+            scheduler.schedule_interval(pipeline_name, job_func, seconds=int(value))
+        elif schedule_type == "daily":
+            if ":" in value:
+                h, m = map(int, value.split(":"))
+                scheduler.schedule_daily(pipeline_name, job_func, hour=h, minute=m)
+            else:
+                raise ValueError("Daily value must be HH:MM")
+        elif schedule_type == "hourly":
+            scheduler.schedule_hourly(pipeline_name, job_func, minute=int(value))
+
+        click.echo(f"✓ Schedule created for '{pipeline_name}' ({schedule_type}={value})")
+        click.echo("  Note: Ensure the scheduler service is running to execute this schedule.")
+    except Exception as e:
+        click.echo(f"✗ Error creating schedule: {e}", err=True)
+
+
+@schedule.command("list")
+def list_schedules() -> None:
+    """List all active schedules."""
+    from flowyml.core.scheduler import PipelineScheduler
+
+    scheduler = PipelineScheduler()
+    jobs = scheduler.get_jobs()
+
+    if not jobs:
+        click.echo("No active schedules found.")
+        return
+
+    click.echo(f"Found {len(jobs)} schedules:\n")
+    for job in jobs:
+        click.echo(f"  {job.id} - {job.name}")
+        click.echo(f"    Next run: {job.next_run_time}")
+        click.echo()
+
+
+@schedule.command("start")
+def start_scheduler() -> None:
+    """Start the scheduler service (blocking)."""
+    from flowyml.core.scheduler import PipelineScheduler
+    import time
+
+    click.echo("🚀 Starting Scheduler Service...")
+    scheduler = PipelineScheduler()
+
+    try:
+        # In a real app, this would load definitions from DB and register them
+        # For now, it just runs the scheduler loop for existing in-memory jobs
+        # (which might be empty if we restarted).
+        # To make this persistent, we'd need to serialize job definitions to DB.
+        # The current Scheduler implementation supports SQLite persistence for job state,
+        # but we need to re-register jobs on startup.
+
+        click.echo("  Scheduler running. Press Ctrl+C to stop.")
+        while True:
+            scheduler.run_pending()
+            time.sleep(1)
+    except KeyboardInterrupt:
+        click.echo("\n🛑 Scheduler stopped.")
+
+
 @cli.group()
 def ui() -> None:
     """UI server commands."""

{flowyml-1.2.0 → flowyml-1.4.0}/flowyml/cli/run.py

@@ -6,7 +6,13 @@ from pathlib import Path
 from typing import Any
 
 
-def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any], debug: bool) -> dict[str, Any]:
+def run_pipeline(
+    pipeline_name: str,
+    stack: str,
+    context_params: dict[str, Any],
+    debug: bool,
+    **kwargs,
+) -> dict[str, Any]:
     """Run a pipeline by name.
 
     Args:
@@ -14,6 +20,7 @@ def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any],
         stack: Stack to use for execution
         context_params: Context parameters to override
         debug: Enable debug mode
+        **kwargs: Additional arguments passed to pipeline.run
 
     Returns:
         Dictionary with run results
@@ -65,7 +72,7 @@ def run_pipeline(pipeline_name: str, stack: str, context_params: dict[str, Any],
         pipeline.set_stack(stack)
 
     # Run pipeline
-    result = pipeline.run(debug=debug)
+    result = pipeline.run(debug=debug, **kwargs)
 
     return {
         "run_id": result.run_id,

flowyml-1.4.0/flowyml/core/execution_status.py

@@ -0,0 +1,52 @@
+"""Execution status tracking for pipeline runs."""
+
+from enum import Enum
+
+
+class ExecutionStatus(str, Enum):
+    """Status of a pipeline or step execution."""
+
+    # Pre-execution states
+    INITIALIZING = "initializing"
+    PROVISIONING = "provisioning"
+
+    # Active execution states
+    RUNNING = "running"
+
+    # Terminal success states
+    COMPLETED = "completed"
+    CACHED = "cached"
+
+    # Terminal failure states
+    FAILED = "failed"
+    STOPPED = "stopped"
+    CANCELLED = "cancelled"
+
+    # Intermediate states
+    STOPPING = "stopping"
+    CANCELLING = "cancelling"
+
+    @property
+    def is_finished(self) -> bool:
+        """Check if execution is in a terminal state."""
+        return self in {
+            ExecutionStatus.COMPLETED,
+            ExecutionStatus.CACHED,
+            ExecutionStatus.FAILED,
+            ExecutionStatus.STOPPED,
+            ExecutionStatus.CANCELLED,
+        }
+
+    @property
+    def is_successful(self) -> bool:
+        """Check if execution completed successfully."""
+        return self in {ExecutionStatus.COMPLETED, ExecutionStatus.CACHED}
+
+    @property
+    def is_failed(self) -> bool:
+        """Check if execution failed."""
+        return self in {
+            ExecutionStatus.FAILED,
+            ExecutionStatus.STOPPED,
+            ExecutionStatus.CANCELLED,
+        }

flowyml-1.4.0/flowyml/core/hooks.py

@@ -0,0 +1,106 @@
+"""Lifecycle hooks for pipelines and steps."""
+
+from typing import Any, TYPE_CHECKING
+from collections.abc import Callable
+from dataclasses import dataclass, field
+
+if TYPE_CHECKING:
+    from flowyml.core.pipeline import Pipeline, PipelineResult
+    from flowyml.core.step import Step
+    from flowyml.core.executor import ExecutionResult
+
+
+@dataclass
+class HookRegistry:
+    """Registry for pipeline and step lifecycle hooks."""
+
+    # Pipeline-level hooks
+    on_pipeline_start: list[Callable[["Pipeline"], None]] = field(default_factory=list)
+    on_pipeline_end: list[Callable[["Pipeline", "PipelineResult"], None]] = field(default_factory=list)
+
+    # Step-level hooks
+    on_step_start: list[Callable[["Step", dict[str, Any]], None]] = field(default_factory=list)
+    on_step_end: list[Callable[["Step", "ExecutionResult"], None]] = field(default_factory=list)
+
+    def register_pipeline_start_hook(self, hook: Callable[["Pipeline"], None]) -> None:
+        """Register a hook to run at pipeline start."""
+        self.on_pipeline_start.append(hook)
+
+    def register_pipeline_end_hook(self, hook: Callable[["Pipeline", "PipelineResult"], None]) -> None:
+        """Register a hook to run at pipeline end."""
+        self.on_pipeline_end.append(hook)
+
+    def register_step_start_hook(self, hook: Callable[["Step", dict[str, Any]], None]) -> None:
+        """Register a hook to run before step execution."""
+        self.on_step_start.append(hook)
+
+    def register_step_end_hook(self, hook: Callable[["Step", "ExecutionResult"], None]) -> None:
+        """Register a hook to run after step execution."""
+        self.on_step_end.append(hook)
+
+    def run_pipeline_start_hooks(self, pipeline: "Pipeline") -> None:
+        """Execute all pipeline start hooks."""
+        for hook in self.on_pipeline_start:
+            try:
+                hook(pipeline)
+            except Exception as e:
+                print(f"Warning: Pipeline start hook failed: {e}")
+
+    def run_pipeline_end_hooks(self, pipeline: "Pipeline", result: "PipelineResult") -> None:
+        """Execute all pipeline end hooks."""
+        for hook in self.on_pipeline_end:
+            try:
+                hook(pipeline, result)
+            except Exception as e:
+                print(f"Warning: Pipeline end hook failed: {e}")
+
+    def run_step_start_hooks(self, step: "Step", inputs: dict[str, Any]) -> None:
+        """Execute all step start hooks."""
+        for hook in self.on_step_start:
+            try:
+                hook(step, inputs)
+            except Exception as e:
+                print(f"Warning: Step start hook failed: {e}")
+
+    def run_step_end_hooks(self, step: "Step", result: "ExecutionResult") -> None:
+        """Execute all step end hooks."""
+        for hook in self.on_step_end:
+            try:
+                hook(step, result)
+            except Exception as e:
+                print(f"Warning: Step end hook failed: {e}")
+
+
+# Global hook registry
+_global_hooks = HookRegistry()
+
+
+def get_global_hooks() -> HookRegistry:
+    """Get the global hook registry."""
+    return _global_hooks
+
+
+def on_pipeline_start(func: Callable[["Pipeline"], None]) -> Callable[["Pipeline"], None]:
+    """Decorator to register a pipeline start hook."""
+    _global_hooks.register_pipeline_start_hook(func)
+    return func
+
+
+def on_pipeline_end(
+    func: Callable[["Pipeline", "PipelineResult"], None],
+) -> Callable[["Pipeline", "PipelineResult"], None]:
+    """Decorator to register a pipeline end hook."""
+    _global_hooks.register_pipeline_end_hook(func)
+    return func
+
+
+def on_step_start(func: Callable[["Step", dict[str, Any]], None]) -> Callable[["Step", dict[str, Any]], None]:
+    """Decorator to register a step start hook."""
+    _global_hooks.register_step_start_hook(func)
+    return func
+
+
+def on_step_end(func: Callable[["Step", "ExecutionResult"], None]) -> Callable[["Step", "ExecutionResult"], None]:
+    """Decorator to register a step end hook."""
+    _global_hooks.register_step_end_hook(func)
+    return func