flowyml 1.4.0__py3-none-any.whl → 1.6.0__py3-none-any.whl

flowyml/core/scheduler.py

@@ -3,7 +3,6 @@
 import contextlib
 import json
 import logging
-import sqlite3
 import threading
 import time
 from collections.abc import Callable
@@ -12,6 +11,26 @@ from datetime import datetime, timedelta
 from pathlib import Path
 from typing import Any
 
+from sqlalchemy import (
+    create_engine,
+    MetaData,
+    Table,
+    Column,
+    String,
+    Integer,
+    Float,
+    Text,
+    Boolean,
+    DateTime,
+    ForeignKey,
+    select,
+    insert,
+    update,
+    delete,
+    func,
+)
+from sqlalchemy.pool import StaticPool
+
 from flowyml.core.scheduler_config import SchedulerConfig
 
 logger = logging.getLogger(__name__)
@@ -81,6 +100,7 @@ class ScheduleExecution:
     success: bool = False
     error: str | None = None
     duration_seconds: float | None = None
+    run_id: str | None = None  # Pipeline run_id if available
 
 
 class SchedulerMetrics:
@@ -119,54 +139,83 @@ class SchedulerMetrics:
 
 
 class SchedulerPersistence:
-    """Persist schedules to SQLite database."""
+    """Persist schedules to SQLite database using SQLAlchemy."""
 
     def __init__(self, db_path: str | None = None):
         self.db_path = db_path or str(Path.cwd() / ".flowyml_scheduler.db")
+        # Convert to absolute path for SQLite URL
+        abs_path = Path(self.db_path).resolve()
+        abs_path.parent.mkdir(parents=True, exist_ok=True)
+        self.db_url = f"sqlite:///{abs_path}"
         self._init_db()
 
     def _init_db(self):
-        """Initialize database schema."""
-        with sqlite3.connect(self.db_path) as conn:
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS schedules (
-                    name TEXT PRIMARY KEY,
-                    data TEXT NOT NULL,
-                    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
-                )
-                """,
-            )
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS executions (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    schedule_name TEXT NOT NULL,
-                    started_at TIMESTAMP NOT NULL,
-                    completed_at TIMESTAMP,
-                    success BOOLEAN,
-                    error TEXT,
-                    duration_seconds REAL,
-                    FOREIGN KEY(schedule_name) REFERENCES schedules(name)
-                )
-                """,
-            )
+        """Initialize database schema using SQLAlchemy."""
+        # Create engine
+        self.engine = create_engine(
+            self.db_url,
+            connect_args={"check_same_thread": False},
+            poolclass=StaticPool,
+        )
+
+        self.metadata = MetaData()
+
+        # Define schedules table
+        self.schedules = Table(
+            "schedules",
+            self.metadata,
+            Column("name", String, primary_key=True),
+            Column("data", Text, nullable=False),
+            Column("updated_at", DateTime, server_default=func.current_timestamp()),
+        )
+
+        # Define executions table
+        self.executions = Table(
+            "executions",
+            self.metadata,
+            Column("id", Integer, primary_key=True, autoincrement=True),
+            Column("schedule_name", String, ForeignKey("schedules.name"), nullable=False),
+            Column("started_at", DateTime, nullable=False),
+            Column("completed_at", DateTime, nullable=True),
+            Column("success", Boolean, nullable=True),
+            Column("error", Text, nullable=True),
+            Column("duration_seconds", Float, nullable=True),
+            Column("run_id", String, nullable=True),
+        )
+
+        # Create all tables
+        self.metadata.create_all(self.engine)
 
     def save_schedule(self, schedule: Schedule) -> None:
-        """Save schedule to database."""
+        """Save schedule to database using SQLAlchemy."""
         data = schedule.to_dict()
-        with sqlite3.connect(self.db_path) as conn:
-            conn.execute(
-                "INSERT OR REPLACE INTO schedules (name, data) VALUES (?, ?)",
-                (schedule.pipeline_name, json.dumps(data)),
+        with self.engine.connect() as conn:
+            # Use INSERT OR REPLACE equivalent in SQLAlchemy
+            stmt = (
+                update(self.schedules)
+                .where(self.schedules.c.name == schedule.pipeline_name)
+                .values(data=json.dumps(data), updated_at=func.current_timestamp())
             )
+            result = conn.execute(stmt)
+            conn.commit()
+
+            # If no rows were updated, insert new record
+            if result.rowcount == 0:
+                stmt = insert(self.schedules).values(
+                    name=schedule.pipeline_name,
+                    data=json.dumps(data),
+                )
+                conn.execute(stmt)
+                conn.commit()
 
     def load_schedules(self, pipeline_funcs: dict[str, Callable]) -> dict[str, Schedule]:
-        """Load all schedules from database."""
+        """Load all schedules from database using SQLAlchemy."""
         schedules = {}
-        with sqlite3.connect(self.db_path) as conn:
-            cursor = conn.execute("SELECT name, data FROM schedules")
-            for name, data_json in cursor:
+        with self.engine.connect() as conn:
+            stmt = select(self.schedules.c.name, self.schedules.c.data)
+            result = conn.execute(stmt)
+            for row in result:
+                name, data_json = row
                 try:
                     data = json.loads(data_json)
                     if name in pipeline_funcs:
@@ -176,52 +225,72 @@ class SchedulerPersistence:
         return schedules
 
     def delete_schedule(self, name: str) -> None:
-        """Delete schedule from database."""
-        with sqlite3.connect(self.db_path) as conn:
-            conn.execute("DELETE FROM schedules WHERE name = ?", (name,))
-            conn.execute("DELETE FROM executions WHERE schedule_name = ?", (name,))
+        """Delete schedule from database using SQLAlchemy."""
+        with self.engine.connect() as conn:
+            # Delete executions first (foreign key constraint)
+            stmt = delete(self.executions).where(self.executions.c.schedule_name == name)
+            conn.execute(stmt)
+
+            # Delete schedule
+            stmt = delete(self.schedules).where(self.schedules.c.name == name)
+            conn.execute(stmt)
+            conn.commit()
 
     def save_execution(self, execution: ScheduleExecution) -> None:
-        """Save execution record."""
-        with sqlite3.connect(self.db_path) as conn:
-            conn.execute(
-                """
-                INSERT INTO executions
-                (schedule_name, started_at, completed_at, success, error, duration_seconds)
-                VALUES (?, ?, ?, ?, ?, ?)
-                """,
-                (
-                    execution.schedule_name,
-                    execution.started_at,
-                    execution.completed_at,
-                    execution.success,
-                    execution.error,
-                    execution.duration_seconds,
-                ),
+        """Save execution record using SQLAlchemy."""
+        with self.engine.connect() as conn:
+            stmt = insert(self.executions).values(
+                schedule_name=execution.schedule_name,
+                started_at=execution.started_at,
+                completed_at=execution.completed_at,
+                success=execution.success,
+                error=execution.error,
+                duration_seconds=execution.duration_seconds,
+                run_id=execution.run_id,
             )
+            conn.execute(stmt)
+            conn.commit()
 
     def get_history(self, schedule_name: str, limit: int = 50) -> list[dict[str, Any]]:
-        """Get execution history for a schedule."""
+        """Get execution history for a schedule using SQLAlchemy."""
         history = []
-        with sqlite3.connect(self.db_path) as conn:
-            cursor = conn.execute(
-                """
-                SELECT started_at, completed_at, success, error, duration_seconds
-                FROM executions
-                WHERE schedule_name = ?
-                ORDER BY started_at DESC
-                LIMIT ?
-                """,
-                (schedule_name, limit),
+        with self.engine.connect() as conn:
+            stmt = (
+                select(
+                    self.executions.c.started_at,
+                    self.executions.c.completed_at,
+                    self.executions.c.success,
+                    self.executions.c.error,
+                    self.executions.c.duration_seconds,
+                    self.executions.c.run_id,
+                )
+                .where(self.executions.c.schedule_name == schedule_name)
+                .order_by(self.executions.c.started_at.desc())
+                .limit(limit)
             )
-            for row in cursor:
+            result = conn.execute(stmt)
+            for row in result:
+                # Handle datetime conversion
+                started_at = row.started_at
+                if isinstance(started_at, datetime):
+                    started_at = started_at.isoformat()
+                elif started_at is not None:
+                    started_at = str(started_at)
+
+                completed_at = row.completed_at
+                if isinstance(completed_at, datetime):
+                    completed_at = completed_at.isoformat()
+                elif completed_at is not None:
+                    completed_at = str(completed_at)
+
                 history.append(
                     {
-                        "started_at": row[0],
-                        "completed_at": row[1],
-                        "success": bool(row[2]),
-                        "error": row[3],
-                        "duration_seconds": row[4],
+                        "started_at": started_at,
+                        "completed_at": completed_at,
+                        "success": bool(row.success) if row.success is not None else False,
+                        "error": row.error,
+                        "duration_seconds": row.duration_seconds,
+                        "run_id": row.run_id,
                     },
                 )
         return history
@@ -474,11 +543,11 @@ class PipelineScheduler:
         """Remove all schedules."""
         self.schedules.clear()
         if self._persistence:
-            # Re-initialize DB to clear it
-            db_path = Path(self._persistence.db_path)
-            if db_path.exists():
-                db_path.unlink()
-            self._persistence._init_db()
+            # Delete all schedules and executions using SQLAlchemy
+            with self._persistence.engine.connect() as conn:
+                conn.execute(delete(self._persistence.executions))
+                conn.execute(delete(self._persistence.schedules))
+                conn.commit()
 
     def enable(self, name: str) -> None:
         """Enable a schedule."""
@@ -507,17 +576,33 @@ class PipelineScheduler:
             started_at=datetime.now(),
         )
 
+        pipeline_result = None
         try:
             logger.info(f"Starting scheduled run: {schedule.pipeline_name}")
-            schedule.pipeline_func()
-            execution.success = True
+
+            # Execute pipeline and capture result
+            result = schedule.pipeline_func()
+
+            # Check if result is a PipelineResult object
+            from flowyml.core.pipeline import PipelineResult
+
+            if isinstance(result, PipelineResult):
+                pipeline_result = result
+                execution.success = result.success
+                # Store run_id for tracking
+                execution.run_id = result.run_id
+            else:
+                # Assume success if no exception and result is truthy
+                execution.success = bool(result)
+
             schedule.last_run = datetime.now(pytz.timezone(schedule.timezone)) if pytz else datetime.now()
 
             if self.on_success:
                 self.on_success(schedule, execution)
         except Exception as e:
-            logger.error(f"Schedule {schedule.pipeline_name} failed: {e}")
+            logger.error(f"Schedule {schedule.pipeline_name} failed: {e}", exc_info=True)
             execution.error = str(e)
+            execution.success = False
             if self.on_failure:
                 self.on_failure(schedule, execution, e)
         finally:
@@ -525,6 +610,65 @@ class PipelineScheduler:
             execution.duration_seconds = (execution.completed_at - execution.started_at).total_seconds()
             self.metrics.update(execution)
 
+            # Ensure pipeline result is saved to metadata store for UI visibility
+            # The pipeline should have already saved via _save_run, but we ensure it's in the UI's store
+            if pipeline_result:
+                try:
+                    # Get the global metadata store used by UI (same as pipeline should use)
+                    from flowyml.ui.backend.dependencies import get_store
+
+                    # Get the UI's metadata store
+                    ui_store = get_store()
+
+                    # Also ensure the pipeline's metadata store is the same instance/path
+                    # If the pipeline used a different store, sync to UI store
+
+                    # Check if run is already in UI store
+                    existing_run = ui_store.load_run(pipeline_result.run_id)
+                    if not existing_run:
+                        # Run wasn't saved to UI store, save it now
+                        # Build comprehensive metadata
+                        metadata = {
+                            "run_id": pipeline_result.run_id,
+                            "pipeline_name": pipeline_result.pipeline_name,
+                            "status": "completed" if pipeline_result.success else "failed",
+                            "start_time": pipeline_result.start_time.isoformat(),
+                            "end_time": pipeline_result.end_time.isoformat() if pipeline_result.end_time else None,
+                            "duration": pipeline_result.duration_seconds,
+                            "success": pipeline_result.success,
+                            "scheduled": True,  # Mark as scheduled run
+                            "schedule_name": schedule.pipeline_name,
+                            "steps": {
+                                name: {
+                                    "success": result.success,
+                                    "duration": result.duration_seconds,
+                                    "cached": result.cached,
+                                    "retries": result.retries,
+                                    "error": result.error,
+                                }
+                                for name, result in pipeline_result.step_results.items()
+                            },
+                        }
+
+                        # Add outputs if available
+                        if pipeline_result.outputs:
+                            metadata["outputs"] = {
+                                k: str(v)[:200] if not isinstance(v, (dict, list)) else str(v)[:200]
+                                for k, v in pipeline_result.outputs.items()
+                            }
+
+                        ui_store.save_run(pipeline_result.run_id, metadata)
+                        logger.info(f"✅ Saved scheduled run {pipeline_result.run_id} to UI metadata store")
+                    else:
+                        # Update existing run to mark as scheduled
+                        if not existing_run.get("scheduled"):
+                            existing_run["scheduled"] = True
+                            existing_run["schedule_name"] = schedule.pipeline_name
+                            ui_store.save_run(pipeline_result.run_id, existing_run)
+                            logger.debug(f"Updated run {pipeline_result.run_id} to mark as scheduled")
+                except Exception as e:
+                    logger.warning(f"Failed to save scheduled run to UI metadata store: {e}", exc_info=True)
+
             if self.config.distributed:
                 self._lock.release(schedule.pipeline_name)
 

flowyml/core/versioning.py

@@ -27,9 +27,9 @@ class VersionedPipeline:
     Tracks changes between versions and allows comparison.
 
     Examples:
-        >>> from flowyml import VersionedPipeline, step
-        >>> pipeline = VersionedPipeline("training")
-        >>> pipeline.version = "v1.0.0"
+        >>> from flowyml import VersionedPipeline, step, context
+        >>> ctx = context(learning_rate=0.001, epochs=10)
+        >>> pipeline = VersionedPipeline("training", context=ctx, version="v1.0.0", project_name="ml_project")
         >>> pipeline.add_step(load_data)
         >>> pipeline.add_step(train_model)
         >>> pipeline.save_version()
@@ -39,6 +39,10 @@ class VersionedPipeline:
         >>> pipeline.save_version()
         >>> # Compare versions
         >>> diff = pipeline.compare_with("v1.0.0")
+
+        # Or use Pipeline with version parameter (automatically creates VersionedPipeline)
+        >>> from flowyml import Pipeline
+        >>> pipeline = Pipeline("training", context=ctx, version="v1.0.1", project_name="ml_project")
     """
 
     def __init__(
@@ -46,12 +50,17 @@ class VersionedPipeline:
         name: str,
         version: str = "v0.1.0",
         versions_dir: str = ".flowyml/versions",
+        context: Any | None = None,
+        **kwargs,
     ):
         from flowyml.core.pipeline import Pipeline
 
         self.name = name
         self._version = version
-        self.pipeline = Pipeline(name)
+        # Pass context and other kwargs to the internal Pipeline
+        # Remove 'version' from kwargs to avoid recursion
+        pipeline_kwargs = {k: v for k, v in kwargs.items() if k != "version"}
+        self.pipeline = Pipeline(name, context=context, **pipeline_kwargs)
 
         # Version storage
         self.versions_dir = Path(versions_dir) / name

flowyml/registry/model_registry.py

@@ -184,7 +184,7 @@ class ModelRegistry:
             path: Path to save to
             framework: Framework name
         """
-        from flowyml.storage.materializers import get_materializer
+        from flowyml.storage.materializers.base import get_materializer
 
         # Try to get appropriate materializer
         materializer = get_materializer(model)

flowyml/storage/sql.py

@@ -884,26 +884,66 @@ class SQLMetadataStore(MetadataStore):
                 "period_days": days,
             }
 
-    def get_statistics(self) -> dict:
+    def get_statistics(self, project: str | None = None) -> dict:
         """Get global statistics."""
         with self.engine.connect() as conn:
-            # Total runs
-            total_runs = conn.execute(select(func.count()).select_from(self.runs)).scalar()
+            # 1. Total runs
+            runs_stmt = select(func.count()).select_from(self.runs)
+            if project:
+                runs_stmt = runs_stmt.where(self.runs.c.project == project)
+            total_runs = conn.execute(runs_stmt).scalar() or 0
 
-            # Total pipelines
-            total_pipelines = conn.execute(
-                select(func.count(func.distinct(self.runs.c.pipeline_name))),
-            ).scalar()
+            # 2. Total pipelines (unique names)
+            pipelines_stmt = select(func.count(func.distinct(self.runs.c.pipeline_name)))
+            if project:
+                pipelines_stmt = pipelines_stmt.where(self.runs.c.project == project)
+            total_pipelines = conn.execute(pipelines_stmt).scalar() or 0
 
-            # Total experiments
-            total_experiments = conn.execute(select(func.count()).select_from(self.experiments)).scalar()
+            # 3. Total artifacts
+            artifacts_stmt = select(func.count()).select_from(self.artifacts)
+            if project:
+                artifacts_stmt = artifacts_stmt.where(self.artifacts.c.project == project)
+            total_artifacts = conn.execute(artifacts_stmt).scalar() or 0
 
-            # Total models (unique model names in metrics)
-            total_models = conn.execute(
-                select(func.count(func.distinct(self.model_metrics.c.model_name))),
-            ).scalar()
+            # 4. Total experiments
+            experiments_stmt = select(func.count()).select_from(self.experiments)
+            if project:
+                experiments_stmt = experiments_stmt.where(self.experiments.c.project == project)
+            total_experiments = conn.execute(experiments_stmt).scalar() or 0
+
+            # 5. Total models
+            models_stmt = select(func.count(func.distinct(self.model_metrics.c.model_name)))
+            if project:
+                models_stmt = models_stmt.where(self.model_metrics.c.project == project)
+            total_models = conn.execute(models_stmt).scalar() or 0
+
+            # 6. Status counts (completed vs failed)
+            status_stmt = select(self.runs.c.status, func.count()).group_by(self.runs.c.status)
+            if project:
+                status_stmt = status_stmt.where(self.runs.c.project == project)
+
+            status_rows = conn.execute(status_stmt).fetchall()
+            status_map = {row[0]: row[1] for row in status_rows if row[0]}
+
+            completed_runs = status_map.get("completed", 0)
+            failed_runs = status_map.get("failed", 0)
+
+            # 7. Avg duration (only completed runs)
+            dur_stmt = select(func.avg(self.runs.c.duration)).where(self.runs.c.status == "completed")
+            if project:
+                dur_stmt = dur_stmt.where(self.runs.c.project == project)
+
+            avg_duration = conn.execute(dur_stmt).scalar() or 0.0
 
             return {
+                # Frontend-friendly keys
+                "pipelines": total_pipelines,
+                "runs": total_runs,
+                "artifacts": total_artifacts,
+                "completed_runs": completed_runs,
+                "failed_runs": failed_runs,
+                "avg_duration": avg_duration,
+                # Backward compatibility
                 "total_runs": total_runs,
                 "total_pipelines": total_pipelines,
                 "total_experiments": total_experiments,

flowyml/ui/backend/main.py

@@ -24,6 +24,7 @@ from flowyml.ui.backend.routers import (
     metrics,
     client,
     stats,
+    websocket,
 )
 
 app = FastAPI(
@@ -77,6 +78,7 @@ app.include_router(metrics.router, prefix="/api/metrics", tags=["metrics"])
 app.include_router(plugins.router, prefix="/api", tags=["plugins"])
 app.include_router(client.router, prefix="/api/client", tags=["client"])
 app.include_router(stats.router, prefix="/api/stats", tags=["stats"])
+app.include_router(websocket.router, tags=["websocket"])
 
 
 # Static file serving for frontend

flowyml/ui/backend/routers/assets.py

@@ -476,6 +476,42 @@ async def download_asset(artifact_id: str):
     )
 
 
+@router.get("/{artifact_id}/content")
+async def get_asset_content(artifact_id: str):
+    """Get the artifact content for inline viewing."""
+    import mimetypes
+
+    asset, _ = _find_asset_with_store(artifact_id)
+    if not asset:
+        raise HTTPException(status_code=404, detail="Asset not found")
+
+    artifact_path = asset.get("path")
+    if not artifact_path:
+        raise HTTPException(status_code=404, detail="Artifact path not available")
+
+    # Handle relative paths for local store
+    from flowyml.utils.config import get_config
+
+    config = get_config()
+
+    file_path = Path(artifact_path)
+    if not file_path.is_absolute():
+        file_path = config.artifacts_dir / file_path
+
+    if not file_path.exists():
+        raise HTTPException(status_code=404, detail="Artifact file not found on disk")
+
+    # Guess mime type
+    mime_type, _ = mimetypes.guess_type(file_path.name)
+    if not mime_type:
+        mime_type = "text/plain"  # Default fallback
+
+    return FileResponse(
+        path=file_path,
+        media_type=mime_type,
+    )
+
+
 class ProjectUpdate(BaseModel):
     project_name: str
 

flowyml/ui/backend/routers/execution.py

@@ -97,10 +97,10 @@ async def execute_pipeline(
         run_kwargs = request.parameters.copy()
 
         if request.retry_count > 0:
-            from flowyml.core.retry import OrchestratorRetryPolicy
+            from flowyml.core.retry_policy import OrchestratorRetryPolicy
 
             run_kwargs["retry_policy"] = OrchestratorRetryPolicy(
-                max_retries=min(request.retry_count, 5),  # Cap at 5
+                max_attempts=min(request.retry_count, 5),  # Cap at 5
             )
 
         result = pipeline.run(**run_kwargs)