flowyml 1.7.2__py3-none-any.whl → 1.8.0__py3-none-any.whl

flowyml/storage/sql.py

@@ -2,32 +2,34 @@
 
 import json
 import logging
-from pathlib import Path
 from datetime import datetime
+from pathlib import Path
 
 # Python 3.11+ has UTC, but Python 3.10 doesn't
 try:
     from datetime import UTC
 except ImportError:
-    UTC = UTC
+    from datetime import timezone
+
+    UTC = timezone.utc
 
 from sqlalchemy import (
-    create_engine,
-    MetaData,
-    Table,
     Column,
-    String,
-    Integer,
     Float,
-    Text,
     ForeignKey,
-    select,
-    insert,
-    update,
+    Integer,
+    MetaData,
+    String,
+    Table,
+    Text,
+    create_engine,
     delete,
     func,
-    text,
+    insert,
     inspect,
+    select,
+    text,
+    update,
 )
 from sqlalchemy.pool import StaticPool
 
@@ -204,6 +206,53 @@ class SQLMetadataStore(MetadataStore):
             Column("updated_at", String, nullable=False),
         )
 
+        # Projects table for proper project storage
+        self.projects = Table(
+            "projects",
+            self.metadata,
+            Column("name", String, primary_key=True),
+            Column("description", Text),
+            Column("tags", Text),  # JSON
+            Column("created_at", String, server_default=func.current_timestamp()),
+            Column("updated_at", String),
+        )
+
+        # Model registry table for versioned model management
+        self.model_versions = Table(
+            "model_versions",
+            self.metadata,
+            Column("id", Integer, primary_key=True, autoincrement=True),
+            Column("name", String, nullable=False),
+            Column("version", String, nullable=False),
+            Column("stage", String, nullable=False),  # development, staging, production, archived
+            Column("framework", String, nullable=False),
+            Column("model_path", String, nullable=False),
+            Column("metrics", Text),  # JSON
+            Column("tags", Text),  # JSON
+            Column("schema", Text),  # JSON - model input/output schema
+            Column("description", Text),
+            Column("author", String),
+            Column("parent_version", String),
+            Column("created_at", String, server_default=func.current_timestamp()),
+            Column("updated_at", String),
+        )
+
+        # Pipeline templates for reusable pipeline configurations
+        self.pipeline_templates = Table(
+            "pipeline_templates",
+            self.metadata,
+            Column("template_id", String, primary_key=True),
+            Column("name", String, nullable=False),
+            Column("description", Text),
+            Column("definition", Text, nullable=False),  # JSON pipeline config
+            Column("tags", Text),  # JSON
+            Column("author", String),
+            Column("category", String),  # e.g., "training", "inference", "data-processing"
+            Column("is_public", Integer, default=0),  # 0=private, 1=public
+            Column("created_at", String, server_default=func.current_timestamp()),
+            Column("updated_at", String),
+        )
+
         # Create tables
         self.metadata.create_all(self.engine)
 
@@ -639,6 +688,33 @@ class SQLMetadataStore(MetadataStore):
                 )
             return experiments
 
+    def list_experiment_runs(self, experiment_id: str) -> list[dict]:
+        """List all runs in an experiment."""
+        with self.engine.connect() as conn:
+            stmt = (
+                select(
+                    self.experiment_runs.c.run_id,
+                    self.experiment_runs.c.metrics,
+                    self.experiment_runs.c.parameters,
+                    self.experiment_runs.c.timestamp,
+                )
+                .where(
+                    self.experiment_runs.c.experiment_id == experiment_id,
+                )
+                .order_by(self.experiment_runs.c.timestamp.desc())
+            )
+
+            rows = conn.execute(stmt).fetchall()
+            return [
+                {
+                    "run_id": row[0],
+                    "metrics": json.loads(row[1]) if row[1] else {},
+                    "parameters": json.loads(row[2]) if row[2] else {},
+                    "created_at": str(row[3]),
+                }
+                for row in rows
+            ]
+
     def update_experiment_project(self, experiment_name: str, project_name: str) -> None:
         """Update the project for an experiment."""
         with self.engine.connect() as conn:
@@ -967,3 +1043,316 @@ class SQLMetadataStore(MetadataStore):
                 "total_experiments": total_experiments,
                 "total_models": total_models,
             }
+
+    # ==================== Project CRUD ====================
+
+    def save_project(self, name: str, description: str = "", tags: dict = None) -> None:
+        """Save or update a project."""
+        now = datetime.now(UTC).isoformat()
+        with self.engine.connect() as conn:
+            stmt = select(self.projects).where(self.projects.c.name == name)
+            existing = conn.execute(stmt).fetchone()
+
+            values = {
+                "name": name,
+                "description": description,
+                "tags": json.dumps(tags or {}),
+                "updated_at": now,
+            }
+
+            if existing:
+                conn.execute(
+                    update(self.projects).where(self.projects.c.name == name).values(**values),
+                )
+            else:
+                values["created_at"] = now
+                conn.execute(insert(self.projects).values(**values))
+
+            conn.commit()
+
+    def get_project(self, name: str) -> dict | None:
+        """Get a project by name."""
+        with self.engine.connect() as conn:
+            stmt = select(self.projects).where(self.projects.c.name == name)
+            row = conn.execute(stmt).fetchone()
+            if row:
+                return {
+                    "name": row.name,
+                    "description": row.description,
+                    "tags": json.loads(row.tags) if row.tags else {},
+                    "created_at": row.created_at,
+                    "updated_at": row.updated_at,
+                }
+            return None
+
+    def list_projects(self) -> list[dict]:
+        """List all projects."""
+        with self.engine.connect() as conn:
+            stmt = select(self.projects).order_by(self.projects.c.created_at.desc())
+            rows = conn.execute(stmt).fetchall()
+            return [
+                {
+                    "name": row.name,
+                    "description": row.description,
+                    "tags": json.loads(row.tags) if row.tags else {},
+                    "created_at": row.created_at,
+                    "updated_at": row.updated_at,
+                }
+                for row in rows
+            ]
+
+    def delete_project(self, name: str) -> None:
+        """Delete a project."""
+        with self.engine.connect() as conn:
+            conn.execute(delete(self.projects).where(self.projects.c.name == name))
+            conn.commit()
+
+    # ========== Model Registry Methods ==========
+
+    def save_model_version(
+        self,
+        name: str,
+        version: str,
+        stage: str,
+        framework: str,
+        model_path: str,
+        metrics: dict | None = None,
+        tags: dict | None = None,
+        schema: dict | None = None,
+        description: str = "",
+        author: str | None = None,
+        parent_version: str | None = None,
+    ) -> int:
+        """Save a model version to the registry.
+
+        Returns:
+            The ID of the saved model version.
+        """
+        now = datetime.now(UTC).isoformat()
+        with self.engine.connect() as conn:
+            # Check if version already exists
+            stmt = select(self.model_versions).where(
+                (self.model_versions.c.name == name) & (self.model_versions.c.version == version),
+            )
+            existing = conn.execute(stmt).fetchone()
+
+            values = {
+                "name": name,
+                "version": version,
+                "stage": stage,
+                "framework": framework,
+                "model_path": model_path,
+                "metrics": json.dumps(metrics or {}),
+                "tags": json.dumps(tags or {}),
+                "schema": json.dumps(schema or {}),
+                "description": description,
+                "author": author,
+                "parent_version": parent_version,
+                "updated_at": now,
+            }
+
+            if existing:
+                conn.execute(
+                    update(self.model_versions)
+                    .where(
+                        (self.model_versions.c.name == name) & (self.model_versions.c.version == version),
+                    )
+                    .values(**values),
+                )
+                conn.commit()
+                return existing.id
+            else:
+                result = conn.execute(insert(self.model_versions).values(**values))
+                conn.commit()
+                return result.lastrowid
+
+    def get_model_version(self, name: str, version: str) -> dict | None:
+        """Get a specific model version."""
+        with self.engine.connect() as conn:
+            stmt = select(self.model_versions).where(
+                (self.model_versions.c.name == name) & (self.model_versions.c.version == version),
+            )
+            row = conn.execute(stmt).fetchone()
+            if row:
+                return self._row_to_model_version(row)
+            return None
+
+    def _row_to_model_version(self, row) -> dict:
+        """Convert a row to a model version dict."""
+        return {
+            "id": row.id,
+            "name": row.name,
+            "version": row.version,
+            "stage": row.stage,
+            "framework": row.framework,
+            "model_path": row.model_path,
+            "metrics": json.loads(row.metrics) if row.metrics else {},
+            "tags": json.loads(row.tags) if row.tags else {},
+            "schema": json.loads(row.schema) if row.schema else {},
+            "description": row.description,
+            "author": row.author,
+            "parent_version": row.parent_version,
+            "created_at": row.created_at,
+            "updated_at": row.updated_at,
+        }
+
+    def list_model_versions(self, name: str | None = None, stage: str | None = None) -> list[dict]:
+        """List model versions with optional filters."""
+        with self.engine.connect() as conn:
+            stmt = select(self.model_versions)
+
+            if name:
+                stmt = stmt.where(self.model_versions.c.name == name)
+            if stage:
+                stmt = stmt.where(self.model_versions.c.stage == stage)
+
+            stmt = stmt.order_by(self.model_versions.c.created_at.desc())
+            rows = conn.execute(stmt).fetchall()
+            return [self._row_to_model_version(row) for row in rows]
+
+    def list_registered_models(self) -> list[str]:
+        """List all unique model names in the registry."""
+        with self.engine.connect() as conn:
+            stmt = select(self.model_versions.c.name).distinct()
+            rows = conn.execute(stmt).fetchall()
+            return [row[0] for row in rows]
+
+    def promote_model(self, name: str, version: str, to_stage: str) -> bool:
+        """Promote a model version to a different stage."""
+        now = datetime.now(UTC).isoformat()
+        with self.engine.connect() as conn:
+            stmt = (
+                update(self.model_versions)
+                .where(
+                    (self.model_versions.c.name == name) & (self.model_versions.c.version == version),
+                )
+                .values(stage=to_stage, updated_at=now)
+            )
+            result = conn.execute(stmt)
+            conn.commit()
+            return result.rowcount > 0
+
+    def delete_model_version(self, name: str, version: str) -> bool:
+        """Delete a model version from the registry."""
+        with self.engine.connect() as conn:
+            stmt = delete(self.model_versions).where(
+                (self.model_versions.c.name == name) & (self.model_versions.c.version == version),
+            )
+            result = conn.execute(stmt)
+            conn.commit()
+            return result.rowcount > 0
+
+    def get_latest_model_version(self, name: str, stage: str | None = None) -> dict | None:
+        """Get the latest version of a model, optionally filtered by stage."""
+        with self.engine.connect() as conn:
+            stmt = select(self.model_versions).where(self.model_versions.c.name == name)
+
+            if stage:
+                stmt = stmt.where(self.model_versions.c.stage == stage)
+
+            stmt = stmt.order_by(self.model_versions.c.created_at.desc()).limit(1)
+            row = conn.execute(stmt).fetchone()
+            if row:
+                return self._row_to_model_version(row)
+            return None
+
+    # ========== Pipeline Template Methods ==========
+
+    def save_pipeline_template(
+        self,
+        template_id: str,
+        name: str,
+        definition: dict,
+        description: str = "",
+        tags: list[str] | None = None,
+        author: str | None = None,
+        category: str | None = None,
+        is_public: bool = False,
+    ) -> None:
+        """Save a pipeline template."""
+        now = datetime.now(UTC).isoformat()
+        with self.engine.connect() as conn:
+            stmt = select(self.pipeline_templates).where(
+                self.pipeline_templates.c.template_id == template_id,
+            )
+            existing = conn.execute(stmt).fetchone()
+
+            values = {
+                "template_id": template_id,
+                "name": name,
+                "description": description,
+                "definition": json.dumps(definition),
+                "tags": json.dumps(tags or []),
+                "author": author,
+                "category": category,
+                "is_public": 1 if is_public else 0,
+                "updated_at": now,
+            }
+
+            if existing:
+                conn.execute(
+                    update(self.pipeline_templates)
+                    .where(self.pipeline_templates.c.template_id == template_id)
+                    .values(**values),
+                )
+            else:
+                conn.execute(insert(self.pipeline_templates).values(**values))
+
+            conn.commit()
+
+    def get_pipeline_template(self, template_id: str) -> dict | None:
+        """Get a pipeline template by ID."""
+        with self.engine.connect() as conn:
+            stmt = select(self.pipeline_templates).where(
+                self.pipeline_templates.c.template_id == template_id,
+            )
+            row = conn.execute(stmt).fetchone()
+            if row:
+                return self._row_to_template(row)
+            return None
+
+    def _row_to_template(self, row) -> dict:
+        """Convert a row to a template dict."""
+        return {
+            "template_id": row.template_id,
+            "name": row.name,
+            "description": row.description,
+            "definition": json.loads(row.definition) if row.definition else {},
+            "tags": json.loads(row.tags) if row.tags else [],
+            "author": row.author,
+            "category": row.category,
+            "is_public": bool(row.is_public),
+            "created_at": row.created_at,
+            "updated_at": row.updated_at,
+        }
+
+    def list_pipeline_templates(
+        self,
+        category: str | None = None,
+        include_public: bool = True,
+        author: str | None = None,
+    ) -> list[dict]:
+        """List pipeline templates with optional filters."""
+        with self.engine.connect() as conn:
+            stmt = select(self.pipeline_templates)
+
+            if category:
+                stmt = stmt.where(self.pipeline_templates.c.category == category)
+            if author:
+                stmt = stmt.where(self.pipeline_templates.c.author == author)
+            if not include_public:
+                stmt = stmt.where(self.pipeline_templates.c.is_public == 0)
+
+            stmt = stmt.order_by(self.pipeline_templates.c.created_at.desc())
+            rows = conn.execute(stmt).fetchall()
+            return [self._row_to_template(row) for row in rows]
+
+    def delete_pipeline_template(self, template_id: str) -> bool:
+        """Delete a pipeline template."""
+        with self.engine.connect() as conn:
+            stmt = delete(self.pipeline_templates).where(
+                self.pipeline_templates.c.template_id == template_id,
+            )
+            result = conn.execute(stmt)
+            conn.commit()
+            return result.rowcount > 0

flowyml/tracking/experiment.py

@@ -42,8 +42,12 @@ class Experiment:
         description: str = "",
         tags: dict[str, str] | None = None,
         parameters: dict[str, Any] | None = None,
-        experiment_dir: str = ".flowyml/experiments",
+        experiment_dir: str | None = None,
+        metadata_store: Any = None,
     ):
+        from flowyml.utils.config import get_config
+        from flowyml.storage.sql import SQLMetadataStore
+
         self.name = name
         self.description = description
         self.config = ExperimentConfig(
@@ -54,13 +58,12 @@ class Experiment:
         )
 
         # Storage
-        self.experiment_dir = Path(experiment_dir) / name
+        base_dir = Path(experiment_dir) if experiment_dir else get_config().experiments_dir
+        self.experiment_dir = base_dir / name
         self.experiment_dir.mkdir(parents=True, exist_ok=True)
 
         # Metadata store for UI
-        from flowyml.storage.metadata import SQLiteMetadataStore
-
-        self.metadata_store = SQLiteMetadataStore()
+        self.metadata_store = metadata_store or SQLMetadataStore(db_path=str(get_config().metadata_db))
 
         # Save experiment to DB
         self.metadata_store.save_experiment(

flowyml/ui/backend/Dockerfile

@@ -1,31 +1,102 @@
-FROM python:3.10-slim
+# =============================================================================
+# FlowyML Backend - Multi-Stage Optimized Dockerfile
+# =============================================================================
+# This Dockerfile uses multi-stage builds to create a minimal, secure, and
+# optimized production image. Build artifacts and dev tools are excluded.
+# =============================================================================
 
-WORKDIR /app
+# -----------------------------------------------------------------------------
+# Stage 1: Builder - Install dependencies and build the application
+# -----------------------------------------------------------------------------
+FROM python:3.11-slim AS builder
+
+# Set environment variables for Python
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    POETRY_VERSION=1.8.2 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_VIRTUALENVS_CREATE=false \
+    POETRY_NO_INTERACTION=1
+
+# Add Poetry to PATH
+ENV PATH="$POETRY_HOME/bin:$PATH"
 
-# Install system dependencies
-RUN apt-get update && apt-get install -y \
+WORKDIR /build
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     curl \
+    libpq-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Install poetry
-RUN curl -sSL https://install.python-poetry.org | python3 -
-ENV PATH="/root/.local/bin:$PATH"
+# Install Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 - \
+    && chmod +x $POETRY_HOME/bin/poetry
 
-# Copy project files
+# Copy only dependency files first (better layer caching)
 COPY pyproject.toml poetry.lock ./
+
+# Install dependencies only (no dev dependencies for production)
+RUN poetry install --only main --no-root --no-directory
+
+# Copy application source
 COPY README.md ./
 COPY flowyml ./flowyml
 
-# Install dependencies
-RUN poetry config virtualenvs.create false \
-    && poetry install --no-interaction --no-ansi --no-root
+# Install the application package
+RUN poetry install --only main
+
+# -----------------------------------------------------------------------------
+# Stage 2: Runtime - Minimal production image
+# -----------------------------------------------------------------------------
+FROM python:3.11-slim AS runtime
 
-# Install the package itself
-RUN pip install .
+# Security: Run as non-root user
+RUN groupadd --gid 1000 flowyml \
+    && useradd --uid 1000 --gid 1000 --shell /bin/bash --create-home flowyml
+
+# Set environment variables
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH="/app" \
+    # Application defaults
+    FLOWYML_ENV=production \
+    FLOWYML_HOST=0.0.0.0 \
+    FLOWYML_PORT=8080
+
+WORKDIR /app
+
+# Install runtime dependencies only (no build tools)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libpq5 \
+    curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Copy installed packages from builder
+COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+
+# Copy application code
+COPY --from=builder /build/flowyml ./flowyml
+COPY --from=builder /build/README.md ./
+
+# Create data directories with correct permissions
+RUN mkdir -p /app/data /app/artifacts \
+    && chown -R flowyml:flowyml /app
+
+# Switch to non-root user
+USER flowyml
 
 # Expose port
-EXPOSE 8000
+EXPOSE 8080
+
+# Health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
+    CMD curl -f http://localhost:8080/api/health || exit 1
 
-# Run the application
-CMD ["uvicorn", "flowyml.ui.backend.main:app", "--host", "0.0.0.0", "--port", "8000"]
+# Run the application with uvicorn
+CMD ["uvicorn", "flowyml.ui.backend.main:app", "--host", "0.0.0.0", "--port", "8080"]

flowyml/ui/backend/auth.py

@@ -23,8 +23,18 @@ class TokenManager:
     def _load_tokens(self) -> None:
         """Load tokens from file."""
         if self.tokens_file.exists():
-            with open(self.tokens_file) as f:
-                self.tokens = json.load(f)
+            try:
+                with open(self.tokens_file) as f:
+                    content = f.read().strip()
+                    if not content:
+                        self.tokens = {}
+                        self._save_tokens()
+                    else:
+                        self.tokens = json.loads(content)
+            except (json.JSONDecodeError, OSError) as e:
+                print(f"Warning: Failed to load tokens from {self.tokens_file}: {e}")
+                self.tokens = {}
+                self._save_tokens()
         else:
             self.tokens = {}
             self._save_tokens()