flowyml 1.1.0__py3-none-any.whl

flowyml/tracking/__init__.py

@@ -0,0 +1 @@
+"""Experiment tracking and run management."""

flowyml/tracking/experiment.py

@@ -0,0 +1,211 @@
+"""Experiment Tracking - Track ML experiments and compare results."""
+
+import json
+from pathlib import Path
+from typing import Any
+from datetime import datetime
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ExperimentConfig:
+    """Configuration for an experiment."""
+
+    name: str
+    description: str
+    tags: dict[str, str] = field(default_factory=dict)
+    parameters: dict[str, Any] = field(default_factory=dict)
+    created_at: datetime = field(default_factory=datetime.now)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "name": self.name,
+            "description": self.description,
+            "tags": self.tags,
+            "parameters": self.parameters,
+            "created_at": self.created_at.isoformat(),
+        }
+
+
+class Experiment:
+    """Experiment for tracking multiple pipeline runs.
+
+    Example:
+        >>> exp = Experiment(name="baseline_training", description="Baseline model training")
+        >>> exp.log_run(run_id, metrics={"accuracy": 0.95})
+    """
+
+    def __init__(
+        self,
+        name: str,
+        description: str = "",
+        tags: dict[str, str] | None = None,
+        parameters: dict[str, Any] | None = None,
+        experiment_dir: str = ".flowyml/experiments",
+    ):
+        self.name = name
+        self.description = description
+        self.config = ExperimentConfig(
+            name=name,
+            description=description,
+            tags=tags or {},
+            parameters=parameters or {},
+        )
+
+        # Storage
+        self.experiment_dir = Path(experiment_dir) / name
+        self.experiment_dir.mkdir(parents=True, exist_ok=True)
+
+        # Metadata store for UI
+        from flowyml.storage.metadata import SQLiteMetadataStore
+
+        self.metadata_store = SQLiteMetadataStore()
+
+        # Save experiment to DB
+        self.metadata_store.save_experiment(
+            experiment_id=self.name,
+            name=self.name,
+            description=self.description,
+            tags=tags,
+        )
+
+        # Runs
+        self.runs: list[str] = []  # run IDs
+        self.run_metrics: dict[str, dict[str, Any]] = {}
+
+        self._load_experiment()
+
+    def log_run(
+        self,
+        run_id: str,
+        metrics: dict[str, Any] | None = None,
+        parameters: dict[str, Any] | None = None,
+    ) -> None:
+        """Log a pipeline run to this experiment.
+
+        Args:
+            run_id: Run identifier
+            metrics: Metrics from the run
+            parameters: Parameters used in the run
+        """
+        if run_id not in self.runs:
+            self.runs.append(run_id)
+
+        self.run_metrics[run_id] = {
+            "metrics": metrics or {},
+            "parameters": parameters or {},
+            "timestamp": datetime.now().isoformat(),
+        }
+
+        self._save_experiment()
+
+        # Log to DB
+        self.metadata_store.log_experiment_run(
+            experiment_id=self.name,
+            run_id=run_id,
+            metrics=metrics,
+            parameters=parameters,
+        )
+
+    def get_run_metrics(self, run_id: str) -> dict[str, Any] | None:
+        """Get metrics for a specific run."""
+        return self.run_metrics.get(run_id)
+
+    def list_runs(self) -> list[str]:
+        """List all runs in this experiment."""
+        return self.runs
+
+    def compare_runs(
+        self,
+        run_ids: list[str] | None = None,
+        metric: str | None = None,
+    ) -> dict[str, Any]:
+        """Compare runs in this experiment.
+
+        Args:
+            run_ids: Specific runs to compare (or all if None)
+            metric: Specific metric to compare (or all if None)
+
+        Returns:
+            Comparison results
+        """
+        runs_to_compare = run_ids or self.runs
+
+        comparison = {
+            "experiment": self.name,
+            "runs": {},
+        }
+
+        for run_id in runs_to_compare:
+            if run_id not in self.run_metrics:
+                continue
+
+            run_data = self.run_metrics[run_id]
+            metrics = run_data.get("metrics", {})
+
+            if metric:
+                comparison["runs"][run_id] = {metric: metrics.get(metric)}
+            else:
+                comparison["runs"][run_id] = metrics
+
+        return comparison
+
+    def get_best_run(self, metric: str, maximize: bool = True) -> str | None:
+        """Get the best run based on a metric.
+
+        Args:
+            metric: Metric to optimize
+            maximize: Whether to maximize (True) or minimize (False)
+
+        Returns:
+            Best run ID
+        """
+        best_run = None
+        best_value = float("-inf") if maximize else float("inf")
+
+        for run_id, run_data in self.run_metrics.items():
+            metrics = run_data.get("metrics", {})
+            value = metrics.get(metric)
+
+            if value is None:
+                continue
+
+            if maximize and value > best_value or not maximize and value < best_value:
+                best_value = value
+                best_run = run_id
+
+        return best_run
+
+    def _save_experiment(self) -> None:
+        """Save experiment data to disk."""
+        experiment_file = self.experiment_dir / "experiment.json"
+
+        data = {
+            "config": self.config.to_dict(),
+            "runs": self.runs,
+            "run_metrics": self.run_metrics,
+        }
+
+        with open(experiment_file, "w") as f:
+            json.dump(data, f, indent=2)
+
+    def _load_experiment(self) -> None:
+        """Load experiment data from disk."""
+        experiment_file = self.experiment_dir / "experiment.json"
+
+        if not experiment_file.exists():
+            return
+
+        try:
+            with open(experiment_file) as f:
+                data = json.load(f)
+
+            self.runs = data.get("runs", [])
+            self.run_metrics = data.get("run_metrics", {})
+
+        except Exception:
+            pass
+
+    def __repr__(self) -> str:
+        return f"Experiment(name='{self.name}', runs={len(self.runs)})"

flowyml/tracking/leaderboard.py

@@ -0,0 +1,191 @@
+"""Model comparison and leaderboard functionality."""
+
+from typing import Any
+from dataclasses import dataclass
+from flowyml.storage.metadata import SQLiteMetadataStore
+
+
+@dataclass
+class ModelScore:
+    """A model's score on a leaderboard."""
+
+    model_name: str
+    run_id: str
+    metric_value: float
+    timestamp: str
+    metadata: dict[str, Any]
+
+
+class ModelLeaderboard:
+    """Compare and rank models based on metrics.
+
+    Examples:
+        >>> leaderboard = ModelLeaderboard(metric="accuracy")
+        >>> # Add model scores
+        >>> leaderboard.add_score(model_name="bert-base", run_id="run_123", score=0.92)
+        >>> # Get top models
+        >>> top_models = leaderboard.get_top(n=5)
+        >>> leaderboard.display()
+    """
+
+    def __init__(
+        self,
+        metric: str,
+        higher_is_better: bool = True,
+        metadata_store: SQLiteMetadataStore | None = None,
+    ):
+        """Args:
+        metric: Metric to compare on
+        higher_is_better: Whether higher values are better
+        metadata_store: Optional metadata store.
+        """
+        self.metric = metric
+        self.higher_is_better = higher_is_better
+        self.metadata_store = metadata_store or SQLiteMetadataStore()
+
+    def add_score(
+        self,
+        model_name: str,
+        run_id: str,
+        score: float,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Add a model score to the leaderboard."""
+        # Save to metadata store
+        self.metadata_store.save_metric(
+            run_id=run_id,
+            name=f"leaderboard_{self.metric}",
+            value=score,
+        )
+
+        # Also save model info as artifact
+        self.metadata_store.save_artifact(
+            artifact_id=f"{run_id}_leaderboard",
+            metadata={
+                "name": model_name,
+                "type": "leaderboard_entry",
+                "run_id": run_id,
+                "value": str(score),
+                "metric": self.metric,
+                "metadata": metadata or {},
+            },
+        )
+
+    def get_top(self, n: int = 10) -> list[ModelScore]:
+        """Get top N models."""
+        # Query all leaderboard entries
+        artifacts = self.metadata_store.list_assets(type="leaderboard_entry")
+
+        # Filter by metric
+        entries = [a for a in artifacts if a.get("metric") == self.metric]
+
+        # Sort
+        entries.sort(
+            key=lambda x: float(x.get("value", 0)),
+            reverse=self.higher_is_better,
+        )
+
+        # Convert to ModelScore
+        scores = []
+        for entry in entries[:n]:
+            scores.append(
+                ModelScore(
+                    model_name=entry.get("name"),
+                    run_id=entry.get("run_id"),
+                    metric_value=float(entry.get("value", 0)),
+                    timestamp=entry.get("created_at", ""),
+                    metadata=entry.get("metadata", {}),
+                ),
+            )
+
+        return scores
+
+    def display(self, n: int = 10) -> None:
+        """Display the leaderboard."""
+        scores = self.get_top(n)
+
+        if not scores:
+            return
+
+        for _i, _score in enumerate(scores, 1):
+            pass
+
+    def compare_models(self, run_ids: list[str]) -> dict[str, Any]:
+        """Compare specific models side-by-side."""
+        comparison = {
+            "metric": self.metric,
+            "models": [],
+        }
+
+        for run_id in run_ids:
+            # Get metrics for this run
+            metrics = self.metadata_store.get_metrics(run_id)
+            metric_val = next(
+                (m["value"] for m in metrics if m["name"] == self.metric),
+                None,
+            )
+
+            # Get run details
+            run = self.metadata_store.load_run(run_id)
+
+            comparison["models"].append(
+                {
+                    "run_id": run_id,
+                    "metric_value": metric_val,
+                    "pipeline": run.get("pipeline_name") if run else None,
+                    "status": run.get("status") if run else None,
+                },
+            )
+
+        # Sort by metric value
+        comparison["models"].sort(
+            key=lambda x: x.get("metric_value", 0) or 0,
+            reverse=self.higher_is_better,
+        )
+
+        return comparison
+
+
+def compare_runs(
+    run_ids: list[str],
+    metrics: list[str] | None = None,
+) -> dict[str, Any]:
+    """Compare multiple runs across metrics.
+
+    Args:
+        run_ids: List of run IDs to compare
+        metrics: Optional list of specific metrics to compare
+
+    Returns:
+        Comparison dictionary
+    """
+    store = SQLiteMetadataStore()
+
+    comparison = {
+        "runs": {},
+        "metrics": {},
+    }
+
+    # Collect all metrics across runs
+    all_metric_names = set()
+
+    for run_id in run_ids:
+        run_metrics = store.get_metrics(run_id)
+        run_data = {
+            "metrics": {m["name"]: m["value"] for m in run_metrics},
+        }
+
+        all_metric_names.update(run_data["metrics"].keys())
+        comparison["runs"][run_id] = run_data
+
+    # Filter metrics if specified
+    if metrics:
+        all_metric_names = set(metrics)
+
+    # Organize by metric
+    for metric_name in all_metric_names:
+        comparison["metrics"][metric_name] = {
+            run_id: comparison["runs"][run_id]["metrics"].get(metric_name) for run_id in run_ids
+        }
+
+    return comparison

flowyml/tracking/runs.py

@@ -0,0 +1,145 @@
+"""Run Management - Track individual pipeline runs."""
+
+import json
+from pathlib import Path
+from typing import Any, Optional
+from datetime import datetime
+from dataclasses import dataclass, field
+
+
+@dataclass
+class RunMetadata:
+    """Metadata for a pipeline run."""
+
+    run_id: str
+    pipeline_name: str
+    started_at: datetime
+    ended_at: datetime | None = None
+    status: str = "running"  # running, success, failed
+    parameters: dict[str, Any] = field(default_factory=dict)
+    metrics: dict[str, Any] = field(default_factory=dict)
+    tags: dict[str, str] = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to dictionary."""
+        return {
+            "run_id": self.run_id,
+            "pipeline_name": self.pipeline_name,
+            "started_at": self.started_at.isoformat(),
+            "ended_at": self.ended_at.isoformat() if self.ended_at else None,
+            "status": self.status,
+            "parameters": self.parameters,
+            "metrics": self.metrics,
+            "tags": self.tags,
+        }
+
+
+class Run:
+    """Represents a single pipeline execution run.
+
+    Example:
+        >>> run = Run(run_id="training_20240115_120000", pipeline_name="training_pipeline")
+        >>> run.log_metric("accuracy", 0.95)
+        >>> run.complete(status="success")
+    """
+
+    def __init__(
+        self,
+        run_id: str,
+        pipeline_name: str,
+        parameters: dict[str, Any] | None = None,
+        tags: dict[str, str] | None = None,
+        runs_dir: str = ".flowyml/runs",
+    ):
+        self.run_id = run_id
+        self.pipeline_name = pipeline_name
+
+        self.metadata = RunMetadata(
+            run_id=run_id,
+            pipeline_name=pipeline_name,
+            started_at=datetime.now(),
+            parameters=parameters or {},
+            tags=tags or {},
+        )
+
+        # Storage
+        self.runs_dir = Path(runs_dir)
+        self.runs_dir.mkdir(parents=True, exist_ok=True)
+
+        self._save()
+
+    def log_metric(self, name: str, value: Any) -> None:
+        """Log a metric for this run."""
+        self.metadata.metrics[name] = value
+        self._save()
+
+    def log_metrics(self, metrics: dict[str, Any]) -> None:
+        """Log multiple metrics."""
+        self.metadata.metrics.update(metrics)
+        self._save()
+
+    def log_parameter(self, name: str, value: Any) -> None:
+        """Log a parameter."""
+        self.metadata.parameters[name] = value
+        self._save()
+
+    def add_tag(self, key: str, value: str) -> None:
+        """Add a tag to the run."""
+        self.metadata.tags[key] = value
+        self._save()
+
+    def complete(self, status: str = "success") -> None:
+        """Mark run as complete.
+
+        Args:
+            status: Final status ("success" or "failed")
+        """
+        self.metadata.status = status
+        self.metadata.ended_at = datetime.now()
+        self._save()
+
+    def get_duration(self) -> float | None:
+        """Get run duration in seconds."""
+        if self.metadata.ended_at:
+            return (self.metadata.ended_at - self.metadata.started_at).total_seconds()
+        return None
+
+    def _save(self) -> None:
+        """Save run metadata to disk."""
+        run_file = self.runs_dir / f"{self.run_id}.json"
+
+        with open(run_file, "w") as f:
+            json.dump(self.metadata.to_dict(), f, indent=2)
+
+    @classmethod
+    def load(cls, run_id: str, runs_dir: str = ".flowyml/runs") -> Optional["Run"]:
+        """Load a run from disk."""
+        run_file = Path(runs_dir) / f"{run_id}.json"
+
+        if not run_file.exists():
+            return None
+
+        try:
+            with open(run_file) as f:
+                data = json.load(f)
+
+            run = cls(
+                run_id=data["run_id"],
+                pipeline_name=data["pipeline_name"],
+                parameters=data.get("parameters", {}),
+                tags=data.get("tags", {}),
+                runs_dir=runs_dir,
+            )
+
+            run.metadata.status = data["status"]
+            run.metadata.metrics = data.get("metrics", {})
+            if data.get("ended_at"):
+                run.metadata.ended_at = datetime.fromisoformat(data["ended_at"])
+
+            return run
+
+        except Exception:
+            return None
+
+    def __repr__(self) -> str:
+        return f"Run(id='{self.run_id}', status='{self.metadata.status}')"

flowyml/ui/__init__.py

@@ -0,0 +1,15 @@
+"""flowyml UI module - Real-time UI for pipeline monitoring."""
+
+from flowyml.ui.utils import (
+    is_ui_running,
+    get_ui_url,
+    get_run_url,
+    get_pipeline_url,
+)
+
+__all__ = [
+    "is_ui_running",
+    "get_ui_url",
+    "get_run_url",
+    "get_pipeline_url",
+]

flowyml/ui/backend/Dockerfile

@@ -0,0 +1,31 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install poetry
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="/root/.local/bin:$PATH"
+
+# Copy project files
+COPY pyproject.toml poetry.lock ./
+COPY README.md ./
+COPY flowyml ./flowyml
+
+# Install dependencies
+RUN poetry config virtualenvs.create false \
+    && poetry install --no-interaction --no-ansi --no-root
+
+# Install the package itself
+RUN pip install .
+
+# Expose port
+EXPOSE 8000
+
+# Run the application
+CMD ["uvicorn", "flowyml.ui.backend.main:app", "--host", "0.0.0.0", "--port", "8000"]