pipelab 0.1.0__py3-none-any.whl

pipelab/__init__.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""pipelab — ML Pipeline Management Platform."""
+__version__ = "0.1.0"

pipelab/api/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+

pipelab/api/app.py

@@ -0,0 +1,174 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""FastAPI application factory and service registry."""
+from __future__ import annotations
+
+import os
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import Any
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from starlette.exceptions import HTTPException
+
+
+class SpaStaticFiles(StaticFiles):
+    """StaticFiles que faz fallback para index.html em rotas do SPA.
+
+    Paths sem extensão (ex.: /projects, /dashboard) devolvem index.html para
+    que o React Router cuide do roteamento no cliente. Assets com extensão
+    ausentes continuam retornando 404 apropriado.
+    """
+
+    async def get_response(self, path: str, scope):
+        try:
+            return await super().get_response(path, scope)
+        except HTTPException as exc:
+            if exc.status_code != 404:
+                raise
+            last_segment = path.rsplit("/", 1)[-1]
+            if "." in last_segment:
+                raise
+            return await super().get_response("index.html", scope)
+
+from pipelab.infrastructure.mlflow_client import configure as configure_mlflow
+from pipelab.plugins.plugin import PipelabPlugin
+from pipelab.discovery import discover_projects, discover_services, ProjectConfig, DiscoveredService
+
+# Service singletons
+from pipelab.services.mlflow_project_service import MlflowProjectService
+from pipelab.services.mlflow_dataset_service import MlflowDatasetService
+from pipelab.services.mlflow_experiment_service import MlflowExperimentService
+from pipelab.services.mlflow_model_registry_service import MlflowModelRegistryService
+from pipelab.services.mlflow_deployment_service import MlflowDeploymentService
+from pipelab.services.mlflow_monitoring_service import MlflowMonitoringService
+
+_services: dict[str, Any] = {}
+_plugins: list[PipelabPlugin] = []
+
+# Discovered projects and their services
+_projects: list[ProjectConfig] = []
+_discovered_services: dict[str, list[DiscoveredService]] = {}  # project_name -> services
+
+
+def get_service(name: str) -> Any:
+    return _services[name]
+
+
+def get_projects() -> list[ProjectConfig]:
+    return _projects
+
+
+def get_discovered_services(project_name: str) -> list[DiscoveredService]:
+    return _discovered_services.get(project_name, [])
+
+
+def get_project(project_name: str) -> ProjectConfig | None:
+    for p in _projects:
+        if p.name == project_name:
+            return p
+    return None
+
+
+def register_plugin(plugin: PipelabPlugin) -> None:
+    _plugins.append(plugin)
+
+
+def get_workdir() -> str:
+    """Return the active workdir path."""
+    return os.environ.get("PIPELAB_WORKDIR", os.getcwd())
+
+
+def run_discovery() -> None:
+    """Re-scan the workdir for projects and services."""
+    global _projects, _discovered_services
+    workdir = get_workdir()
+    _projects = discover_projects(workdir)
+    _discovered_services = {}
+    for project in _projects:
+        _discovered_services[project.name] = discover_services(project)
+
+
+def refresh_discovery() -> None:
+    """Alias for run_discovery (backward compat)."""
+    run_discovery()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Initialize MLflow — resolve relative tracking URI against workdir
+    tracking_uri = os.environ.get("MLFLOW_TRACKING_URI", "mlruns")
+    workdir = get_workdir()
+    if not tracking_uri.startswith(("/", "http", "sqlite", "file:")):
+        # Relative path — resolve against workdir
+        tracking_uri = str(Path(workdir) / tracking_uri)
+    configure_mlflow(tracking_uri)
+
+    # Register internal services
+    _services["projects"] = MlflowProjectService()
+    _services["datasets"] = MlflowDatasetService()
+    _services["experiments"] = MlflowExperimentService()
+    _services["models"] = MlflowModelRegistryService()
+    _services["deployments"] = MlflowDeploymentService()
+    _services["monitoring"] = MlflowMonitoringService()
+
+    # Discover projects and user pipeline services
+    refresh_discovery()
+
+    # Register plugins
+    for plugin in _plugins:
+        plugin.on_register()
+        overrides = plugin.get_services()
+        if overrides:
+            _services.update(overrides)
+
+    yield
+
+
+def create_app() -> FastAPI:
+    app = FastAPI(
+        title="PipeLab",
+        description="ML Pipeline Management Platform",
+        version="0.1.0",
+        lifespan=lifespan,
+    )
+
+    # CORS
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # API routers
+    from pipelab.api.routers import (
+        projects, datasets, experiments, models,
+        pipelines, deployments, monitoring, settings,
+    )
+    app.include_router(projects.router, prefix="/api/v1/projects", tags=["Projects"])
+    app.include_router(datasets.router, prefix="/api/v1/datasets", tags=["Datasets"])
+    app.include_router(experiments.router, prefix="/api/v1/experiments", tags=["Experiments"])
+    app.include_router(models.router, prefix="/api/v1/models", tags=["Models"])
+    app.include_router(pipelines.router, prefix="/api/v1/pipelines", tags=["Pipelines"])
+    app.include_router(deployments.router, prefix="/api/v1/deployments", tags=["Deployments"])
+    app.include_router(monitoring.router, prefix="/api/v1/monitoring", tags=["Monitoring"])
+    app.include_router(settings.router, prefix="/api/v1/settings", tags=["Settings"])
+
+    # Plugin routes
+    for plugin in _plugins:
+        routes = plugin.get_routes()
+        if routes:
+            for r in routes:
+                app.include_router(r, prefix=f"/api/v1/plugins/{plugin.name}")
+
+    # Serve React static files
+    static_dir = Path(__file__).resolve().parent.parent / "static"
+    if static_dir.is_dir():
+        app.mount("/", SpaStaticFiles(directory=str(static_dir), html=True), name="static")
+
+    return app

pipelab/api/routers/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+

pipelab/api/routers/datasets.py

@@ -0,0 +1,79 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""Datasets API router — CRUD, versioning, splits."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+
+from pipelab.api.app import get_service
+from pipelab.api.schemas import (
+    DatasetCreate, DatasetOut,
+    DatasetVersionCreate, DatasetVersionOut,
+    DataSplitCreate, DataSplitOut,
+)
+
+router = APIRouter()
+
+
+@router.get("/", response_model=list[DatasetOut])
+async def list_datasets(project_id: str | None = None):
+    svc = get_service("datasets")
+    return [DatasetOut(**{k: getattr(d, k) for k in DatasetOut.model_fields})
+            for d in svc.list_datasets(project_id)]
+
+
+@router.get("/{dataset_id}", response_model=DatasetOut)
+async def get_dataset(dataset_id: str):
+    svc = get_service("datasets")
+    d = svc.get_dataset(dataset_id)
+    if d is None:
+        raise HTTPException(404, "Dataset not found")
+    return DatasetOut(**{k: getattr(d, k) for k in DatasetOut.model_fields})
+
+
+@router.post("/", response_model=DatasetOut, status_code=201)
+async def create_dataset(body: DatasetCreate):
+    svc = get_service("datasets")
+    d = svc.create_dataset(body.label, body.description, body.project_id)
+    return DatasetOut(**{k: getattr(d, k) for k in DatasetOut.model_fields})
+
+
+@router.delete("/{dataset_id}")
+async def delete_dataset(dataset_id: str):
+    svc = get_service("datasets")
+    if not svc.delete_dataset(dataset_id):
+        raise HTTPException(404, "Dataset not found")
+    return {"ok": True}
+
+
+# --- Versions -----------------------------------------------------------------
+
+@router.get("/{dataset_id}/versions", response_model=list[DatasetVersionOut])
+async def list_versions(dataset_id: str):
+    svc = get_service("datasets")
+    return [DatasetVersionOut(**{k: getattr(v, k) for k in DatasetVersionOut.model_fields})
+            for v in svc.list_versions(dataset_id)]
+
+
+@router.post("/{dataset_id}/versions", response_model=DatasetVersionOut, status_code=201)
+async def create_version(dataset_id: str, body: DatasetVersionCreate):
+    svc = get_service("datasets")
+    v = svc.create_version(dataset_id, body.version, body.uri, body.metadata)
+    return DatasetVersionOut(**{k: getattr(v, k) for k in DatasetVersionOut.model_fields})
+
+
+# --- Splits -------------------------------------------------------------------
+
+@router.get("/{dataset_id}/splits", response_model=list[DataSplitOut])
+async def list_splits(dataset_id: str):
+    svc = get_service("datasets")
+    return [DataSplitOut(**{k: getattr(s, k) for k in DataSplitOut.model_fields})
+            for s in svc.list_splits(dataset_id)]
+
+
+@router.post("/{dataset_id}/splits", response_model=DataSplitOut, status_code=201)
+async def create_split(dataset_id: str, body: DataSplitCreate):
+    svc = get_service("datasets")
+    s = svc.create_split(dataset_id, body.name, body.method, body.params)
+    return DataSplitOut(**{k: getattr(s, k) for k in DataSplitOut.model_fields})

pipelab/api/routers/deployments.py

@@ -0,0 +1,175 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""Deployments API router — deploy/undeploy models via discovered DeployService providers."""
+from __future__ import annotations
+
+import time
+
+from fastapi import APIRouter, HTTPException
+
+from pydantic import BaseModel
+
+from pipelab.api.app import get_project, get_discovered_services, refresh_discovery
+from pipelab.api.schemas import DeployRequest, DeploymentOut
+from pipelab.discovery import instantiate_service
+from pipelab.pipeline import PipelineContext
+from pipelab.entities.services import DeployService
+
+router = APIRouter()
+
+# In-memory deployment registry (production system would use persistent storage)
+_active_deployments: dict[str, dict] = {}
+
+
+@router.get("/", response_model=list[DeploymentOut])
+async def list_deployments():
+    """List all active deployments."""
+    return [DeploymentOut(**d) for d in _active_deployments.values()]
+
+
+@router.post("/", response_model=DeploymentOut, status_code=201)
+async def deploy_model(body: DeployRequest):
+    """Deploy a model using a discovered DeployService provider."""
+    proj = get_project(body.project_name)
+    if proj is None:
+        raise HTTPException(404, f"Project '{body.project_name}' not found")
+
+    # Instantiate the selected deploy service
+    try:
+        svc = instantiate_service(proj, body.deploy_service)
+    except ValueError as exc:
+        raise HTTPException(404, str(exc))
+
+    if not isinstance(svc, DeployService):
+        raise HTTPException(400, f"'{body.deploy_service}' is not a DeployService")
+
+    # Build context for the deploy service
+    ctx = PipelineContext(
+        project_name=proj.name,
+        project_path=proj.path,
+        config={
+            "model_name": body.model_name,
+            "model_version": body.version,
+            "alias": body.alias,
+            "port": body.port,
+        },
+    )
+
+    try:
+        result = svc.execute(ctx)
+    except Exception as exc:
+        raise HTTPException(500, f"Deploy failed: {exc}")
+
+    key = f"{body.model_name}-{body.version}"
+    dep = {
+        "model_name": body.model_name,
+        "version": body.version,
+        "alias": body.alias,
+        "deploy_service": body.deploy_service,
+        "endpoint_uri": result.get("endpoint_uri") if isinstance(result, dict) else None,
+        "status": "active",
+        "created_at": time.time(),
+        "port": body.port,
+        "pid": result.get("pid") if isinstance(result, dict) else None,
+    }
+    _active_deployments[key] = dep
+
+    # Record deployment in project YAML
+    from pipelab.discovery import load_project_config, save_project_config
+    from pathlib import Path
+    yaml_path = Path(proj.path) / "pipelab.yaml"
+    fresh_proj = load_project_config(yaml_path)
+    fresh_proj.deployments.append({
+        "model_name": body.model_name,
+        "version": body.version,
+        "alias": body.alias,
+        "deploy_service": body.deploy_service,
+        "port": body.port,
+    })
+    save_project_config(fresh_proj)
+    refresh_discovery()
+
+    return DeploymentOut(**dep)
+
+
+@router.delete("/{model_name}/{version}")
+async def undeploy_model(model_name: str, version: str, project: str | None = None):
+    """Undeploy a model — stops the serving process."""
+    key = f"{model_name}-{version}"
+    dep = _active_deployments.get(key)
+    if dep is None:
+        raise HTTPException(404, "Deployment not found")
+
+    deploy_service_name = dep.get("deploy_service", "MLflowServeProvider")
+
+    # Try to call undeploy on the original provider
+    if project:
+        proj = get_project(project)
+        if proj:
+            try:
+                svc = instantiate_service(proj, deploy_service_name)
+                if isinstance(svc, DeployService):
+                    ctx = PipelineContext(
+                        project_name=proj.name,
+                        project_path=proj.path,
+                        config={
+                            "model_name": model_name,
+                            "model_version": version,
+                            "pid": dep.get("pid"),
+                        },
+                    )
+                    svc.undeploy(ctx)
+            except Exception:
+                pass  # best-effort cleanup
+    else:
+        # Fallback: kill process directly if we have a pid
+        pid = dep.get("pid")
+        if pid:
+            import os, signal
+            try:
+                os.kill(pid, signal.SIGTERM)
+            except ProcessLookupError:
+                pass
+
+    dep["status"] = "inactive"
+    _active_deployments[key] = dep
+
+    return {"ok": True, "model_name": model_name, "version": version, "status": "inactive"}
+
+
+class TestEndpointRequest(BaseModel):
+    endpoint_uri: str
+    payload: dict | list | str
+    content_type: str = "application/json"
+
+
+@router.post("/test-endpoint")
+async def test_endpoint(body: TestEndpointRequest):
+    """Proxy a test payload to a deployed model endpoint and return the response."""
+    import httpx
+
+    headers = {"Content-Type": body.content_type}
+    payload = body.payload if isinstance(body.payload, str) else body.payload
+
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            import json
+            data = json.dumps(payload) if not isinstance(payload, str) else payload
+            resp = await client.post(body.endpoint_uri, content=data, headers=headers)
+            try:
+                resp_body = resp.json()
+            except Exception:
+                resp_body = resp.text
+            return {
+                "status_code": resp.status_code,
+                "headers": dict(resp.headers),
+                "body": resp_body,
+                "elapsed_ms": resp.elapsed.total_seconds() * 1000,
+            }
+    except httpx.ConnectError:
+        raise HTTPException(502, f"Cannot connect to {body.endpoint_uri} — is the model server running?")
+    except httpx.TimeoutException:
+        raise HTTPException(504, f"Request to {body.endpoint_uri} timed out (30s)")
+    except Exception as exc:
+        raise HTTPException(500, f"Test request failed: {exc}")

pipelab/api/routers/experiments.py

@@ -0,0 +1,162 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""Experiments API router — experiments, runs, compare."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+from pydantic import BaseModel
+from typing import Any
+
+from pipelab.api.app import get_service
+from pipelab.api.schemas import (
+    ExperimentCreate, ExperimentOut,
+    RunOut, CompareRequest,
+)
+
+router = APIRouter()
+
+
+@router.get("/", response_model=list[ExperimentOut])
+async def list_experiments(project_id: str | None = None):
+    svc = get_service("experiments")
+    return [ExperimentOut(**{k: getattr(e, k) for k in ExperimentOut.model_fields})
+            for e in svc.list_experiments(project_id)]
+
+
+@router.get("/{name}", response_model=ExperimentOut)
+async def get_experiment(name: str):
+    svc = get_service("experiments")
+    e = svc.get_experiment(name)
+    if e is None:
+        raise HTTPException(404, "Experiment not found")
+    return ExperimentOut(**{k: getattr(e, k) for k in ExperimentOut.model_fields})
+
+
+@router.post("/", response_model=ExperimentOut, status_code=201)
+async def create_experiment(body: ExperimentCreate):
+    svc = get_service("experiments")
+    e = svc.create_experiment(body.name, body.project_id, body.description)
+    return ExperimentOut(**{k: getattr(e, k) for k in ExperimentOut.model_fields})
+
+
+@router.delete("/{name}")
+async def delete_experiment(name: str):
+    svc = get_service("experiments")
+    if not svc.delete_experiment(name):
+        raise HTTPException(404, "Experiment not found")
+    return {"ok": True}
+
+
+# --- Runs ---------------------------------------------------------------------
+
+@router.get("/{name}/runs", response_model=list[RunOut])
+async def list_runs(name: str):
+    svc = get_service("experiments")
+    return [RunOut(**{k: getattr(r, k) for k in RunOut.model_fields})
+            for r in svc.list_runs(name)]
+
+
+@router.get("/runs/{run_id}", response_model=RunOut)
+async def get_run(run_id: str):
+    svc = get_service("experiments")
+    r = svc.get_run(run_id)
+    if r is None:
+        raise HTTPException(404, "Run not found")
+    return RunOut(**{k: getattr(r, k) for k in RunOut.model_fields})
+
+
+@router.get("/runs/{run_id}/artifacts")
+async def get_run_artifacts(run_id: str) -> list[dict[str, Any]]:
+    svc = get_service("experiments")
+    return svc.get_run_artifacts(run_id)
+
+
+@router.post("/runs/compare", response_model=list[RunOut])
+async def compare_runs(body: CompareRequest):
+    svc = get_service("experiments")
+    return [RunOut(**{k: getattr(r, k) for k in RunOut.model_fields})
+            for r in svc.compare_runs(body.run_ids)]
+
+
+# --- Benchmark ----------------------------------------------------------------
+
+class BenchmarkRequest(BaseModel):
+    experiment_names: list[str]
+
+
+@router.post("/benchmark")
+async def benchmark_models(body: BenchmarkRequest) -> list[dict[str, Any]]:
+    """
+    Aggregate runs from the given experiments, group by model name
+    (derived from mlflow.runName tag or run name), and compute
+    mean / std of every metric across the runs of each model group.
+    Returns a list of model summaries for comparison.
+    """
+    from statistics import mean, stdev
+
+    svc = get_service("experiments")
+    # Collect all runs across selected experiments
+    all_runs = []
+    for exp_name in body.experiment_names:
+        runs = svc.list_runs(exp_name)
+        all_runs.extend(runs)
+
+    # Group runs by model name (strip k-fold suffixes like _fold_0, _k0, etc.)
+    import re
+    def _model_group_name(run_name: str) -> str:
+        """Strip trailing fold/k-fold index patterns to group CV runs."""
+        # Patterns: _fold_0, _fold0, _k0, _k_0, _cv0, _cv_0, _split_0
+        cleaned = re.sub(r'[_\-](fold|k|cv|split)[_\-]?\d+$', '', run_name, flags=re.IGNORECASE)
+        return cleaned if cleaned else run_name
+
+    model_groups: dict[str, list] = {}
+    for r in all_runs:
+        raw_name = r.name or r.tags.get("mlflow.runName") or r.run_id[:8]
+        model_name = _model_group_name(raw_name)
+        model_groups.setdefault(model_name, []).append(r)
+
+    results = []
+    for model_name, runs in model_groups.items():
+        # Collect all metric keys
+        all_metric_keys = set()
+        for r in runs:
+            all_metric_keys.update(r.metrics.keys())
+
+        # Compute mean/std for each metric
+        metric_summary = {}
+        for key in sorted(all_metric_keys):
+            values = [r.metrics[key] for r in runs if key in r.metrics]
+            if values:
+                m = mean(values)
+                s = stdev(values) if len(values) > 1 else 0.0
+                metric_summary[key] = {
+                    "mean": round(m, 6),
+                    "std": round(s, 6),
+                    "values": [round(v, 6) for v in values],
+                    "count": len(values),
+                }
+
+        # Parameters (take from first run as representative)
+        first_run = runs[0]
+        params = dict(first_run.parameters)
+
+        # Collect run IDs
+        run_ids = [r.run_id for r in runs]
+
+        # Get experiment name from first run
+        experiment_name = first_run.experiment_name
+
+        results.append({
+            "model_name": model_name,
+            "experiment_name": experiment_name,
+            "run_count": len(runs),
+            "run_ids": run_ids,
+            "metrics": metric_summary,
+            "parameters": params,
+            "tags": dict(first_run.tags),
+            "model_uri": first_run.model_uri,
+            "status": first_run.status,
+        })
+
+    return results

pipelab/api/routers/models.py

@@ -0,0 +1,51 @@
+# Copyright (c) 2026 Tarlis Portela <tarlis@tarlis.com.br>
+# Licensed under the Apache License, Version 2.0.
+
+"""Models API router — registry, versions, stage transitions."""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException
+from typing import Any
+
+from pipelab.api.app import get_service
+from pipelab.api.schemas import (
+    ModelRegisterRequest, ModelTransitionRequest,
+    ModelRegistryOut,
+)
+
+router = APIRouter()
+
+
+@router.get("/", response_model=list[dict[str, Any]])
+async def list_models():
+    svc = get_service("models")
+    return svc.list_registered_models()
+
+
+@router.get("/{model_name}/versions", response_model=list[ModelRegistryOut])
+async def get_model_versions(model_name: str):
+    svc = get_service("models")
+    return [ModelRegistryOut(**{k: getattr(v, k) for k in ModelRegistryOut.model_fields})
+            for v in svc.get_model_versions(model_name)]
+
+
+@router.post("/register", response_model=ModelRegistryOut, status_code=201)
+async def register_model(body: ModelRegisterRequest):
+    svc = get_service("models")
+    v = svc.register_model(body.run_id, body.model_name, body.artifact_path)
+    return ModelRegistryOut(**{k: getattr(v, k) for k in ModelRegistryOut.model_fields})
+
+
+@router.put("/{model_name}/versions/{version}/transition", response_model=ModelRegistryOut)
+async def transition_stage(model_name: str, version: str, body: ModelTransitionRequest):
+    svc = get_service("models")
+    v = svc.transition_model_stage(model_name, version, body.stage)
+    return ModelRegistryOut(**{k: getattr(v, k) for k in ModelRegistryOut.model_fields})
+
+
+@router.delete("/{model_name}")
+async def delete_model(model_name: str):
+    svc = get_service("models")
+    if not svc.delete_model(model_name):
+        raise HTTPException(404, "Model not found")
+    return {"ok": True}