flashstudio 0.1.0__py3-none-any.whl

flashstudio/pages/training.py

@@ -0,0 +1,672 @@
+"""FlashStudio — Training Dashboard Page (reads real FlashDet workspace output)."""
+
+import os
+import re
+import json
+import glob as glob_module
+import time
+import importlib.util
+
+import streamlit as st
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import numpy as np
+from PIL import Image
+
+
+def _get_default_workspace():
+    """Auto-detect workspace: use save_dir from config or scan common locations."""
+    save_dir = st.session_state.get("save_dir", "")
+    if save_dir and os.path.isdir(save_dir):
+        return save_dir
+
+    # Try to find FlashDet workspace relative to this package
+    candidates = [
+        os.path.join(os.getcwd(), "workspace"),
+        os.path.join(os.getcwd(), "..", "FlashDet", "workspace"),
+        os.path.join(os.path.dirname(__file__), "..", "..", "..", "FlashDet", "workspace"),
+    ]
+    for c in candidates:
+        c = os.path.abspath(c)
+        if os.path.isdir(c):
+            return c
+
+    return os.getcwd()
+
+
+def render_training_page():
+    """Render training dashboard — monitor real FlashDet runs or start new training."""
+    from flashstudio.components.styles import render_page_header
+    render_page_header("🏋️", "Training Dashboard",
+                       "Monitor real FlashDet training runs — logs, curves, visualizations.")
+
+    if "training_active" not in st.session_state:
+        st.session_state["training_active"] = False
+        st.session_state["training_status"] = "Not started"
+
+    tab_start, tab_monitor = st.tabs(["▶️ Start Training", "📊 Monitor Run"])
+
+    with tab_start:
+        _render_start_tab()
+
+    with tab_monitor:
+        _render_monitor_tab()
+
+
+def _render_monitor_tab():
+    """Tab: monitor an existing training run from workspace."""
+    workspace = st.text_input(
+        "Workspace path",
+        value=_get_default_workspace(),
+        key="workspace_path",
+        help="Path containing training run folders (auto-detected from save_dir config or FlashDet workspace)",
+    )
+
+    if not os.path.isdir(workspace):
+        st.warning(f"Workspace not found: `{workspace}`")
+        return
+
+    runs = sorted(
+        [d for d in os.listdir(workspace) if os.path.isdir(os.path.join(workspace, d))],
+        key=lambda d: os.path.getmtime(os.path.join(workspace, d)),
+        reverse=True,
+    )
+
+    if not runs:
+        st.info("No training runs found in workspace.")
+        return
+
+    selected_run = st.selectbox("Select Training Run", runs, key="selected_run")
+    run_dir = os.path.join(workspace, selected_run)
+
+    st.divider()
+    _render_run_dashboard(run_dir)
+
+
+def _render_run_dashboard(run_dir: str):
+    """Render full dashboard for a single training run."""
+    log_file = _find_log_file(run_dir)
+    history = _parse_training_log(log_file) if log_file else None
+
+    # Top metrics row
+    _render_metrics_from_history(history, run_dir)
+
+    # Main content in tabs
+    tab_curves, tab_viz, tab_gt, tab_log, tab_files = st.tabs([
+        "📈 Training Curves", "🖼️ Visualizations", "✅ GT Verification",
+        "📜 Full Log", "📁 Checkpoints"
+    ])
+
+    with tab_curves:
+        _render_curves(history, run_dir)
+
+    with tab_viz:
+        _render_visualizations(run_dir)
+
+    with tab_gt:
+        _render_gt_verification(run_dir)
+
+    with tab_log:
+        _render_full_log(log_file)
+
+    with tab_files:
+        _render_checkpoints(run_dir)
+
+
+def _find_log_file(run_dir: str):
+    """Find the training log file in a run directory."""
+    logs = glob_module.glob(os.path.join(run_dir, "train_*.log"))
+    if logs:
+        return max(logs, key=os.path.getmtime)
+    return None
+
+
+def _parse_training_log(log_path: str):
+    """Parse FlashDet training log and extract metrics per epoch."""
+    if not log_path or not os.path.isfile(log_path):
+        return None
+
+    history = {
+        "epochs": [], "lr": [], "train_loss": [],
+        "val_loss": [], "mAP50": [],
+        "o2m_cls": [], "o2m_box": [], "o2o_cls": [], "o2o_box": [],
+        "ema_decay": [], "epoch_time": [],
+        "model_info": "", "device": "", "classes": [],
+        "total_epochs": 0, "batch_size": 0,
+    }
+
+    with open(log_path, "r") as f:
+        lines = f.readlines()
+
+    current_epoch = 0
+    for line in lines:
+        # Parse header info
+        if "Model Size:" in line:
+            history["model_info"] = line.split("Model Size:")[-1].strip()
+        if "Device:" in line:
+            history["device"] = line.split("Device:")[-1].strip()
+        if "Classes" in line and ":" in line:
+            m = re.search(r"Classes \((\d+)\): \[(.+)\]", line)
+            if m:
+                history["classes"] = [c.strip().strip("'") for c in m.group(2).split(",")]
+        if "Epochs:" in line and "Batch" not in line:
+            m = re.search(r"Epochs: (\d+)", line)
+            if m:
+                history["total_epochs"] = int(m.group(1))
+        if "Batch Size:" in line:
+            m = re.search(r"Batch Size: (\d+)", line)
+            if m:
+                history["batch_size"] = int(m.group(1))
+
+        # Parse epoch header: Epoch 1/10 (lr=0.000010, ema_decay=0.000500)
+        epoch_m = re.search(r"Epoch (\d+)/(\d+) \(lr=([\d.e+-]+),\s*ema_decay=([\d.e+-]+)\)", line)
+        if epoch_m:
+            current_epoch = int(epoch_m.group(1))
+            if history["total_epochs"] == 0:
+                history["total_epochs"] = int(epoch_m.group(2))
+            history["lr"].append(float(epoch_m.group(3)))
+            history["ema_decay"].append(float(epoch_m.group(4)))
+
+        # Parse batch loss: Epoch [1] Batch [10/16] Loss: 1156.5105 (...)
+        batch_m = re.search(
+            r"Epoch \[(\d+)\] Batch \[\d+/\d+\] Loss: ([\d.]+) "
+            r"\(.*?o2m_cls: ([\d.]+), o2m_box: ([\d.]+).*?o2o_cls: ([\d.]+), o2o_box: ([\d.]+)",
+            line
+        )
+        if batch_m:
+            epoch_num = int(batch_m.group(1))
+            total_loss = float(batch_m.group(2))
+            # Keep only the last batch of each epoch as the "epoch loss"
+            while len(history["train_loss"]) < epoch_num:
+                history["train_loss"].append(None)
+            history["train_loss"][epoch_num - 1] = total_loss
+
+            while len(history["o2m_cls"]) < epoch_num:
+                history["o2m_cls"].append(None)
+                history["o2m_box"].append(None)
+                history["o2o_cls"].append(None)
+                history["o2o_box"].append(None)
+            history["o2m_cls"][epoch_num - 1] = float(batch_m.group(3))
+            history["o2m_box"][epoch_num - 1] = float(batch_m.group(4))
+            history["o2o_cls"][epoch_num - 1] = float(batch_m.group(5))
+            history["o2o_box"][epoch_num - 1] = float(batch_m.group(6))
+
+        # Parse epoch time: Epoch time: 366.7s
+        time_m = re.search(r"Epoch time: ([\d.]+)s", line)
+        if time_m:
+            history["epoch_time"].append(float(time_m.group(1)))
+
+        # Parse validation: Validation - Loss: X.XXXX | mAP@0.5: X.XXXX
+        val_m = re.search(r"Validation.*Loss: ([\d.]+).*mAP@0.5: ([\d.]+)", line)
+        if val_m:
+            history["val_loss"].append(float(val_m.group(1)))
+            history["mAP50"].append(float(val_m.group(2)))
+
+    # Fill in epochs list
+    n = max(len(history["train_loss"]), len(history["lr"]))
+    history["epochs"] = list(range(1, n + 1))
+
+    return history
+
+
+def _render_metrics_from_history(history, run_dir):
+    """Render top metric cards from parsed history."""
+    cols = st.columns(6)
+
+    if history:
+        n_epochs = len(history["epochs"])
+        total = history["total_epochs"] or n_epochs
+
+        with cols[0]:
+            st.metric("Epoch", f"{n_epochs}/{total}")
+        with cols[1]:
+            losses = [x for x in history["train_loss"] if x is not None]
+            val = f"{losses[-1]:.1f}" if losses else "—"
+            st.metric("Train Loss", val)
+        with cols[2]:
+            val = f"{history['val_loss'][-1]:.4f}" if history["val_loss"] else "—"
+            st.metric("Val Loss", val)
+        with cols[3]:
+            val = f"{history['mAP50'][-1]:.4f}" if history["mAP50"] else "—"
+            st.metric("mAP@0.5", val)
+        with cols[4]:
+            best = f"{max(history['mAP50']):.4f}" if history["mAP50"] else "—"
+            st.metric("Best mAP", best)
+        with cols[5]:
+            lr_val = f"{history['lr'][-1]:.2e}" if history["lr"] else "—"
+            st.metric("Current LR", lr_val)
+
+        if n_epochs < total:
+            st.progress(n_epochs / max(total, 1))
+    else:
+        for col in cols:
+            with col:
+                st.metric("—", "No data")
+
+
+def _render_curves(history, run_dir):
+    """Render training curves — from parsed log or saved plots."""
+    # Check for pre-generated plot images
+    plots_dir = os.path.join(run_dir, "plots")
+    training_curves_img = os.path.join(plots_dir, "training_curves.png")
+    map_curve_img = os.path.join(plots_dir, "mAP_curve.png")
+
+    if os.path.isfile(training_curves_img):
+        st.markdown("#### Saved Training Curves (matplotlib)")
+        st.image(training_curves_img, caption="training_curves.png")
+        if os.path.isfile(map_curve_img):
+            st.image(map_curve_img, caption="mAP_curve.png")
+        st.divider()
+
+    # Interactive plotly charts from parsed log
+    if not history or not history["train_loss"]:
+        st.info("No training data parsed yet. Training may still be starting.")
+        return
+
+    st.markdown("#### Interactive Charts (parsed from log)")
+
+    losses = [x for x in history["train_loss"] if x is not None]
+    epochs_for_loss = [i + 1 for i, x in enumerate(history["train_loss"]) if x is not None]
+
+    fig = make_subplots(
+        rows=2, cols=2,
+        subplot_titles=("Total Loss", "mAP@0.5", "Sub-Losses (o2m/o2o)", "Learning Rate"),
+    )
+
+    # Total loss
+    fig.add_trace(
+        go.Scatter(x=epochs_for_loss, y=losses, mode="lines+markers",
+                   name="Train Loss", line=dict(color="#7C3AED", width=2), marker=dict(size=4)),
+        row=1, col=1,
+    )
+    if history["val_loss"]:
+        val_epochs = list(range(1, len(history["val_loss"]) + 1))
+        fig.add_trace(
+            go.Scatter(x=val_epochs, y=history["val_loss"], mode="lines+markers",
+                       name="Val Loss", line=dict(color="#F59E0B", width=2), marker=dict(size=4)),
+            row=1, col=1,
+        )
+
+    # mAP
+    if history["mAP50"]:
+        map_epochs = list(range(1, len(history["mAP50"]) + 1))
+        fig.add_trace(
+            go.Scatter(x=map_epochs, y=history["mAP50"], mode="lines+markers",
+                       name="mAP@0.5", line=dict(color="#10B981", width=2), marker=dict(size=5)),
+            row=1, col=2,
+        )
+
+    # Sub-losses
+    o2m_cls = [x for x in history["o2m_cls"] if x is not None]
+    o2o_cls = [x for x in history["o2o_cls"] if x is not None]
+    o2m_box = [x for x in history["o2m_box"] if x is not None]
+    if o2m_cls:
+        ep = list(range(1, len(o2m_cls) + 1))
+        fig.add_trace(
+            go.Scatter(x=ep, y=o2m_cls, mode="lines", name="o2m_cls",
+                       line=dict(color="#EF4444", width=1.5)),
+            row=2, col=1,
+        )
+    if o2o_cls:
+        ep = list(range(1, len(o2o_cls) + 1))
+        fig.add_trace(
+            go.Scatter(x=ep, y=o2o_cls, mode="lines", name="o2o_cls",
+                       line=dict(color="#3B82F6", width=1.5)),
+            row=2, col=1,
+        )
+    if o2m_box:
+        ep = list(range(1, len(o2m_box) + 1))
+        fig.add_trace(
+            go.Scatter(x=ep, y=o2m_box, mode="lines", name="o2m_box",
+                       line=dict(color="#F97316", width=1.5, dash="dash")),
+            row=2, col=1,
+        )
+
+    # LR
+    if history["lr"]:
+        lr_epochs = list(range(1, len(history["lr"]) + 1))
+        fig.add_trace(
+            go.Scatter(x=lr_epochs, y=history["lr"], mode="lines",
+                       name="LR", line=dict(color="#6366F1", width=2)),
+            row=2, col=2,
+        )
+
+    fig.update_layout(
+        template="plotly_white",
+        height=500,
+        margin=dict(l=40, r=20, t=40, b=40),
+        showlegend=True,
+        legend=dict(orientation="h", yanchor="bottom", y=1.05),
+    )
+    st.plotly_chart(fig, use_container_width=True)
+
+
+def _render_visualizations(run_dir):
+    """Show per-epoch GT vs Prediction visualizations."""
+    vis_dir = os.path.join(run_dir, "visualizations")
+
+    if not os.path.isdir(vis_dir):
+        st.info("No visualizations directory found. Training may not have generated any yet.")
+        return
+
+    images = sorted([
+        f for f in os.listdir(vis_dir)
+        if f.endswith(".jpg") and f != "latest_visualization.jpg"
+    ])
+
+    if not images:
+        st.info("No visualization images found yet.")
+        return
+
+    st.markdown(f"**{len(images)} epoch visualizations** (GT vs Predictions)")
+
+    # Show latest first
+    latest = os.path.join(vis_dir, "latest_visualization.jpg")
+    if os.path.isfile(latest):
+        st.markdown("#### Latest Visualization")
+        st.image(latest, caption="Latest (GT left | Predictions right)")
+
+    st.divider()
+    st.markdown("#### All Epoch Visualizations")
+
+    # Display in a grid (2 per row)
+    for i in range(0, len(images), 2):
+        cols = st.columns(2)
+        for j, col in enumerate(cols):
+            idx = i + j
+            if idx < len(images):
+                img_path = os.path.join(vis_dir, images[idx])
+                with col:
+                    st.image(img_path, caption=images[idx], use_container_width=True)
+
+
+def _render_gt_verification(run_dir):
+    """Show GT verification images and report."""
+    gt_dir = os.path.join(run_dir, "gt_verification")
+
+    if not os.path.isdir(gt_dir):
+        st.info("No GT verification data found.")
+        return
+
+    # Summary
+    summary_path = os.path.join(gt_dir, "verification_summary.txt")
+    if os.path.isfile(summary_path):
+        with open(summary_path) as f:
+            st.code(f.read(), language="text")
+
+    # Report JSON highlights
+    report_path = os.path.join(gt_dir, "verification_report.json")
+    if os.path.isfile(report_path):
+        with open(report_path) as f:
+            report = json.load(f)
+
+        passed = report.get("passed", False)
+        if passed:
+            st.success("Annotation verification: PASSED")
+        else:
+            st.error("Annotation verification: FAILED")
+
+        cols = st.columns(4)
+        train_coco = report.get("splits", {}).get("train", {}).get("coco", {})
+        val_coco = report.get("splits", {}).get("val", {}).get("coco", {})
+        with cols[0]:
+            st.metric("Train Images", train_coco.get("num_images", 0))
+        with cols[1]:
+            st.metric("Train Annotations", train_coco.get("num_annotations", 0))
+        with cols[2]:
+            st.metric("Val Images", val_coco.get("num_images", 0))
+        with cols[3]:
+            st.metric("Val Annotations", val_coco.get("num_annotations", 0))
+
+    # GT Images
+    st.divider()
+    raw_dir = os.path.join(gt_dir, "images", "raw")
+    dl_dir = os.path.join(gt_dir, "images", "dataloader")
+
+    gt_tab_raw, gt_tab_dl = st.tabs(["Raw GT Images", "Dataloader GT Images"])
+
+    with gt_tab_raw:
+        _render_image_grid(raw_dir, "Raw ground truth with bounding boxes")
+
+    with gt_tab_dl:
+        _render_image_grid(dl_dir, "After dataloader transforms (letterbox, normalize)")
+
+
+def _render_image_grid(img_dir, description):
+    """Render images from a directory in a grid."""
+    if not os.path.isdir(img_dir):
+        st.info(f"Directory not found: {img_dir}")
+        return
+
+    images = sorted([f for f in os.listdir(img_dir) if f.endswith((".jpg", ".png"))])
+    if not images:
+        st.info("No images found.")
+        return
+
+    st.caption(f"{description} — {len(images)} images")
+
+    for i in range(0, len(images), 4):
+        cols = st.columns(4)
+        for j, col in enumerate(cols):
+            idx = i + j
+            if idx < len(images):
+                img_path = os.path.join(img_dir, images[idx])
+                with col:
+                    st.image(img_path, caption=images[idx][:30], use_container_width=True)
+
+
+def _render_full_log(log_path):
+    """Show full training log file."""
+    if not log_path or not os.path.isfile(log_path):
+        st.info("No training log found.")
+        return
+
+    with open(log_path) as f:
+        content = f.read()
+
+    st.markdown(f"**Log file:** `{os.path.basename(log_path)}`")
+
+    # Show last 100 lines by default, with option to expand
+    lines = content.strip().split("\n")
+    show_all = st.checkbox("Show full log", value=False, key="show_full_log")
+
+    if show_all:
+        st.code(content, language="bash")
+    else:
+        st.code("\n".join(lines[-50:]), language="bash")
+        if len(lines) > 50:
+            st.caption(f"Showing last 50 of {len(lines)} lines. Check 'Show full log' to see all.")
+
+
+def _render_checkpoints(run_dir):
+    """Show checkpoint files and their sizes."""
+    st.markdown("#### Saved Checkpoints & Weights")
+
+    files_info = []
+    for f in sorted(os.listdir(run_dir)):
+        fpath = os.path.join(run_dir, f)
+        if os.path.isfile(fpath) and f.endswith((".pth", ".json", ".csv", ".log")):
+            size = os.path.getsize(fpath)
+            if size > 1024 * 1024:
+                size_str = f"{size / (1024*1024):.1f} MB"
+            elif size > 1024:
+                size_str = f"{size / 1024:.1f} KB"
+            else:
+                size_str = f"{size} B"
+            files_info.append({"File": f, "Size": size_str, "Type": _file_type(f)})
+
+    if files_info:
+        st.dataframe(files_info, use_container_width=True, hide_index=True)
+    else:
+        st.info("No checkpoint files found.")
+
+    # results.json
+    results_path = os.path.join(run_dir, "results.json")
+    if os.path.isfile(results_path):
+        st.divider()
+        st.markdown("#### Training Results Summary")
+        with open(results_path) as f:
+            results = json.load(f)
+
+        cols = st.columns(4)
+        with cols[0]:
+            st.metric("Epochs Trained", results.get("epochs_trained", "?"))
+        with cols[1]:
+            st.metric("Best mAP@0.5", f"{results.get('best_mAP50', 0):.4f}")
+        with cols[2]:
+            st.metric("Best Val Loss", f"{results.get('best_val_loss', 0):.4f}")
+        with cols[3]:
+            st.metric("Model Params", f"{results.get('model_params_M', 0):.2f}M")
+
+
+def _file_type(filename):
+    """Categorize a file by its name."""
+    if "best" in filename:
+        return "Best checkpoint"
+    if "last" in filename:
+        return "Latest checkpoint"
+    if "inference" in filename:
+        return "Inference weights"
+    if "fp16" in filename:
+        return "FP16 weights"
+    if filename.endswith(".json"):
+        return "Results/Report"
+    if filename.endswith(".csv"):
+        return "Training log CSV"
+    if filename.endswith(".log"):
+        return "Training log"
+    return "Other"
+
+
+# ---- Start New Training Tab ----
+
+def _render_start_tab():
+    """Tab: start a new training run."""
+    st.markdown("### Start New Training")
+
+    col_info, col_actions = st.columns([3, 1])
+
+    with col_info:
+        arch = st.session_state.get("model_arch", "FlashDet-Pico")
+        dataset = st.session_state.get("dataset_name", "Not selected")
+        epochs = st.session_state.get("epochs", 100)
+        bs = st.session_state.get("batch_size", 16)
+        lr = st.session_state.get("lr", 0.001)
+
+        st.markdown(
+            f"**Config:** `{arch}` · Dataset: `{dataset}` · "
+            f"Epochs: `{epochs}` · BS: `{bs}` · LR: `{lr}`"
+        )
+
+    with col_actions:
+        if not st.session_state.get("training_active", False):
+            if st.button("▶️ Start Training", use_container_width=True, type="primary",
+                         key="btn_start_training"):
+                st.session_state["training_active"] = True
+                st.session_state["training_status"] = "Running"
+                _start_training()
+        else:
+            if st.button("⏹️ Stop Training", use_container_width=True, type="secondary",
+                         key="btn_stop_training"):
+                st.session_state["training_active"] = False
+                st.session_state["training_status"] = "Stopped"
+                st.rerun()
+
+    if st.session_state.get("training_active"):
+        st.info("Training is running... Switch to 'Monitor Run' tab and select latest run to see progress.")
+
+
+def _start_training():
+    """Start training — real FlashDet or demo mode."""
+    if importlib.util.find_spec("flashdet") is not None:
+        _run_flashdet_training()
+    else:
+        st.warning("FlashDet not installed. Install with `pip install flashdet` to run real training.")
+        st.session_state["training_active"] = False
+        st.session_state["training_status"] = "FlashDet not found"
+
+
+def _run_flashdet_training():
+    """Run actual FlashDet training via subprocess (non-blocking)."""
+    import subprocess
+    import sys
+
+    size_map = {
+        "FlashDet-Pico": "p", "FlashDet-Nano": "n", "FlashDet-Small": "s",
+        "FlashDet-Medium": "m", "FlashDet-Large": "l", "FlashDet-X": "x",
+    }
+
+    model_arch = st.session_state.get("model_arch", "FlashDet-Pico")
+    save_dir = st.session_state.get("save_dir", os.path.join(_get_default_workspace(), "flashstudio_run"))
+
+    # Auto-detect device
+    device = "cpu"
+    try:
+        import torch
+        if torch.cuda.is_available():
+            device = "cuda"
+    except ImportError:
+        pass
+
+    # Resolve dataset paths from session state
+    train_images = st.session_state.get("train_img_path", "")
+    val_images = st.session_state.get("val_img_path", "")
+
+    # Try to auto-detect from dataset download command
+    if not train_images or not os.path.isdir(train_images):
+        dataset_name = st.session_state.get("dataset_name", "")
+        dataset_id = dataset_name.lower().replace(" ", "").replace("(demo)", "sample")
+        for candidate_id in ["sample", "coco2017", "coco2017-val", "voc2007", "voc2012"]:
+            if candidate_id in dataset_id or dataset_id in candidate_id:
+                candidate_dir = os.path.join("data", candidate_id)
+                if os.path.isdir(os.path.join(candidate_dir, "train")):
+                    train_images = os.path.join(candidate_dir, "train")
+                    val_images = os.path.join(candidate_dir, "valid")
+                    break
+
+    if not train_images or not val_images:
+        st.error(
+            "No dataset paths configured. Please go to the **Data** page and either:\n"
+            "- Enter train/val image paths manually, or\n"
+            "- Download a dataset first (`flashdet download --dataset sample`)"
+        )
+        st.session_state["training_active"] = False
+        st.session_state["training_status"] = "No dataset"
+        return
+
+    cmd = [
+        sys.executable, "-m", "flashdet.cli", "train",
+        "--model-size", size_map.get(model_arch, "n"),
+        "--epochs", str(st.session_state.get("epochs", 100)),
+        "--batch-size", str(st.session_state.get("batch_size", 16)),
+        "--lr", str(st.session_state.get("lr", 1e-3)),
+        "--save-dir", save_dir,
+        "--device", device,
+        "--train-images", train_images,
+        "--val-images", val_images,
+    ]
+
+    if st.session_state.get("amp", True) and device == "cuda":
+        cmd.append("--amp")
+    if st.session_state.get("aug_mosaic", True):
+        cmd.append("--mosaic")
+    if st.session_state.get("aug_mixup", False):
+        cmd.append("--mixup")
+
+    optimizer = st.session_state.get("optimizer", "AdamW").lower()
+    if optimizer in ("adamw", "sgd", "musgd"):
+        cmd.extend(["--optimizer", optimizer])
+
+    workers = st.session_state.get("num_workers", 2)
+    cmd.extend(["--workers", str(workers)])
+
+    # Run as non-blocking subprocess so Streamlit UI stays responsive
+    process = subprocess.Popen(
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,
+    )
+
+    st.session_state["training_pid"] = process.pid
+
+    st.info(f"Training started (PID {process.pid}): `{' '.join(cmd)}`")
+    st.caption(f"Output will be saved to: `{save_dir}`")
+    st.caption("Switch to 'Monitor Run' tab to see live progress.")