mcli-framework 7.1.1__py3-none-any.whl → 7.1.3__py3-none-any.whl

mcli/ml/dashboard/app_training.py

@@ -1,27 +1,28 @@
 """Enhanced training dashboard with Bitcoin-style model comparison and analysis"""
 
-import streamlit as st
+from datetime import datetime, timedelta
+
+import numpy as np
 import pandas as pd
 import plotly.express as px
 import plotly.graph_objects as go
+import streamlit as st
 from plotly.subplots import make_subplots
-import numpy as np
-from datetime import datetime, timedelta
 from scipy import stats
 
+from mcli.ml.database.models import Experiment, Model, ModelStatus
 from mcli.ml.database.session import SessionLocal
-from mcli.ml.database.models import Model, ModelStatus, Experiment
-
 
 st.set_page_config(
     page_title="MCLI Training Dashboard",
     page_icon="🔬",
     layout="wide",
-    initial_sidebar_state="expanded"
+    initial_sidebar_state="expanded",
 )
 
 # Custom CSS
-st.markdown("""
+st.markdown(
+    """
 <style>
     .metric-card {
         background-color: #f0f2f6;
@@ -40,7 +41,9 @@ st.markdown("""
         border-left: 4px solid #28a745;
     }
 </style>
-""", unsafe_allow_html=True)
+""",
+    unsafe_allow_html=True,
+)
 
 
 @st.cache_data(ttl=60)
@@ -49,23 +52,23 @@ def get_training_jobs():
     db = SessionLocal()
 
     try:
-        experiments = db.query(Experiment).order_by(
-            Experiment.created_at.desc()
-        ).limit(50).all()
+        experiments = db.query(Experiment).order_by(Experiment.created_at.desc()).limit(50).all()
 
         data = []
         for exp in experiments:
-            data.append({
-                'name': exp.name,
-                'status': exp.status,
-                'started_at': exp.started_at,
-                'completed_at': exp.completed_at,
-                'duration_seconds': exp.duration_seconds,
-                'hyperparameters': exp.hyperparameters,
-                'train_metrics': exp.train_metrics or {},
-                'val_metrics': exp.val_metrics or {},
-                'test_metrics': exp.test_metrics or {},
-            })
+            data.append(
+                {
+                    "name": exp.name,
+                    "status": exp.status,
+                    "started_at": exp.started_at,
+                    "completed_at": exp.completed_at,
+                    "duration_seconds": exp.duration_seconds,
+                    "hyperparameters": exp.hyperparameters,
+                    "train_metrics": exp.train_metrics or {},
+                    "val_metrics": exp.val_metrics or {},
+                    "test_metrics": exp.test_metrics or {},
+                }
+            )
 
         return pd.DataFrame(data)
     finally:
@@ -78,44 +81,43 @@ def get_model_comparison():
     db = SessionLocal()
 
     try:
-        models = db.query(Model).filter(
-            Model.status.in_([ModelStatus.TRAINED, ModelStatus.DEPLOYED])
-        ).all()
+        models = (
+            db.query(Model)
+            .filter(Model.status.in_([ModelStatus.TRAINED, ModelStatus.DEPLOYED]))
+            .all()
+        )
 
         data = []
         for model in models:
             metrics = model.metrics or {}
 
             # Extract metrics similar to bitcoin project
-            data.append({
-                'name': model.name,
-                'version': model.version,
-                'model_type': model.model_type,
-                'status': model.status.value,
-
-                # Training metrics
-                'train_accuracy': model.train_accuracy or 0,
-                'train_loss': model.train_loss or 0,
-
-                # Validation metrics
-                'val_accuracy': model.val_accuracy or 0,
-                'val_loss': model.val_loss or 0,
-
-                # Test metrics
-                'test_accuracy': model.test_accuracy or 0,
-                'test_loss': model.test_loss or 0,
-
-                # Additional metrics
-                'rmse': metrics.get('rmse', 0),
-                'mae': metrics.get('mae', 0),
-                'r2': metrics.get('r2', 0),
-                'mape': metrics.get('mape', 0),
-
-                # Metadata
-                'is_deployed': model.status == ModelStatus.DEPLOYED,
-                'created_at': model.created_at,
-                'updated_at': model.updated_at,
-            })
+            data.append(
+                {
+                    "name": model.name,
+                    "version": model.version,
+                    "model_type": model.model_type,
+                    "status": model.status.value,
+                    # Training metrics
+                    "train_accuracy": model.train_accuracy or 0,
+                    "train_loss": model.train_loss or 0,
+                    # Validation metrics
+                    "val_accuracy": model.val_accuracy or 0,
+                    "val_loss": model.val_loss or 0,
+                    # Test metrics
+                    "test_accuracy": model.test_accuracy or 0,
+                    "test_loss": model.test_loss or 0,
+                    # Additional metrics
+                    "rmse": metrics.get("rmse", 0),
+                    "mae": metrics.get("mae", 0),
+                    "r2": metrics.get("r2", 0),
+                    "mape": metrics.get("mape", 0),
+                    # Metadata
+                    "is_deployed": model.status == ModelStatus.DEPLOYED,
+                    "created_at": model.created_at,
+                    "updated_at": model.updated_at,
+                }
+            )
 
         return pd.DataFrame(data)
     finally:
@@ -129,16 +131,16 @@ def get_feature_importance(model_id: str):
 
     try:
         from sqlalchemy.dialects.postgresql import UUID
+
         model = db.query(Model).filter(Model.id == model_id).first()
 
         if model and model.feature_names:
             # Simulate feature importance (in real scenario, load from model artifacts)
             importance = np.random.dirichlet(np.ones(len(model.feature_names)))
 
-            return pd.DataFrame({
-                'feature': model.feature_names,
-                'importance': importance
-            }).sort_values('importance', ascending=False)
+            return pd.DataFrame(
+                {"feature": model.feature_names, "importance": importance}
+            ).sort_values("importance", ascending=False)
 
         return pd.DataFrame()
     finally:
@@ -162,53 +164,46 @@ def show_model_comparison():
         st.metric(
             label="Total Models",
             value=len(models_df),
-            delta=f"{len(models_df[models_df['status'] == 'deployed'])} deployed"
+            delta=f"{len(models_df[models_df['status'] == 'deployed'])} deployed",
         )
 
     with col2:
-        best_model = models_df.loc[models_df['test_accuracy'].idxmax()]
+        best_model = models_df.loc[models_df["test_accuracy"].idxmax()]
         st.metric(
             label="Best Test Accuracy",
             value=f"{best_model['test_accuracy']:.4f}",
-            delta=best_model['name']
+            delta=best_model["name"],
         )
 
     with col3:
-        if models_df['rmse'].max() > 0:
-            best_rmse = models_df[models_df['rmse'] > 0].loc[models_df['rmse'].idxmin()]
-            st.metric(
-                label="Best RMSE",
-                value=f"{best_rmse['rmse']:.4f}",
-                delta=best_rmse['name']
-            )
+        if models_df["rmse"].max() > 0:
+            best_rmse = models_df[models_df["rmse"] > 0].loc[models_df["rmse"].idxmin()]
+            st.metric(label="Best RMSE", value=f"{best_rmse['rmse']:.4f}", delta=best_rmse["name"])
 
     with col4:
-        if models_df['r2'].max() > 0:
-            best_r2 = models_df.loc[models_df['r2'].idxmax()]
-            st.metric(
-                label="Best R² Score",
-                value=f"{best_r2['r2']:.4f}",
-                delta=best_r2['name']
-            )
+        if models_df["r2"].max() > 0:
+            best_r2 = models_df.loc[models_df["r2"].idxmax()]
+            st.metric(label="Best R² Score", value=f"{best_r2['r2']:.4f}", delta=best_r2["name"])
 
     # Model comparison table
     st.subheader("Model Performance Table")
 
     # Select metrics to display
-    display_cols = ['name', 'version', 'model_type', 'test_accuracy', 'test_loss']
+    display_cols = ["name", "version", "model_type", "test_accuracy", "test_loss"]
 
-    if models_df['rmse'].max() > 0:
-        display_cols.extend(['rmse', 'mae', 'r2'])
+    if models_df["rmse"].max() > 0:
+        display_cols.extend(["rmse", "mae", "r2"])
 
-    display_cols.extend(['status', 'created_at'])
+    display_cols.extend(["status", "created_at"])
 
     # Sort by test accuracy
-    sorted_df = models_df[display_cols].sort_values('test_accuracy', ascending=False)
+    sorted_df = models_df[display_cols].sort_values("test_accuracy", ascending=False)
 
     st.dataframe(
-        sorted_df.style.highlight_max(subset=['test_accuracy', 'r2'], color='lightgreen')
-                      .highlight_min(subset=['test_loss', 'rmse', 'mae'], color='lightgreen'),
-        use_container_width=True
+        sorted_df.style.highlight_max(
+            subset=["test_accuracy", "r2"], color="lightgreen"
+        ).highlight_min(subset=["test_loss", "rmse", "mae"], color="lightgreen"),
+        use_container_width=True,
     )
 
     # Visualization section
@@ -220,11 +215,11 @@ def show_model_comparison():
         # Accuracy comparison
         fig = px.bar(
             sorted_df.head(10),
-            x='name',
-            y=['train_accuracy', 'val_accuracy', 'test_accuracy'],
+            x="name",
+            y=["train_accuracy", "val_accuracy", "test_accuracy"],
             title="Accuracy Comparison (Train/Val/Test)",
-            barmode='group',
-            labels={'value': 'Accuracy', 'variable': 'Split'}
+            barmode="group",
+            labels={"value": "Accuracy", "variable": "Split"},
         )
         fig.update_layout(xaxis_tickangle=-45)
         st.plotly_chart(fig, use_container_width=True)
@@ -233,43 +228,43 @@ def show_model_comparison():
         # Loss comparison
         fig = px.bar(
             sorted_df.head(10),
-            x='name',
-            y=['train_loss', 'val_loss', 'test_loss'],
+            x="name",
+            y=["train_loss", "val_loss", "test_loss"],
             title="Loss Comparison (Train/Val/Test)",
-            barmode='group',
-            labels={'value': 'Loss', 'variable': 'Split'}
+            barmode="group",
+            labels={"value": "Loss", "variable": "Split"},
         )
         fig.update_layout(xaxis_tickangle=-45)
         st.plotly_chart(fig, use_container_width=True)
 
     # Additional metrics if available
-    if models_df['rmse'].max() > 0:
+    if models_df["rmse"].max() > 0:
         col1, col2 = st.columns(2)
 
         with col1:
             # RMSE vs MAE
-            valid_models = models_df[(models_df['rmse'] > 0) & (models_df['mae'] > 0)]
+            valid_models = models_df[(models_df["rmse"] > 0) & (models_df["mae"] > 0)]
             fig = px.scatter(
                 valid_models,
-                x='rmse',
-                y='mae',
-                size='r2',
-                color='model_type',
-                hover_data=['name'],
-                title="RMSE vs MAE (sized by R²)"
+                x="rmse",
+                y="mae",
+                size="r2",
+                color="model_type",
+                hover_data=["name"],
+                title="RMSE vs MAE (sized by R²)",
             )
             st.plotly_chart(fig, use_container_width=True)
 
         with col2:
             # R² score comparison
-            valid_r2 = models_df[models_df['r2'] > 0].sort_values('r2', ascending=False).head(10)
+            valid_r2 = models_df[models_df["r2"] > 0].sort_values("r2", ascending=False).head(10)
             fig = px.bar(
                 valid_r2,
-                x='name',
-                y='r2',
+                x="name",
+                y="r2",
                 title="R² Score Comparison (Higher is Better)",
-                color='r2',
-                color_continuous_scale='Greens'
+                color="r2",
+                color_continuous_scale="Greens",
             )
             fig.update_layout(xaxis_tickangle=-45)
             st.plotly_chart(fig, use_container_width=True)
@@ -286,7 +281,7 @@ def show_residual_analysis():
         return
 
     # Model selector
-    model_options = models_df['name'].unique()
+    model_options = models_df["name"].unique()
     selected_model = st.selectbox("Select Model for Analysis", model_options)
 
     # Generate simulated residuals (in real scenario, load actual predictions)
@@ -299,27 +294,24 @@ def show_residual_analysis():
     residuals = actual - predicted
 
     # Create tabs for different analyses
-    tab1, tab2, tab3, tab4 = st.tabs([
-        "Residuals Over Time",
-        "Distribution",
-        "Q-Q Plot",
-        "Residuals vs Predicted"
-    ])
+    tab1, tab2, tab3, tab4 = st.tabs(
+        ["Residuals Over Time", "Distribution", "Q-Q Plot", "Residuals vs Predicted"]
+    )
 
     with tab1:
         st.subheader("Residuals Over Time")
         fig = go.Figure()
-        fig.add_trace(go.Scatter(
-            y=residuals,
-            mode='lines+markers',
-            name='Residuals',
-            line=dict(color='blue', width=1)
-        ))
+        fig.add_trace(
+            go.Scatter(
+                y=residuals,
+                mode="lines+markers",
+                name="Residuals",
+                line=dict(color="blue", width=1),
+            )
+        )
         fig.add_hline(y=0, line_dash="dash", line_color="red")
         fig.update_layout(
-            xaxis_title="Prediction Index",
-            yaxis_title="Residuals",
-            hovermode='x unified'
+            xaxis_title="Prediction Index", yaxis_title="Residuals", hovermode="x unified"
         )
         st.plotly_chart(fig, use_container_width=True)
 
@@ -335,31 +327,26 @@ def show_residual_analysis():
     with tab2:
         st.subheader("Residual Distribution")
         fig = go.Figure()
-        fig.add_trace(go.Histogram(
-            x=residuals,
-            nbinsx=50,
-            name='Residuals',
-            marker_color='lightblue'
-        ))
+        fig.add_trace(
+            go.Histogram(x=residuals, nbinsx=50, name="Residuals", marker_color="lightblue")
+        )
 
         # Add normal distribution overlay
         x_range = np.linspace(residuals.min(), residuals.max(), 100)
         y_norm = stats.norm.pdf(x_range, np.mean(residuals), np.std(residuals))
         y_norm_scaled = y_norm * len(residuals) * (residuals.max() - residuals.min()) / 50
 
-        fig.add_trace(go.Scatter(
-            x=x_range,
-            y=y_norm_scaled,
-            mode='lines',
-            name='Normal Distribution',
-            line=dict(color='red', width=2)
-        ))
-
-        fig.update_layout(
-            xaxis_title="Residuals",
-            yaxis_title="Frequency",
-            showlegend=True
+        fig.add_trace(
+            go.Scatter(
+                x=x_range,
+                y=y_norm_scaled,
+                mode="lines",
+                name="Normal Distribution",
+                line=dict(color="red", width=2),
+            )
         )
+
+        fig.update_layout(xaxis_title="Residuals", yaxis_title="Frequency", showlegend=True)
         st.plotly_chart(fig, use_container_width=True)
 
         # Normality tests
@@ -376,27 +363,31 @@ def show_residual_analysis():
         (osm, osr), (slope, intercept, r) = stats.probplot(residuals, dist="norm")
 
         fig = go.Figure()
-        fig.add_trace(go.Scatter(
-            x=osm,
-            y=osr,
-            mode='markers',
-            name='Sample Quantiles',
-            marker=dict(color='blue', size=5)
-        ))
+        fig.add_trace(
+            go.Scatter(
+                x=osm,
+                y=osr,
+                mode="markers",
+                name="Sample Quantiles",
+                marker=dict(color="blue", size=5),
+            )
+        )
 
         # Add reference line
-        fig.add_trace(go.Scatter(
-            x=osm,
-            y=slope * osm + intercept,
-            mode='lines',
-            name='Theoretical Line',
-            line=dict(color='red', width=2, dash='dash')
-        ))
+        fig.add_trace(
+            go.Scatter(
+                x=osm,
+                y=slope * osm + intercept,
+                mode="lines",
+                name="Theoretical Line",
+                line=dict(color="red", width=2, dash="dash"),
+            )
+        )
 
         fig.update_layout(
             xaxis_title="Theoretical Quantiles",
             yaxis_title="Sample Quantiles",
-            title="Q-Q Plot (Normal Distribution)"
+            title="Q-Q Plot (Normal Distribution)",
         )
         st.plotly_chart(fig, use_container_width=True)
 
@@ -405,26 +396,30 @@ def show_residual_analysis():
     with tab4:
         st.subheader("Residuals vs Predicted Values")
         fig = go.Figure()
-        fig.add_trace(go.Scatter(
-            x=predicted,
-            y=residuals,
-            mode='markers',
-            marker=dict(
-                color=np.abs(residuals),
-                colorscale='Reds',
-                showscale=True,
-                colorbar=dict(title="Abs Residual")
+        fig.add_trace(
+            go.Scatter(
+                x=predicted,
+                y=residuals,
+                mode="markers",
+                marker=dict(
+                    color=np.abs(residuals),
+                    colorscale="Reds",
+                    showscale=True,
+                    colorbar=dict(title="Abs Residual"),
+                ),
             )
-        ))
+        )
         fig.add_hline(y=0, line_dash="dash", line_color="black")
         fig.update_layout(
             xaxis_title="Predicted Values",
             yaxis_title="Residuals",
-            title="Residuals vs Predicted (looking for patterns)"
+            title="Residuals vs Predicted (looking for patterns)",
         )
         st.plotly_chart(fig, use_container_width=True)
 
-        st.info("💡 Ideally, residuals should be randomly scattered around zero with no clear patterns.")
+        st.info(
+            "💡 Ideally, residuals should be randomly scattered around zero with no clear patterns."
+        )
 
 
 def show_feature_importance():
@@ -438,7 +433,7 @@ def show_feature_importance():
         return
 
     # Model selector
-    model_options = models_df['name'].unique()
+    model_options = models_df["name"].unique()
     selected_model = st.selectbox("Select Model", model_options, key="feature_imp_model")
 
     # Get model details
@@ -449,10 +444,9 @@ def show_feature_importance():
         if model and model.feature_names:
             # Generate simulated feature importance
             importance = np.random.dirichlet(np.ones(len(model.feature_names)))
-            feature_df = pd.DataFrame({
-                'feature': model.feature_names,
-                'importance': importance
-            }).sort_values('importance', ascending=False)
+            feature_df = pd.DataFrame(
+                {"feature": model.feature_names, "importance": importance}
+            ).sort_values("importance", ascending=False)
 
             # Top N features
             top_n = st.slider("Number of top features to show", 5, min(50, len(feature_df)), 20)
@@ -461,14 +455,14 @@ def show_feature_importance():
             # Visualization
             fig = px.bar(
                 top_features,
-                y='feature',
-                x='importance',
-                orientation='h',
+                y="feature",
+                x="importance",
+                orientation="h",
                 title=f"Top {top_n} Most Important Features - {selected_model}",
-                color='importance',
-                color_continuous_scale='Viridis'
+                color="importance",
+                color_continuous_scale="Viridis",
             )
-            fig.update_layout(height=600, yaxis={'categoryorder':'total ascending'})
+            fig.update_layout(height=600, yaxis={"categoryorder": "total ascending"})
             st.plotly_chart(fig, use_container_width=True)
 
             # Feature importance table
@@ -480,12 +474,26 @@ def show_feature_importance():
 
             # Categorize features
             categories = {
-                'Lag Features': [f for f in feature_df['feature'] if 'lag' in f.lower()],
-                'Moving Averages': [f for f in feature_df['feature'] if 'ma' in f.lower() or 'sma' in f.lower() or 'ema' in f.lower()],
-                'Volatility': [f for f in feature_df['feature'] if 'volatility' in f.lower() or 'std' in f.lower()],
-                'Price Changes': [f for f in feature_df['feature'] if 'change' in f.lower() or 'pct' in f.lower()],
-                'Technical': [f for f in feature_df['feature'] if any(x in f.lower() for x in ['rsi', 'macd', 'bollinger'])],
-                'Other': []
+                "Lag Features": [f for f in feature_df["feature"] if "lag" in f.lower()],
+                "Moving Averages": [
+                    f
+                    for f in feature_df["feature"]
+                    if "ma" in f.lower() or "sma" in f.lower() or "ema" in f.lower()
+                ],
+                "Volatility": [
+                    f
+                    for f in feature_df["feature"]
+                    if "volatility" in f.lower() or "std" in f.lower()
+                ],
+                "Price Changes": [
+                    f for f in feature_df["feature"] if "change" in f.lower() or "pct" in f.lower()
+                ],
+                "Technical": [
+                    f
+                    for f in feature_df["feature"]
+                    if any(x in f.lower() for x in ["rsi", "macd", "bollinger"])
+                ],
+                "Other": [],
             }
 
             # Assign uncategorized features
@@ -493,26 +501,28 @@ def show_feature_importance():
             for cat_features in categories.values():
                 all_categorized.update(cat_features)
 
-            categories['Other'] = [f for f in feature_df['feature'] if f not in all_categorized]
+            categories["Other"] = [f for f in feature_df["feature"] if f not in all_categorized]
 
             # Calculate importance by category
             category_importance = {}
             for cat, features in categories.items():
                 if features:
-                    cat_imp = feature_df[feature_df['feature'].isin(features)]['importance'].sum()
+                    cat_imp = feature_df[feature_df["feature"].isin(features)]["importance"].sum()
                     category_importance[cat] = cat_imp
 
             if category_importance:
-                cat_df = pd.DataFrame({
-                    'Category': list(category_importance.keys()),
-                    'Total Importance': list(category_importance.values())
-                }).sort_values('Total Importance', ascending=False)
+                cat_df = pd.DataFrame(
+                    {
+                        "Category": list(category_importance.keys()),
+                        "Total Importance": list(category_importance.values()),
+                    }
+                ).sort_values("Total Importance", ascending=False)
 
                 fig = px.pie(
                     cat_df,
-                    values='Total Importance',
-                    names='Category',
-                    title="Feature Importance by Category"
+                    values="Total Importance",
+                    names="Category",
+                    title="Feature Importance by Category",
                 )
                 st.plotly_chart(fig, use_container_width=True)
         else:
@@ -538,35 +548,35 @@ def show_training_history():
         st.metric("Total Experiments", len(jobs_df))
 
     with col2:
-        completed = len(jobs_df[jobs_df['status'] == 'completed'])
+        completed = len(jobs_df[jobs_df["status"] == "completed"])
         st.metric("Completed", completed)
 
     with col3:
-        running = len(jobs_df[jobs_df['status'] == 'running'])
+        running = len(jobs_df[jobs_df["status"] == "running"])
         st.metric("Running", running)
 
     with col4:
-        failed = len(jobs_df[jobs_df['status'] == 'failed'])
+        failed = len(jobs_df[jobs_df["status"] == "failed"])
         st.metric("Failed", failed)
 
     # Training jobs table
     st.subheader("Recent Training Jobs")
 
-    display_df = jobs_df[['name', 'status', 'started_at', 'duration_seconds']].copy()
-    display_df['duration_minutes'] = display_df['duration_seconds'] / 60
+    display_df = jobs_df[["name", "status", "started_at", "duration_seconds"]].copy()
+    display_df["duration_minutes"] = display_df["duration_seconds"] / 60
 
     st.dataframe(display_df, use_container_width=True)
 
     # Training duration distribution
-    if not jobs_df['duration_seconds'].isna().all():
-        valid_durations = jobs_df[jobs_df['duration_seconds'].notna()]
+    if not jobs_df["duration_seconds"].isna().all():
+        valid_durations = jobs_df[jobs_df["duration_seconds"].notna()]
 
         fig = px.histogram(
             valid_durations,
-            x='duration_seconds',
+            x="duration_seconds",
             nbins=30,
             title="Training Duration Distribution",
-            labels={'duration_seconds': 'Duration (seconds)'}
+            labels={"duration_seconds": "Duration (seconds)"},
         )
         st.plotly_chart(fig, use_container_width=True)
 
@@ -580,18 +590,14 @@ def main():
     st.sidebar.title("Navigation")
     page = st.sidebar.selectbox(
         "Choose a view",
-        [
-            "Model Comparison",
-            "Residual Analysis",
-            "Feature Importance",
-            "Training History"
-        ]
+        ["Model Comparison", "Residual Analysis", "Feature Importance", "Training History"],
     )
 
     # Auto-refresh toggle
     auto_refresh = st.sidebar.checkbox("Auto-refresh (60s)", value=False)
     if auto_refresh:
         import time
+
         time.sleep(60)
         st.rerun()