mcli-framework 7.1.3__py3-none-any.whl → 7.3.1__py3-none-any.whl

mcli/ml/dashboard/components/tables.py

@@ -0,0 +1,228 @@
+"""Reusable table components"""
+
+import streamlit as st
+import pandas as pd
+from typing import Optional, List, Callable, Any
+
+
+def display_dataframe_with_search(
+    df: pd.DataFrame,
+    search_columns: Optional[List[str]] = None,
+    default_sort_column: Optional[str] = None,
+    page_size: int = 20,
+    key_prefix: str = "table"
+) -> pd.DataFrame:
+    """Display a dataframe with search and pagination"""
+
+    if df.empty:
+        st.info("No data available")
+        return df
+
+    # Search functionality
+    if search_columns:
+        search_term = st.text_input(
+            "🔍 Search",
+            key=f"{key_prefix}_search",
+            placeholder=f"Search in: {', '.join(search_columns)}"
+        )
+
+        if search_term:
+            mask = pd.Series([False] * len(df))
+            for col in search_columns:
+                if col in df.columns:
+                    mask |= df[col].astype(str).str.contains(search_term, case=False, na=False)
+            df = df[mask]
+
+    # Sorting
+    if default_sort_column and default_sort_column in df.columns:
+        df = df.sort_values(by=default_sort_column, ascending=False)
+
+    # Display count
+    st.caption(f"Showing {len(df)} records")
+
+    # Pagination
+    if len(df) > page_size:
+        total_pages = (len(df) - 1) // page_size + 1
+        page = st.number_input(
+            "Page",
+            min_value=1,
+            max_value=total_pages,
+            value=1,
+            key=f"{key_prefix}_page"
+        )
+        start_idx = (page - 1) * page_size
+        end_idx = start_idx + page_size
+        df_display = df.iloc[start_idx:end_idx]
+    else:
+        df_display = df
+
+    # Display dataframe
+    st.dataframe(df_display, width="stretch", height=400)
+
+    return df
+
+
+def display_filterable_dataframe(
+    df: pd.DataFrame,
+    filter_columns: Optional[dict] = None,
+    key_prefix: str = "filter"
+) -> pd.DataFrame:
+    """Display a dataframe with column-specific filters"""
+
+    if df.empty:
+        st.info("No data available")
+        return df
+
+    if filter_columns:
+        with st.expander("🎯 Filters", expanded=False):
+            cols = st.columns(len(filter_columns))
+
+            for idx, (col_name, filter_type) in enumerate(filter_columns.items()):
+                if col_name not in df.columns:
+                    continue
+
+                with cols[idx]:
+                    if filter_type == "multiselect":
+                        unique_values = df[col_name].unique().tolist()
+                        selected = st.multiselect(
+                            col_name,
+                            options=unique_values,
+                            default=unique_values,
+                            key=f"{key_prefix}_{col_name}"
+                        )
+                        if selected:
+                            df = df[df[col_name].isin(selected)]
+
+                    elif filter_type == "text":
+                        search_text = st.text_input(
+                            col_name,
+                            key=f"{key_prefix}_{col_name}"
+                        )
+                        if search_text:
+                            df = df[df[col_name].astype(str).str.contains(search_text, case=False, na=False)]
+
+                    elif filter_type == "date_range":
+                        if pd.api.types.is_datetime64_any_dtype(df[col_name]):
+                            min_date = df[col_name].min()
+                            max_date = df[col_name].max()
+                            date_range = st.date_input(
+                                col_name,
+                                value=(min_date, max_date),
+                                key=f"{key_prefix}_{col_name}"
+                            )
+                            if len(date_range) == 2:
+                                df = df[(df[col_name] >= pd.Timestamp(date_range[0])) &
+                                       (df[col_name] <= pd.Timestamp(date_range[1]))]
+
+    st.dataframe(df, width="stretch")
+
+    return df
+
+
+def display_table_with_actions(
+    df: pd.DataFrame,
+    actions: List[dict],
+    row_id_column: str = "id",
+    key_prefix: str = "action"
+):
+    """Display a table with action buttons for each row
+
+    actions: List of dicts with keys: 'label', 'callback', 'icon' (optional)
+    """
+
+    if df.empty:
+        st.info("No data available")
+        return
+
+    for idx, row in df.iterrows():
+        with st.container():
+            # Display row data in columns
+            data_cols = st.columns([3] + [1] * len(actions))
+
+            with data_cols[0]:
+                st.write(row.to_dict())
+
+            # Action buttons
+            for action_idx, action in enumerate(actions):
+                with data_cols[action_idx + 1]:
+                    icon = action.get('icon', '')
+                    button_label = f"{icon} {action['label']}" if icon else action['label']
+
+                    if st.button(
+                        button_label,
+                        key=f"{key_prefix}_{row[row_id_column]}_{action_idx}"
+                    ):
+                        action['callback'](row)
+
+            st.divider()
+
+
+def display_expandable_table(
+    df: pd.DataFrame,
+    summary_columns: List[str],
+    detail_callback: Callable[[Any], None],
+    row_id_column: str = "id",
+    key_prefix: str = "expand"
+):
+    """Display a table where each row can be expanded for details"""
+
+    if df.empty:
+        st.info("No data available")
+        return
+
+    for idx, row in df.iterrows():
+        # Summary view
+        summary_data = {col: row[col] for col in summary_columns if col in row}
+        summary_text = " | ".join([f"{k}: {v}" for k, v in summary_data.items()])
+
+        with st.expander(summary_text, expanded=False):
+            detail_callback(row)
+
+
+def export_dataframe(
+    df: pd.DataFrame,
+    filename: str = "data",
+    formats: List[str] = ["csv", "json"],
+    key_prefix: str = "export"
+):
+    """Provide export buttons for a dataframe"""
+
+    if df.empty:
+        return
+
+    cols = st.columns(len(formats))
+
+    for idx, fmt in enumerate(formats):
+        with cols[idx]:
+            if fmt == "csv":
+                csv = df.to_csv(index=False).encode('utf-8')
+                st.download_button(
+                    label="📥 Download CSV",
+                    data=csv,
+                    file_name=f"{filename}.csv",
+                    mime="text/csv",
+                    key=f"{key_prefix}_csv"
+                )
+            elif fmt == "json":
+                json_str = df.to_json(orient='records', indent=2)
+                st.download_button(
+                    label="📥 Download JSON",
+                    data=json_str,
+                    file_name=f"{filename}.json",
+                    mime="application/json",
+                    key=f"{key_prefix}_json"
+                )
+            elif fmt == "excel":
+                # Requires openpyxl
+                try:
+                    buffer = BytesIO()
+                    df.to_excel(buffer, index=False, engine='openpyxl')
+                    st.download_button(
+                        label="📥 Download Excel",
+                        data=buffer.getvalue(),
+                        file_name=f"{filename}.xlsx",
+                        mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                        key=f"{key_prefix}_excel"
+                    )
+                except ImportError:
+                    st.warning("Excel export requires openpyxl package")

mcli/ml/dashboard/pages/__init__.py

@@ -0,0 +1,6 @@
+"""Dashboard page modules"""
+
+from .cicd import show_cicd_dashboard
+from .workflows import show_workflows_dashboard
+
+__all__ = ["show_cicd_dashboard", "show_workflows_dashboard"]

mcli/ml/dashboard/pages/cicd.py

@@ -0,0 +1,382 @@
+"""CI/CD Pipeline Monitoring Dashboard"""
+
+import streamlit as st
+import pandas as pd
+import requests
+import os
+from datetime import datetime, timedelta
+import plotly.graph_objects as go
+import plotly.express as px
+from typing import Optional
+
+# Import components
+try:
+    from ..components.metrics import display_kpi_row, display_status_badge, display_health_indicator
+    from ..components.charts import create_timeline_chart, create_status_pie_chart, render_chart
+    from ..components.tables import display_filterable_dataframe, export_dataframe
+except ImportError:
+    # Fallback for when imported outside package context
+    from components.metrics import display_kpi_row, display_status_badge, display_health_indicator
+    from components.charts import create_timeline_chart, create_status_pie_chart, render_chart
+    from components.tables import display_filterable_dataframe, export_dataframe
+
+
+def get_cicd_api_url() -> str:
+    """Get CI/CD API URL from environment"""
+    lsh_url = os.getenv("LSH_API_URL", "http://localhost:3034")
+    return f"{lsh_url}/api/cicd"
+
+
+def fetch_cicd_builds(limit: int = 100) -> pd.DataFrame:
+    """Fetch CI/CD build data from API"""
+    try:
+        api_url = get_cicd_api_url()
+        response = requests.get(f"{api_url}/builds", params={"limit": limit}, timeout=5)
+        response.raise_for_status()
+
+        builds = response.json().get("builds", [])
+        if builds:
+            return pd.DataFrame(builds)
+
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404:
+            # API endpoint not implemented yet - use demo data silently
+            pass
+        else:
+            st.warning(f"Could not fetch CI/CD data: {e}")
+    except requests.exceptions.ConnectionError:
+        st.warning("⚠️ LSH Daemon connection failed. Using demo data.")
+    except Exception as e:
+        # Only show warning for unexpected errors
+        st.warning(f"Could not fetch CI/CD data: {e}")
+
+    # Return mock data for demonstration
+    return create_mock_cicd_data()
+
+
+def create_mock_cicd_data() -> pd.DataFrame:
+    """Create mock CI/CD data for demonstration"""
+    import random
+    from datetime import datetime, timedelta
+
+    pipelines = ["main-build", "develop-build", "feature-test", "release-deploy", "hotfix-deploy"]
+    statuses = ["success", "failed", "running", "cancelled"]
+    branches = ["main", "develop", "feature/new-dashboard", "release/v1.2.0", "hotfix/bug-123"]
+
+    data = []
+    for i in range(50):
+        start_time = datetime.now() - timedelta(days=random.randint(0, 30), hours=random.randint(0, 23))
+        duration = random.randint(60, 600)  # seconds
+        status = random.choices(statuses, weights=[70, 15, 10, 5])[0]
+
+        data.append({
+            "id": f"build-{i+1}",
+            "pipeline_name": random.choice(pipelines),
+            "branch": random.choice(branches),
+            "status": status,
+            "started_at": start_time.isoformat(),
+            "duration_sec": duration if status != "running" else None,
+            "commit_sha": f"{random.randint(1000000, 9999999):07x}",
+            "triggered_by": random.choice(["github-webhook", "manual", "schedule"]),
+            "success_rate": random.uniform(0.7, 1.0) if status == "success" else random.uniform(0, 0.5)
+        })
+
+    return pd.DataFrame(data)
+
+
+def fetch_webhooks() -> list:
+    """Fetch configured webhooks"""
+    try:
+        api_url = get_cicd_api_url()
+        response = requests.get(f"{api_url}/webhooks", timeout=5)
+        response.raise_for_status()
+        return response.json().get("webhooks", [])
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 404:
+            # API endpoint not implemented yet - use demo data silently
+            pass
+        else:
+            st.warning(f"Could not fetch webhooks: {e}")
+    except requests.exceptions.ConnectionError:
+        st.warning("⚠️ LSH Daemon connection failed. Using demo data.")
+    except Exception as e:
+        st.warning(f"Could not fetch webhooks: {e}")
+
+    # Return mock data
+    return [
+        {"id": "wh-1", "name": "GitHub Main", "url": "https://github.com/user/repo", "events": ["push", "pull_request"], "active": True},
+        {"id": "wh-2", "name": "GitLab CI", "url": "https://gitlab.com/user/repo", "events": ["push"], "active": True},
+    ]
+
+
+def show_cicd_dashboard():
+    """Main CI/CD dashboard page"""
+
+    st.title("🔧 CI/CD Pipeline Dashboard")
+    st.markdown("Monitor build pipelines, deployments, and CI/CD metrics")
+
+    # Refresh button
+    col1, col2, col3 = st.columns([1, 1, 8])
+    with col1:
+        if st.button("🔄 Refresh"):
+            st.rerun()
+    with col2:
+        auto_refresh = st.checkbox("Auto-refresh", value=False)
+
+    if auto_refresh:
+        import time
+        time.sleep(5)
+        st.rerun()
+
+    st.divider()
+
+    # Fetch data
+    with st.spinner("Loading CI/CD data..."):
+        builds_df = fetch_cicd_builds()
+
+    if builds_df.empty:
+        st.warning("No CI/CD build data available")
+        return
+
+    # Convert timestamps
+    if "started_at" in builds_df.columns:
+        builds_df["started_at"] = pd.to_datetime(builds_df["started_at"])
+
+    # === KPIs ===
+    st.subheader("📊 Pipeline Metrics")
+
+    total_builds = len(builds_df)
+    success_builds = len(builds_df[builds_df["status"] == "success"])
+    failed_builds = len(builds_df[builds_df["status"] == "failed"])
+    running_builds = len(builds_df[builds_df["status"] == "running"])
+
+    success_rate = (success_builds / total_builds * 100) if total_builds > 0 else 0
+    avg_duration = builds_df[builds_df["duration_sec"].notna()]["duration_sec"].mean()
+
+    metrics = {
+        "Total Builds": {"value": total_builds, "icon": "📦"},
+        "Success Rate": {"value": f"{success_rate:.1f}%", "delta": "+5.2%", "delta_color": "normal", "icon": "✅"},
+        "Failed Builds": {"value": failed_builds, "icon": "❌"},
+        "Running": {"value": running_builds, "icon": "🔵"},
+        "Avg Duration": {"value": f"{avg_duration:.0f}s" if pd.notna(avg_duration) else "N/A", "icon": "⏱️"}
+    }
+
+    display_kpi_row(metrics, columns=5)
+
+    st.divider()
+
+    # === Tabs for different views ===
+    tab1, tab2, tab3, tab4 = st.tabs(["📈 Overview", "🔍 Build History", "🔔 Webhooks", "⚙️ Configuration"])
+
+    with tab1:
+        show_cicd_overview(builds_df)
+
+    with tab2:
+        show_build_history(builds_df)
+
+    with tab3:
+        show_webhooks_config()
+
+    with tab4:
+        show_cicd_configuration()
+
+
+def show_cicd_overview(builds_df: pd.DataFrame):
+    """Show CI/CD overview charts"""
+
+    col1, col2 = st.columns(2)
+
+    with col1:
+        st.markdown("### Status Distribution")
+        if "status" in builds_df.columns:
+            fig = create_status_pie_chart(builds_df, "status", "Build Status Distribution")
+            render_chart(fig)
+
+    with col2:
+        st.markdown("### Pipeline Activity")
+        if "pipeline_name" in builds_df.columns:
+            pipeline_counts = builds_df["pipeline_name"].value_counts().head(10)
+            fig = px.bar(
+                x=pipeline_counts.values,
+                y=pipeline_counts.index,
+                orientation='h',
+                title="Top Pipelines by Build Count",
+                labels={"x": "Number of Builds", "y": "Pipeline"}
+            )
+            render_chart(fig)
+
+    # Success rate trend
+    st.markdown("### 📊 Success Rate Trend")
+
+    if "started_at" in builds_df.columns and "status" in builds_df.columns:
+        # Group by date and calculate success rate
+        builds_df["date"] = builds_df["started_at"].dt.date
+        daily_stats = builds_df.groupby("date").agg({
+            "status": lambda x: (x == "success").sum() / len(x) * 100
+        }).reset_index()
+        daily_stats.columns = ["date", "success_rate"]
+
+        fig = px.line(
+            daily_stats,
+            x="date",
+            y="success_rate",
+            title="Daily Success Rate",
+            labels={"date": "Date", "success_rate": "Success Rate (%)"},
+            markers=True
+        )
+        fig.add_hline(y=90, line_dash="dash", line_color="green", annotation_text="Target: 90%")
+        render_chart(fig)
+
+    # Build duration trend
+    st.markdown("### ⏱️ Build Duration Trend")
+
+    if "duration_sec" in builds_df.columns:
+        duration_data = builds_df[builds_df["duration_sec"].notna()].copy()
+
+        if not duration_data.empty:
+            duration_data["duration_min"] = duration_data["duration_sec"] / 60
+
+            fig = px.scatter(
+                duration_data,
+                x="started_at",
+                y="duration_min",
+                color="pipeline_name",
+                title="Build Duration Over Time",
+                labels={"started_at": "Time", "duration_min": "Duration (minutes)"}
+            )
+            render_chart(fig)
+
+
+def show_build_history(builds_df: pd.DataFrame):
+    """Show detailed build history"""
+
+    st.markdown("### Build History")
+
+    # Filters
+    filter_config = {
+        "pipeline_name": "multiselect",
+        "status": "multiselect",
+        "branch": "multiselect"
+    }
+
+    filtered_df = display_filterable_dataframe(
+        builds_df,
+        filter_columns=filter_config,
+        key_prefix="cicd_filter"
+    )
+
+    # Export option
+    st.markdown("#### 📥 Export Data")
+    export_dataframe(filtered_df, filename="cicd_builds", formats=["csv", "json"])
+
+    # Build details expander
+    st.markdown("#### Build Details")
+
+    if not filtered_df.empty:
+        for _, build in filtered_df.head(20).iterrows():
+            with st.expander(f"{build.get('pipeline_name', 'Unknown')} - {build.get('commit_sha', 'Unknown')[:7]} - {display_status_badge(build.get('status', 'unknown'), 'small')}"):
+                col1, col2 = st.columns(2)
+
+                with col1:
+                    st.markdown(f"**Pipeline:** {build.get('pipeline_name', 'N/A')}")
+                    st.markdown(f"**Branch:** {build.get('branch', 'N/A')}")
+                    st.markdown(f"**Commit:** `{build.get('commit_sha', 'N/A')}`")
+                    st.markdown(f"**Triggered By:** {build.get('triggered_by', 'N/A')}")
+
+                with col2:
+                    st.markdown(f"**Status:** {display_status_badge(build.get('status', 'unknown'), 'small')}")
+                    st.markdown(f"**Started:** {build.get('started_at', 'N/A')}")
+                    if pd.notna(build.get('duration_sec')):
+                        st.markdown(f"**Duration:** {build['duration_sec']}s ({build['duration_sec']/60:.1f}m)")
+
+                # Mock logs
+                if st.button(f"View Logs", key=f"logs_{build.get('id')}"):
+                    st.code(f"""
+[INFO] Starting build for {build.get('pipeline_name')}
+[INFO] Checking out branch: {build.get('branch')}
+[INFO] Installing dependencies...
+[INFO] Running tests...
+[INFO] Build {'completed successfully' if build.get('status') == 'success' else 'failed'}
+                    """, language="bash")
+
+
+def show_webhooks_config():
+    """Show webhook configuration"""
+
+    st.markdown("### 🔔 Configured Webhooks")
+
+    webhooks = fetch_webhooks()
+
+    if not webhooks:
+        st.info("No webhooks configured")
+        return
+
+    for webhook in webhooks:
+        with st.expander(f"{webhook['name']} - {'✅ Active' if webhook['active'] else '❌ Inactive'}"):
+            st.markdown(f"**URL:** `{webhook['url']}`")
+            st.markdown(f"**Events:** {', '.join(webhook['events'])}")
+            st.markdown(f"**Status:** {'Active' if webhook['active'] else 'Inactive'}")
+
+            col1, col2, col3 = st.columns(3)
+            with col1:
+                if st.button("🔄 Test", key=f"test_{webhook['id']}"):
+                    st.success("Webhook test triggered!")
+            with col2:
+                if st.button("✏️ Edit", key=f"edit_{webhook['id']}"):
+                    st.info("Edit functionality coming soon")
+            with col3:
+                if st.button("🗑️ Delete", key=f"delete_{webhook['id']}"):
+                    st.warning("Delete confirmation required")
+
+    st.divider()
+
+    # Add new webhook
+    with st.expander("➕ Add New Webhook"):
+        name = st.text_input("Webhook Name")
+        url = st.text_input("Webhook URL")
+        events = st.multiselect("Events", ["push", "pull_request", "release", "tag"])
+
+        if st.button("Create Webhook"):
+            if name and url and events:
+                st.success(f"Webhook '{name}' created successfully!")
+            else:
+                st.error("Please fill in all fields")
+
+
+def show_cicd_configuration():
+    """Show CI/CD configuration options"""
+
+    st.markdown("### ⚙️ CI/CD Configuration")
+
+    with st.form("cicd_config"):
+        st.markdown("#### Pipeline Settings")
+
+        max_concurrent_builds = st.number_input("Max Concurrent Builds", min_value=1, max_value=10, value=3)
+        build_timeout = st.number_input("Build Timeout (minutes)", min_value=5, max_value=120, value=30)
+        retry_failed_builds = st.checkbox("Auto-retry Failed Builds", value=True)
+        max_retries = st.number_input("Max Retries", min_value=1, max_value=5, value=2)
+
+        st.markdown("#### Notifications")
+        notify_on_success = st.checkbox("Notify on Success", value=False)
+        notify_on_failure = st.checkbox("Notify on Failure", value=True)
+        notification_email = st.text_input("Notification Email")
+
+        submitted = st.form_submit_button("Save Configuration")
+
+        if submitted:
+            st.success("✅ Configuration saved successfully!")
+            st.json({
+                "max_concurrent_builds": max_concurrent_builds,
+                "build_timeout_minutes": build_timeout,
+                "retry_failed_builds": retry_failed_builds,
+                "max_retries": max_retries,
+                "notifications": {
+                    "on_success": notify_on_success,
+                    "on_failure": notify_on_failure,
+                    "email": notification_email
+                }
+            })
+
+
+if __name__ == "__main__":
+    show_cicd_dashboard()