solace-agent-mesh 0.2.4__py3-none-any.whl → 1.0.2__py3-none-any.whl

solace_agent_mesh/evaluation/report/chart_section.html

@@ -0,0 +1,141 @@
+<div class="section">
+    <h2>📈 Performance Results</h2>
+    <div class="chart-container" style="position: relative; height: 500px; margin: 20px 0;">
+        <canvas id="performanceChart"></canvas>
+    </div>
+    
+    <!-- Model Execution Times -->
+    <div class="execution-times-section" style="margin-top: 30px;">
+        <h3 style="color: #1f2937; margin-bottom: 15px; font-size: 1.2rem;">Model Execution Times</h3>
+        <div class="execution-times-grid" style="display: grid; grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); gap: 15px;">
+            {model_execution_times}
+        </div>
+    </div>
+    
+</div>
+
+<!-- Chart.js Library -->
+<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+
+<!-- Chart Data and Configuration -->
+<script>
+// Chart data from backend
+const CATEGORIES_DATA = {categories_data};
+const CHART_DATASETS_DATA = {chart_datasets_data};
+
+// Chart.js configuration for benchmark report - Grouped Bar Chart
+const ctx = document.getElementById('performanceChart').getContext('2d');
+const chart = new Chart(ctx, {
+    type: 'bar',
+    data: {
+        labels: CATEGORIES_DATA,
+        datasets: CHART_DATASETS_DATA
+    },
+    options: {
+        responsive: true,
+        maintainAspectRatio: false,
+        interaction: {
+            mode: 'index',
+            intersect: false,
+        },
+        plugins: {
+            title: {
+                display: true,
+                text: 'LLM Evaluation Scores by Category',
+                font: {
+                    size: 20,
+                    weight: 'bold'
+                },
+                color: '#000000',
+                padding: {
+                    top: 10,
+                    bottom: 30
+                }
+            },
+            legend: {
+                display: true,
+                position: 'top',
+                labels: {
+                    color: '#000000',
+                    padding: 20,
+                    usePointStyle: true,
+                    font: {
+                        size: 12
+                    }
+                }
+            },
+            tooltip: {
+                backgroundColor: 'rgba(0, 0, 0, 0.8)',
+                titleColor: '#ffffff',
+                bodyColor: '#ffffff',
+                borderColor: '#ffffff',
+                borderWidth: 1,
+                callbacks: {
+                    label: function(context) {
+                        const score = context.parsed.y;
+                        const percentage = (score * 100).toFixed(1);
+                        return `${context.dataset.label}: ${score.toFixed(3)} (${percentage}%)`;
+                    }
+                }
+            }
+        },
+        scales: {
+            y: {
+                beginAtZero: true,
+                max: 1.0,
+                title: {
+                    display: true,
+                    text: 'Average LLM Evaluation Score',
+                    color: '#000000',
+                    font: {
+                        size: 16,
+                        weight: 'bold'
+                    }
+                },
+                ticks: {
+                    stepSize: 0.1,
+                    color: '#000000',
+                    font: {
+                        size: 12
+                    },
+                    callback: function(value) {
+                        return value.toFixed(1);
+                    }
+                },
+                grid: {
+                    color: 'rgba(0, 0, 0, 0.1)',
+                    lineWidth: 1
+                }
+            },
+            x: {
+                title: {
+                    display: true,
+                    text: 'Task Categories',
+                    color: '#000000',
+                    font: {
+                        size: 16,
+                        weight: 'bold'
+                    }
+                },
+                ticks: {
+                    color: '#000000',
+                    font: {
+                        size: 12,
+                        weight: 'bold'
+                    }
+                },
+                grid: {
+                    display: false
+                }
+            }
+        },
+        elements: {
+            bar: {
+                borderWidth: 1,
+                borderRadius: 4,
+                borderSkipped: false
+            }
+        }
+    }
+});
+</script>

solace_agent_mesh/evaluation/report/detailed_breakdown.html

@@ -0,0 +1,28 @@
+<div class="section">
+    <h2>📋 Detailed Test Breakdown</h2>
+    <div class="breakdown-container">
+        {{BREAKDOWN_CONTENT}}
+    </div>
+</div>
+
+<script>
+// JavaScript for category dropdown functionality
+document.addEventListener('DOMContentLoaded', function() {
+    const categoryHeaders = document.querySelectorAll('.category-header');
+    
+    categoryHeaders.forEach(header => {
+        header.addEventListener('click', function() {
+            const content = this.nextElementSibling;
+            const toggle = this.querySelector('.category-toggle');
+            
+            if (content.classList.contains('active')) {
+                content.classList.remove('active');
+                toggle.textContent = '▶';
+            } else {
+                content.classList.add('active');
+                toggle.textContent = '▼';
+            }
+        });
+    });
+});
+</script>

solace_agent_mesh/evaluation/report/modal.html

@@ -0,0 +1,59 @@
+<!-- Modal for test details -->
+<div id="testModal" class="modal">
+    <div class="modal-content">
+        <div class="modal-header">
+            <h2 class="modal-title" id="modalTitle">Test Details</h2>
+            <button class="modal-close" onclick="closeModal()">&times;</button>
+        </div>
+        <div class="modal-description" id="modalDescription"></div>
+        <div class="modal-charts">
+            <div class="modal-chart-container">
+                <div class="modal-chart-header">
+                    <div class="modal-chart-title">Response Scores by Model</div>
+                    <button class="chart-toggle-btn" onclick="toggleChart('response')">Show Quartiles</button>
+                </div>
+                <div class="chart-flip-container" id="responseChartContainer">
+                    <div class="chart-face chart-front">
+                        <canvas id="responseChart"></canvas>
+                    </div>
+                    <div class="chart-face chart-back">
+                        <canvas id="responseQuartileChart"></canvas>
+                    </div>
+                </div>
+            </div>
+            <div class="modal-chart-container">
+                <div class="modal-chart-header">
+                    <div class="modal-chart-title">Tool Scores by Model</div>
+                    <button class="chart-toggle-btn" onclick="toggleChart('tool')">Show Quartiles</button>
+                </div>
+                <div class="chart-flip-container" id="toolChartContainer">
+                    <div class="chart-face chart-front">
+                        <canvas id="toolChart"></canvas>
+                    </div>
+                    <div class="chart-face chart-back">
+                        <canvas id="toolQuartileChart"></canvas>
+                    </div>
+                </div>
+            </div>
+        </div>
+        <div class="modal-runs-section">
+            <div class="runs-section-header">
+                <h3 class="runs-section-title">Individual Run Details</h3>
+                <div class="runs-controls">
+                    <div class="runs-filter">
+                        <label for="modelFilter">Filter by Model:</label>
+                        <select id="modelFilter" onchange="filterRuns()">
+                            <option value="all">All Models</option>
+                        </select>
+                    </div>
+                    <div class="runs-count">
+                        <span id="runsCount">Showing all runs</span>
+                    </div>
+                </div>
+            </div>
+            <div id="runsContainer" class="runs-container">
+                <!-- Individual runs will be populated here -->
+            </div>
+        </div>
+    </div>
+</div>

solace_agent_mesh/evaluation/report/modal_chart_functions.js

@@ -0,0 +1,411 @@
+// Chart creation functions for modal
+
+function createResponseChart(testData) {
+    const ctx = document.getElementById('responseChart').getContext('2d');
+    
+    const models = Object.keys(testData.model_scores);
+    const responseScores = models.map(model => testData.model_scores[model]);
+    
+    responseChart = new Chart(ctx, {
+        type: 'bar',
+        data: {
+            labels: models.map(model => model),
+            datasets: [{
+                label: 'Response Score',
+                data: responseScores,
+                backgroundColor: responseScores.map(score => getScoreColor(score)),
+                borderColor: responseScores.map(score => getScoreColor(score)),
+                borderWidth: 1
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            scales: {
+                y: {
+                    beginAtZero: true,
+                    max: 1,
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 12,
+                            weight: 'bold'
+                        }
+                    }
+                },
+                x: {
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 10,
+                            weight: 'normal'
+                        }
+                    }
+                }
+            },
+            plugins: {
+                legend: {
+                    display: false
+                }
+            }
+        }
+    });
+}
+
+function createToolChart(testData) {
+    const ctx = document.getElementById('toolChart').getContext('2d');
+    
+    const models = Object.keys(testData.model_scores);
+    const toolScores = models.map(model => {
+        // Use actual tool scores if available, otherwise generate mock data
+        if (testData.tool_scores && testData.tool_scores[model] !== undefined) {
+            return testData.tool_scores[model];
+        } else {
+            // Generate mock tool scores based on response scores with some variation
+            const responseScore = testData.model_scores[model];
+            return Math.max(0, Math.min(1, responseScore + (Math.random() - 0.5) * 0.3));
+        }
+    });
+    
+    toolChart = new Chart(ctx, {
+        type: 'bar',
+        data: {
+            labels: models.map(model => model),
+            datasets: [{
+                label: 'Tool Score',
+                data: toolScores,
+                backgroundColor: toolScores.map(score => getScoreColor(score)),
+                borderColor: toolScores.map(score => getScoreColor(score)),
+                borderWidth: 1
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            scales: {
+                y: {
+                    beginAtZero: true,
+                    max: 1,
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 12,
+                            weight: 'bold'
+                        }
+                    }
+                },
+                x: {
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 10,
+                            weight: 'normal'
+                        }
+                    }
+                }
+            },
+            plugins: {
+                legend: {
+                    display: false
+                }
+            }
+        }
+    });
+}
+
+function createResponseQuartileChart(testData) {
+    const ctx = document.getElementById('responseQuartileChart').getContext('2d');
+    
+    const models = Object.keys(testData.model_scores);
+    const responseScores = models.map(model => testData.model_scores[model]);
+    
+    // Create custom boxplot data
+    const boxplotData = models.map((model, index) => {
+        // Check if we have individual run data for this model
+        const individualRuns = testData.individual_runs && testData.individual_runs[model];
+        
+        if (individualRuns && individualRuns.length > 1) {
+            // Use actual individual run data to calculate real quartiles
+            const scores = individualRuns.map(run => run.response_score).sort((a, b) => a - b);
+            const quartileData = calculateActualQuartiles(scores);
+            
+            return {
+                x: index,
+                min: quartileData.min,
+                q1: quartileData.q1,
+                median: quartileData.median,
+                q3: quartileData.q3,
+                max: quartileData.max,
+                outliers: quartileData.outliers
+            };
+        } else {
+            // Single data point - show as a line (all values the same)
+            const score = testData.model_scores[model];
+            return {
+                x: index,
+                min: score,
+                q1: score,
+                median: score,
+                q3: score,
+                max: score,
+                outliers: []
+            };
+        }
+    });
+    
+    responseQuartileChart = new Chart(ctx, {
+        type: 'boxplot',
+        data: {
+            labels: models,
+            datasets: [{
+                label: 'Response Score Distribution',
+                data: boxplotData,
+                backgroundColor: responseScores.map(score => getScoreColor(score) + '80'),
+                borderColor: responseScores.map(score => getScoreColor(score)),
+                borderWidth: 2,
+                outlierColor: '#999999',
+                outlierRadius: 3
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            scales: {
+                y: {
+                    beginAtZero: true,
+                    max: 1,
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 12,
+                            weight: 'bold'
+                        }
+                    }
+                },
+                x: {
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 10,
+                            weight: 'normal'
+                        }
+                    }
+                }
+            },
+            plugins: {
+                legend: {
+                    display: false
+                },
+                tooltip: {
+                    callbacks: {
+                        label: function(context) {
+                            const data = context.parsed;
+                            if (data.min === data.max) {
+                                return `Single value: ${data.min.toFixed(3)}`;
+                            }
+                            return [
+                                `Min: ${data.min.toFixed(3)}`,
+                                `Q1: ${data.q1.toFixed(3)}`,
+                                `Median: ${data.median.toFixed(3)}`,
+                                `Q3: ${data.q3.toFixed(3)}`,
+                                `Max: ${data.max.toFixed(3)}`
+                            ];
+                        }
+                    }
+                }
+            }
+        }
+    });
+}
+
+function createToolQuartileChart(testData) {
+    const ctx = document.getElementById('toolQuartileChart').getContext('2d');
+    
+    const models = Object.keys(testData.model_scores);
+    const toolScores = models.map(model => {
+        if (testData.tool_scores && testData.tool_scores[model] !== undefined) {
+            return testData.tool_scores[model];
+        } else {
+            const responseScore = testData.model_scores[model];
+            return Math.max(0, Math.min(1, responseScore + (Math.random() - 0.5) * 0.3));
+        }
+    });
+    
+    // Create custom boxplot data
+    const boxplotData = models.map((model, index) => {
+        // Check if we have individual run data for this model
+        const individualRuns = testData.individual_runs && testData.individual_runs[model];
+        
+        if (individualRuns && individualRuns.length > 1) {
+            // Use actual individual run data to calculate real quartiles
+            const scores = individualRuns.map(run => run.tool_score).sort((a, b) => a - b);
+            const quartileData = calculateActualQuartiles(scores);
+            
+            return {
+                x: index,
+                min: quartileData.min,
+                q1: quartileData.q1,
+                median: quartileData.median,
+                q3: quartileData.q3,
+                max: quartileData.max,
+                outliers: quartileData.outliers
+            };
+        } else {
+            // Single data point - show as a line (all values the same)
+            const score = toolScores[index];
+            return {
+                x: index,
+                min: score,
+                q1: score,
+                median: score,
+                q3: score,
+                max: score,
+                outliers: []
+            };
+        }
+    });
+    
+    toolQuartileChart = new Chart(ctx, {
+        type: 'boxplot',
+        data: {
+            labels: models,
+            datasets: [{
+                label: 'Tool Score Distribution',
+                data: boxplotData,
+                backgroundColor: toolScores.map(score => getScoreColor(score) + '80'),
+                borderColor: toolScores.map(score => getScoreColor(score)),
+                borderWidth: 2,
+                outlierColor: '#999999',
+                outlierRadius: 3
+            }]
+        },
+        options: {
+            responsive: true,
+            maintainAspectRatio: false,
+            scales: {
+                y: {
+                    beginAtZero: true,
+                    max: 1,
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 12,
+                            weight: 'bold'
+                        }
+                    }
+                },
+                x: {
+                    ticks: {
+                        color: '#000',
+                        font: {
+                            size: 10,
+                            weight: 'normal'
+                        }
+                    }
+                }
+            },
+            plugins: {
+                legend: {
+                    display: false
+                },
+                tooltip: {
+                    callbacks: {
+                        label: function(context) {
+                            const data = context.parsed;
+                            if (data.min === data.max) {
+                                return `Single value: ${data.min.toFixed(3)}`;
+                            }
+                            return [
+                                `Min: ${data.min.toFixed(3)}`,
+                                `Q1: ${data.q1.toFixed(3)}`,
+                                `Median: ${data.median.toFixed(3)}`,
+                                `Q3: ${data.q3.toFixed(3)}`,
+                                `Max: ${data.max.toFixed(3)}`
+                            ];
+                        }
+                    }
+                }
+            }
+        }
+    });
+}
+
+function calculateActualQuartiles(sortedScores) {
+    const n = sortedScores.length;
+    
+    if (n === 0) {
+        return { min: 0, q1: 0, median: 0, q3: 0, max: 0, outliers: [] };
+    }
+    
+    if (n === 1) {
+        const value = sortedScores[0];
+        return { min: value, q1: value, median: value, q3: value, max: value, outliers: [] };
+    }
+    
+    // Calculate quartiles using the standard method
+    const min = sortedScores[0];
+    const max = sortedScores[n - 1];
+    
+    // Calculate median (Q2)
+    let median;
+    if (n % 2 === 0) {
+        median = (sortedScores[n / 2 - 1] + sortedScores[n / 2]) / 2;
+    } else {
+        median = sortedScores[Math.floor(n / 2)];
+    }
+    
+    // Calculate Q1 (first quartile)
+    const q1Index = Math.floor(n / 4);
+    let q1;
+    if (n % 4 === 0) {
+        q1 = (sortedScores[q1Index - 1] + sortedScores[q1Index]) / 2;
+    } else {
+        q1 = sortedScores[q1Index];
+    }
+    
+    // Calculate Q3 (third quartile)
+    const q3Index = Math.floor(3 * n / 4);
+    let q3;
+    if (n % 4 === 0) {
+        q3 = (sortedScores[q3Index - 1] + sortedScores[q3Index]) / 2;
+    } else {
+        q3 = sortedScores[q3Index];
+    }
+    
+    // For now, we'll not calculate outliers to keep it simple
+    // In a full implementation, outliers would be values outside 1.5 * IQR from Q1/Q3
+    
+    return {
+        min: min,
+        q1: q1,
+        median: median,
+        q3: q3,
+        max: max,
+        outliers: []
+    };
+}
+
+function getScoreColor(score) {
+    if (score >= 0.7) {
+        return '#27ae60'; // Green
+    } else if (score >= 0.4) {
+        return '#f39c12'; // Orange
+    } else {
+        return '#e74c3c'; // Red
+    }
+}
+
+// Close modal when clicking outside of it
+window.onclick = function(event) {
+    const modal = document.getElementById('testModal');
+    if (event.target === modal) {
+        closeModal();
+    }
+}
+
+// Close modal with Escape key
+document.addEventListener('keydown', function(event) {
+    if (event.key === 'Escape') {
+        closeModal();
+    }
+});