levelapp 0.1.15__py3-none-any.whl

levelapp/visualization/charts.py

@@ -0,0 +1,358 @@
+"""levelapp/visualization/charts.py: Chart generation for evaluation results."""
+
+import plotly.graph_objects as go
+import plotly.express as px
+from typing import Dict, Any
+from collections import defaultdict
+
+from levelapp.simulator.schemas import SimulationResults
+from levelapp.aspects import logger
+
+
+class ChartGenerator:
+    """Generate interactive charts for evaluation results."""
+
+    def __init__(self, theme: str = "plotly_white"):
+        """
+        Initialize ChartGenerator with a theme.
+
+        Args:
+            theme: Plotly theme name (plotly, plotly_white, plotly_dark, ggplot2, seaborn, etc.)
+        """
+        self.theme = theme
+        self._color_palette = px.colors.qualitative.Plotly
+
+    def create_score_trend(
+        self, results: SimulationResults, metric: str = "average"
+    ) -> go.Figure:
+        """
+        Create a line chart showing score trends across scripts/attempts.
+
+        Args:
+            results: SimulationResults object
+            metric: Which metric to plot (default: "average")
+
+        Returns:
+            Plotly Figure object
+        """
+        logger.info(f"[ChartGenerator] Creating score trend chart for metric: {metric}")
+
+        fig = go.Figure()
+
+        # Extract scores by provider
+        provider_scores = defaultdict(list)
+        script_ids = []
+
+        if results.script_results:
+            for idx, script_result in enumerate(results.script_results):
+                script_id = script_result.script_id
+                script_ids.append(script_id)
+
+                avg_scores = script_result.average_scores
+                for provider, score in avg_scores.items():
+                    if (
+                        provider != "processing_time"
+                        and provider != "guardrail"
+                        and provider != "metadata"
+                    ):
+                        provider_scores[provider].append(score)
+
+        # Create line for each provider
+        for idx, (provider, scores) in enumerate(provider_scores.items()):
+            fig.add_trace(
+                go.Scatter(
+                    x=script_ids[: len(scores)],
+                    y=scores,
+                    mode="lines+markers",
+                    name=provider.upper(),
+                    line=dict(
+                        width=2,
+                        color=self._color_palette[idx % len(self._color_palette)],
+                    ),
+                    marker=dict(size=8),
+                )
+            )
+
+        fig.update_layout(
+            title="Evaluation Score Trends Across Scripts",
+            xaxis_title="Script ID",
+            yaxis_title="Score",
+            template=self.theme,
+            hovermode="x unified",
+            legend=dict(
+                orientation="h", yanchor="bottom", y=1.02, xanchor="right", x=1
+            ),
+        )
+
+        return fig
+
+    def create_provider_comparison(self, results: SimulationResults) -> go.Figure:
+        """
+        Create a bar chart comparing average scores across providers.
+
+        Args:
+            results: SimulationResults object
+
+        Returns:
+            Plotly Figure object
+        """
+        logger.info("[ChartGenerator] Creating provider comparison chart")
+
+        providers = []
+        scores = []
+
+        if results.average_scores:
+            for provider, score in results.average_scores.items():
+                if provider not in ["processing_time", "guardrail", "metadata"]:
+                    providers.append(provider.upper())
+                    scores.append(score)
+
+        fig = go.Figure(
+            data=[
+                go.Bar(
+                    x=providers,
+                    y=scores,
+                    marker_color=self._color_palette[: len(providers)],
+                    text=[f"{s:.3f}" for s in scores],
+                    textposition="auto",
+                )
+            ]
+        )
+
+        fig.update_layout(
+            title="Average Scores by Provider",
+            xaxis_title="Provider",
+            yaxis_title="Average Score",
+            template=self.theme,
+            yaxis=dict(range=[0, 1]),
+        )
+
+        return fig
+
+    def create_score_distribution(
+        self, results: SimulationResults, provider: str
+    ) -> go.Figure:
+        """
+        Create a histogram and box plot showing score distribution for a provider.
+
+        Args:
+            results: SimulationResults object
+            provider: Provider name to analyze
+
+        Returns:
+            Plotly Figure object with subplots
+        """
+        logger.info(
+            f"[ChartGenerator] Creating score distribution for provider: {provider}"
+        )
+
+        from plotly.subplots import make_subplots
+
+        # Collect all scores for the provider
+        scores = []
+        if results.interaction_results:
+            for script_result in results.interaction_results:
+                avg_scores = script_result.get("average_scores", {})
+                if provider in avg_scores:
+                    scores.append(avg_scores[provider])
+
+        # Create subplots
+        fig = make_subplots(
+            rows=2,
+            cols=1,
+            subplot_titles=("Score Distribution", "Box Plot"),
+            vertical_spacing=0.15,
+        )
+
+        # Histogram
+        fig.add_trace(
+            go.Histogram(
+                x=scores,
+                nbinsx=20,
+                name="Distribution",
+                marker_color=self._color_palette[0],
+            ),
+            row=1,
+            col=1,
+        )
+
+        # Box plot
+        fig.add_trace(
+            go.Box(
+                x=scores,
+                name=provider.upper(),
+                marker_color=self._color_palette[1],
+                boxmean="sd",
+            ),
+            row=2,
+            col=1,
+        )
+
+        fig.update_layout(
+            title=f"Score Distribution for {provider.upper()}",
+            template=self.theme,
+            showlegend=False,
+            height=600,
+        )
+
+        fig.update_xaxes(title_text="Score", row=2, col=1)
+        fig.update_yaxes(title_text="Frequency", row=1, col=1)
+
+        return fig
+
+    def create_metadata_heatmap(self, comparator_results: Dict[str, Any]) -> go.Figure:
+        """
+        Create a heatmap showing field-level accuracy from comparator results.
+
+        Args:
+            comparator_results: Dictionary of comparator evaluation results
+
+        Returns:
+            Plotly Figure object
+        """
+        logger.info("[ChartGenerator] Creating metadata accuracy heatmap")
+
+        fields = []
+        scores = []
+
+        for idx, result in comparator_results.items():
+            field_name = result.get("field_name", f"Field {idx}")
+            set_scores = result.get("set_scores")
+
+            if set_scores is not None:
+                score = set_scores[0] if isinstance(set_scores, list) else set_scores
+                fields.append(field_name)
+                scores.append(float(score) if score is not None else 0.0)
+
+        fig = go.Figure(
+            data=go.Heatmap(
+                z=[scores],
+                x=fields,
+                y=["Accuracy"],
+                colorscale="RdYlGn",
+                text=[[f"{s:.2f}" for s in scores]],
+                texttemplate="%{text}",
+                textfont={"size": 10},
+                colorbar=dict(title="Score"),
+            )
+        )
+
+        fig.update_layout(
+            title="Metadata Field Accuracy Heatmap",
+            xaxis_title="Field Name",
+            template=self.theme,
+            height=300,
+        )
+
+        return fig
+
+    def create_interaction_timeline(self, results: SimulationResults) -> go.Figure:
+        """
+        Create a timeline visualization of interaction performance.
+
+        Args:
+            results: SimulationResults object
+
+        Returns:
+            Plotly Figure object
+        """
+        logger.info("[ChartGenerator] Creating interaction timeline")
+
+        fig = go.Figure()
+
+        if results.interaction_results:
+            for script_idx, script_result in enumerate(results.interaction_results):
+                script_id = script_result.get("script_id", f"Script {script_idx + 1}")
+                attempts = script_result.get("attempts", [])
+
+                for attempt in attempts:
+                    attempt_num = attempt.get("attempt", 1)
+                    duration = attempt.get("total_duration", 0)
+
+                    fig.add_trace(
+                        go.Bar(
+                            x=[duration],
+                            y=[f"{script_id} - Attempt {attempt_num}"],
+                            orientation="h",
+                            name=script_id,
+                            showlegend=False,
+                            marker_color=self._color_palette[
+                                script_idx % len(self._color_palette)
+                            ],
+                        )
+                    )
+
+        fig.update_layout(
+            title="Interaction Processing Timeline",
+            xaxis_title="Duration (seconds)",
+            yaxis_title="Script - Attempt",
+            template=self.theme,
+            height=max(400, len(results.interaction_results or []) * 60),
+        )
+
+        return fig
+
+    def create_summary_metrics(self, results: SimulationResults) -> go.Figure:
+        """
+        Create a summary metrics card visualization.
+
+        Args:
+            results: SimulationResults object
+
+        Returns:
+            Plotly Figure object
+        """
+        logger.info("[ChartGenerator] Creating summary metrics visualization")
+
+        from plotly.subplots import make_subplots
+
+        # Calculate summary metrics
+        total_scripts = len(results.interaction_results or [])
+        avg_score = (
+            results.average_scores.get("openai", 0) if results.average_scores else 0
+        )
+        total_time = results.elapsed_time
+
+        fig = make_subplots(
+            rows=1,
+            cols=3,
+            subplot_titles=("Total Scripts", "Avg Score", "Total Time (s)"),
+            specs=[
+                [{"type": "indicator"}, {"type": "indicator"}, {"type": "indicator"}]
+            ],
+        )
+
+        fig.add_trace(
+            go.Indicator(
+                mode="number",
+                value=total_scripts,
+                number={"font": {"size": 40}},
+            ),
+            row=1,
+            col=1,
+        )
+
+        fig.add_trace(
+            go.Indicator(
+                mode="number+delta",
+                value=avg_score,
+                number={"font": {"size": 40}},
+                delta={"reference": 0.8, "relative": False},
+            ),
+            row=1,
+            col=2,
+        )
+
+        fig.add_trace(
+            go.Indicator(
+                mode="number",
+                value=total_time,
+                number={"font": {"size": 40}, "suffix": "s"},
+            ),
+            row=1,
+            col=3,
+        )
+
+        fig.update_layout(template=self.theme, height=200, showlegend=False)
+
+        return fig

levelapp/visualization/dashboard.py

@@ -0,0 +1,240 @@
+"""levelapp/visualization/dashboard.py: Dashboard generation for evaluation results."""
+
+from pathlib import Path
+from typing import Dict, Any
+from datetime import datetime
+from jinja2 import Environment, FileSystemLoader, select_autoescape
+
+from levelapp.simulator.schemas import SimulationResults
+from levelapp.visualization.charts import ChartGenerator
+from levelapp.aspects import logger
+
+
+class DashboardGenerator:
+    """Generate comprehensive HTML dashboards for evaluation results."""
+
+    def __init__(self, template_dir: str | None = None):
+        """
+        Initialize DashboardGenerator.
+
+        Args:
+            template_dir: Optional custom template directory path
+        """
+        if template_dir is None:
+            # Use default templates directory
+            template_dir = Path(__file__).parent / "templates"
+
+        self.template_dir = Path(template_dir)
+        self.env = Environment(
+            loader=FileSystemLoader(str(self.template_dir)),
+            autoescape=select_autoescape(["html", "xml"]),
+        )
+        self.chart_gen = ChartGenerator()
+
+    def generate_simulator_dashboard(
+        self,
+        results: SimulationResults,
+        output_path: str,
+        title: str = "Evaluation Dashboard",
+    ) -> str:
+        """
+        Generate a complete HTML dashboard for simulator results.
+
+        Args:
+            results: SimulationResults object
+            output_path: Path to save the HTML file
+            title: Dashboard title
+
+        Returns:
+            Path to the generated HTML file
+        """
+        logger.info(
+            f"[DashboardGenerator] Generating simulator dashboard: {output_path}"
+        )
+
+        # Generate all charts
+        charts = {
+            "score_trend": self.chart_gen.create_score_trend(results),
+            "provider_comparison": self.chart_gen.create_provider_comparison(results),
+            "summary_metrics": self.chart_gen.create_summary_metrics(results),
+        }
+
+        # Add distribution charts for each provider
+        if results.average_scores:
+            for provider in results.average_scores.keys():
+                if provider not in ["processing_time", "guardrail", "metadata"]:
+                    try:
+                        charts[f"distribution_{provider}"] = (
+                            self.chart_gen.create_score_distribution(results, provider)
+                        )
+                    except Exception as e:
+                        logger.warning(
+                            f"Failed to create distribution chart for {provider}: {e}"
+                        )
+
+        # Add timeline chart
+        try:
+            charts["timeline"] = self.chart_gen.create_interaction_timeline(results)
+        except Exception as e:
+            logger.warning(f"Failed to create timeline chart: {e}")
+
+        # Convert charts to HTML
+        chart_htmls = {}
+        for name, fig in charts.items():
+            chart_htmls[name] = fig.to_html(
+                include_plotlyjs="cdn",
+                div_id=f"chart_{name}",
+                config={"responsive": True},
+            )
+
+        # Calculate summary statistics
+        summary_stats = self._create_summary_stats(results)
+
+        # Prepare context for template
+        context = {
+            "title": title,
+            "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            "summary_stats": summary_stats,
+            "charts": chart_htmls,
+            "results": results,
+            "has_evaluation_summary": bool(results.evaluation_summary),
+        }
+
+        # Render template
+        html_content = self._render_template("simulator_dashboard.html", context)
+
+        # Save to file
+        output_file = Path(output_path)
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        output_file.write_text(html_content, encoding="utf-8")
+
+        logger.info(f"[DashboardGenerator] Dashboard saved to: {output_path}")
+        return str(output_file.absolute())
+
+    def generate_comparator_dashboard(
+        self,
+        results: Dict[str, Any],
+        output_path: str,
+        title: str = "Comparison Dashboard",
+    ) -> str:
+        """
+        Generate a complete HTML dashboard for comparator results.
+
+        Args:
+            results: Comparator results dictionary
+            output_path: Path to save the HTML file
+            title: Dashboard title
+
+        Returns:
+            Path to the generated HTML file
+        """
+        logger.info(
+            f"[DashboardGenerator] Generating comparator dashboard: {output_path}"
+        )
+
+        # Generate heatmap
+        heatmap = self.chart_gen.create_metadata_heatmap(results)
+
+        # Convert to HTML
+        chart_html = heatmap.to_html(
+            include_plotlyjs="cdn", div_id="chart_heatmap", config={"responsive": True}
+        )
+
+        # Prepare context
+        context = {
+            "title": title,
+            "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+            "chart_html": chart_html,
+            "results": results,
+        }
+
+        # Render template
+        html_content = self._render_template("comparator_dashboard.html", context)
+
+        # Save to file
+        output_file = Path(output_path)
+        output_file.parent.mkdir(parents=True, exist_ok=True)
+        output_file.write_text(html_content, encoding="utf-8")
+
+        logger.info(f"[DashboardGenerator] Dashboard saved to: {output_path}")
+        return str(output_file.absolute())
+
+    def _create_summary_stats(self, results: SimulationResults) -> Dict[str, Any]:
+        """
+        Extract key summary statistics from results.
+
+        Args:
+            results: SimulationResults object
+
+        Returns:
+            Dictionary of summary statistics
+        """
+        stats = {
+            "total_scripts": len(results.interaction_results or []),
+            "total_time": results.elapsed_time,
+            "started_at": results.started_at.strftime("%Y-%m-%d %H:%M:%S"),
+            "finished_at": results.finished_at.strftime("%Y-%m-%d %H:%M:%S"),
+            "average_scores": results.average_scores or {},
+            "providers": list(results.average_scores.keys())
+            if results.average_scores
+            else [],
+        }
+
+        # Calculate overall average (excluding non-score metrics)
+        score_values = [
+            v
+            for k, v in (results.average_scores or {}).items()
+            if k not in ["processing_time", "guardrail", "metadata"]
+        ]
+        stats["overall_average"] = (
+            sum(score_values) / len(score_values) if score_values else 0.0
+        )
+
+        return stats
+
+    def _render_template(self, template_name: str, context: Dict[str, Any]) -> str:
+        """
+        Render a Jinja2 template with the given context.
+
+        Args:
+            template_name: Name of the template file
+            context: Template context dictionary
+
+        Returns:
+            Rendered HTML string
+        """
+        try:
+            template = self.env.get_template(template_name)
+            return template.render(**context)
+        except Exception as e:
+            logger.error(f"[DashboardGenerator] Template rendering failed: {e}")
+            # Return a basic HTML fallback
+            return self._create_fallback_html(context)
+
+    def _create_fallback_html(self, context: Dict[str, Any]) -> str:
+        """Create a basic HTML fallback if template rendering fails."""
+        return f"""
+<!DOCTYPE html>
+<html>
+<head>
+    <title>{context.get("title", "Evaluation Dashboard")}</title>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <style>
+        body {{ font-family: Arial, sans-serif; margin: 20px; }}
+        .container {{ max-width: 1200px; margin: 0 auto; }}
+        h1 {{ color: #333; }}
+        .chart {{ margin: 20px 0; }}
+    </style>
+</head>
+<body>
+    <div class="container">
+        <h1>{context.get("title", "Evaluation Dashboard")}</h1>
+        <p>Generated at: {context.get("generated_at", "N/A")}</p>
+        <div class="charts">
+            {"".join(context.get("charts", {}).values())}
+        </div>
+    </div>
+</body>
+</html>
+"""