levelapp 0.1.15__py3-none-any.whl

levelapp/visualization/exporter.py

@@ -0,0 +1,167 @@
+"""levelapp/visualization/exporter.py: Export utilities for visualization results."""
+
+import zipfile
+from pathlib import Path
+from typing import Dict, List
+import plotly.graph_objects as go
+
+from levelapp.simulator.schemas import SimulationResults
+from levelapp.visualization.dashboard import DashboardGenerator
+from levelapp.aspects import logger
+
+
+class ResultsExporter:
+    """Export evaluation results and visualizations to multiple formats."""
+
+    def __init__(self, output_dir: str):
+        """
+        Initialize ResultsExporter.
+
+        Args:
+            output_dir: Directory to save exported files
+        """
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+    def export_chart(
+        self, figure: go.Figure, filename: str, formats: List[str] = ["html", "png"]
+    ) -> Dict[str, str]:
+        """
+        Export a single chart to multiple formats.
+
+        Args:
+            figure: Plotly Figure object
+            filename: Base filename (without extension)
+            formats: List of formats to export (html, png, pdf, json)
+
+        Returns:
+            Dictionary mapping format to file path
+        """
+        logger.info(
+            f"[ResultsExporter] Exporting chart '{filename}' to formats: {formats}"
+        )
+
+        exported_files = {}
+
+        for fmt in formats:
+            output_path = self.output_dir / f"{filename}.{fmt}"
+
+            try:
+                if fmt == "html":
+                    figure.write_html(
+                        str(output_path),
+                        include_plotlyjs="cdn",
+                        config={"responsive": True},
+                    )
+                elif fmt == "png":
+                    figure.write_image(str(output_path), width=1200, height=800)
+                elif fmt == "pdf":
+                    figure.write_image(str(output_path))
+                elif fmt == "json":
+                    figure.write_json(str(output_path))
+                else:
+                    logger.warning(f"Unsupported format: {fmt}")
+                    continue
+
+                exported_files[fmt] = str(output_path.absolute())
+                logger.info(f"[ResultsExporter] Exported to: {output_path}")
+
+            except Exception as e:
+                logger.error(
+                    f"[ResultsExporter] Failed to export {filename} as {fmt}: {e}"
+                )
+
+        return exported_files
+
+    def export_dashboard(
+        self, results: SimulationResults, formats: List[str] = ["html"]
+    ) -> Dict[str, str]:
+        """
+        Export a complete dashboard with all visualizations.
+
+        Args:
+            results: SimulationResults object
+            formats: List of formats to export
+
+        Returns:
+            Dictionary mapping format to file path
+        """
+        logger.info(f"[ResultsExporter] Exporting dashboard to formats: {formats}")
+
+        exported_files = {}
+        dashboard_gen = DashboardGenerator()
+
+        # Generate HTML dashboard
+        if "html" in formats:
+            html_path = self.output_dir / "dashboard.html"
+            dashboard_path = dashboard_gen.generate_simulator_dashboard(
+                results=results,
+                output_path=str(html_path),
+                title="Evaluation Dashboard",
+            )
+            exported_files["html"] = dashboard_path
+
+        # Export individual charts if PNG/PDF requested
+        if any(fmt in formats for fmt in ["png", "pdf"]):
+            from levelapp.visualization.charts import ChartGenerator
+
+            chart_gen = ChartGenerator()
+
+            charts_to_export = {
+                "score_trend": chart_gen.create_score_trend(results),
+                "provider_comparison": chart_gen.create_provider_comparison(results),
+            }
+
+            for chart_name, figure in charts_to_export.items():
+                chart_formats = [fmt for fmt in formats if fmt in ["png", "pdf"]]
+                self.export_chart(figure, chart_name, chart_formats)
+
+        return exported_files
+
+    def create_archive(
+        self, files: List[str], archive_name: str = "evaluation_results.zip"
+    ) -> str:
+        """
+        Create a ZIP archive of exported files.
+
+        Args:
+            files: List of file paths to include
+            archive_name: Name of the archive file
+
+        Returns:
+            Path to the created archive
+        """
+        logger.info(f"[ResultsExporter] Creating archive: {archive_name}")
+
+        archive_path = self.output_dir / archive_name
+
+        with zipfile.ZipFile(archive_path, "w", zipfile.ZIP_DEFLATED) as zipf:
+            for file_path in files:
+                file_path = Path(file_path)
+                if file_path.exists():
+                    zipf.write(file_path, file_path.name)
+                    logger.info(f"[ResultsExporter] Added to archive: {file_path.name}")
+
+        logger.info(f"[ResultsExporter] Archive created: {archive_path}")
+        return str(archive_path.absolute())
+
+    def export_results_json(
+        self, results: SimulationResults, filename: str = "results.json"
+    ) -> str:
+        """
+        Export raw results to JSON file.
+
+        Args:
+            results: SimulationResults object
+            filename: Output filename
+
+        Returns:
+            Path to the JSON file
+        """
+        logger.info(f"[ResultsExporter] Exporting results to JSON: {filename}")
+
+        output_path = self.output_dir / filename
+        output_path.write_text(results.model_dump_json(indent=2), encoding="utf-8")
+
+        logger.info(f"[ResultsExporter] JSON exported to: {output_path}")
+        return str(output_path.absolute())

levelapp/visualization/templates/base.html

@@ -0,0 +1,158 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>{{ title }}</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+        
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
+            background: #f5f7fa;
+            color: #333;
+            line-height: 1.6;
+        }
+        
+        .container {
+            max-width: 1400px;
+            margin: 0 auto;
+            padding: 20px;
+        }
+        
+        header {
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            color: white;
+            padding: 30px;
+            border-radius: 10px;
+            margin-bottom: 30px;
+            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
+        }
+        
+        header h1 {
+            font-size: 2.5em;
+            margin-bottom: 10px;
+        }
+        
+        header .subtitle {
+            opacity: 0.9;
+            font-size: 1.1em;
+        }
+        
+        .summary-cards {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+            gap: 20px;
+            margin-bottom: 30px;
+        }
+        
+        .card {
+            background: white;
+            padding: 25px;
+            border-radius: 10px;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+            transition: transform 0.2s;
+        }
+        
+        .card:hover {
+            transform: translateY(-5px);
+            box-shadow: 0 4px 8px rgba(0,0,0,0.15);
+        }
+        
+        .card h3 {
+            color: #667eea;
+            font-size: 0.9em;
+            text-transform: uppercase;
+            letter-spacing: 1px;
+            margin-bottom: 10px;
+        }
+        
+        .card .value {
+            font-size: 2.5em;
+            font-weight: bold;
+            color: #333;
+        }
+        
+        .card .label {
+            color: #666;
+            font-size: 0.9em;
+            margin-top: 5px;
+        }
+        
+        .chart-section {
+            background: white;
+            padding: 30px;
+            border-radius: 10px;
+            margin-bottom: 30px;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+        }
+        
+        .chart-section h2 {
+            color: #333;
+            margin-bottom: 20px;
+            padding-bottom: 10px;
+            border-bottom: 2px solid #667eea;
+        }
+        
+        .chart-grid {
+            display: grid;
+            grid-template-columns: repeat(auto-fit, minmax(500px, 1fr));
+            gap: 30px;
+        }
+        
+        footer {
+            text-align: center;
+            padding: 20px;
+            color: #666;
+            font-size: 0.9em;
+        }
+        
+        .badge {
+            display: inline-block;
+            padding: 4px 12px;
+            border-radius: 20px;
+            font-size: 0.85em;
+            font-weight: 600;
+            margin: 2px;
+        }
+        
+        .badge-success {
+            background: #d4edda;
+            color: #155724;
+        }
+        
+        .badge-info {
+            background: #d1ecf1;
+            color: #0c5460;
+        }
+        
+        @media (max-width: 768px) {
+            .chart-grid {
+                grid-template-columns: 1fr;
+            }
+            
+            header h1 {
+                font-size: 1.8em;
+            }
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <header>
+            <h1>{{ title }}</h1>
+            <div class="subtitle">Generated at: {{ generated_at }}</div>
+        </header>
+        
+        {% block content %}{% endblock %}
+        
+        <footer>
+            <p>Generated by LevelApp Evaluation Framework</p>
+        </footer>
+    </div>
+</body>
+</html>

levelapp/visualization/templates/comparator_dashboard.html

@@ -0,0 +1,57 @@
+{% extends "base.html" %}
+
+{% block content %}
+<!-- Metadata Heatmap -->
+<div class="chart-section">
+    <h2>Metadata Field Accuracy</h2>
+    {{ chart_html|safe }}
+</div>
+
+<!-- Results Table -->
+<div class="chart-section">
+    <h2>Detailed Comparison Results</h2>
+    <div style="overflow-x: auto;">
+        <table style="width: 100%; border-collapse: collapse;">
+            <thead>
+                <tr style="background: #667eea; color: white;">
+                    <th style="padding: 12px; text-align: left;">Field Name</th>
+                    <th style="padding: 12px; text-align: left;">Entity Metric</th>
+                    <th style="padding: 12px; text-align: left;">Set Metric</th>
+                    <th style="padding: 12px; text-align: center;">Score</th>
+                    <th style="padding: 12px; text-align: center;">Threshold</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for idx, result in results.items() %}
+                <tr style="border-bottom: 1px solid #ddd;">
+                    <td style="padding: 12px;">{{ result.field_name }}</td>
+                    <td style="padding: 12px;">{{ result.entity_metric }}</td>
+                    <td style="padding: 12px;">{{ result.set_metric }}</td>
+                    <td style="padding: 12px; text-align: center;">
+                        {% set score = result.set_scores[0] if result.set_scores is iterable and
+                        result.set_scores|length > 0 else result.set_scores %}
+                        <span
+                            class="badge {% if score >= result.threshold %}badge-success{% else %}badge-info{% endif %}">
+                            {{ "%.3f"|format(score if score is number else 0) }}
+                        </span>
+                    </td>
+                    <td style="padding: 12px; text-align: center;">{{ result.threshold }}</td>
+                </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+</div>
+
+<!-- Raw Data -->
+<div class="chart-section">
+    <h2>Raw Comparison Data</h2>
+    <details>
+        <summary style="cursor: pointer; padding: 10px; background: #f8f9fa; border-radius: 5px;">
+            Click to view raw JSON data
+        </summary>
+        <pre
+            style="background: #f8f9fa; padding: 20px; border-radius: 5px; overflow-x: auto; margin-top: 10px;">{{ results|tojson(indent=2) }}</pre>
+    </details>
+</div>
+{% endblock %}

levelapp/visualization/templates/simulator_dashboard.html

@@ -0,0 +1,111 @@
+{% extends "base.html" %}
+
+{% block content %}
+<!-- Summary Statistics -->
+<div class="summary-cards">
+    <div class="card">
+        <h3>Total Scripts</h3>
+        <div class="value">{{ summary_stats.total_scripts }}</div>
+        <div class="label">Evaluated</div>
+    </div>
+
+    <div class="card">
+        <h3>Overall Average</h3>
+        <div class="value">{{ "%.3f"|format(summary_stats.overall_average) }}</div>
+        <div class="label">Across all providers</div>
+    </div>
+
+    <div class="card">
+        <h3>Total Time</h3>
+        <div class="value">{{ "%.2f"|format(summary_stats.total_time) }}</div>
+        <div class="label">Seconds</div>
+    </div>
+
+    <div class="card">
+        <h3>Providers</h3>
+        <div class="value">{{ summary_stats.providers|length }}</div>
+        <div class="label">
+            {% for provider in summary_stats.providers %}
+            {% if provider not in ['processing_time', 'guardrail', 'metadata'] %}
+            <span class="badge badge-info">{{ provider|upper }}</span>
+            {% endif %}
+            {% endfor %}
+        </div>
+    </div>
+</div>
+
+<!-- Summary Metrics Chart -->
+{% if charts.summary_metrics %}
+<div class="chart-section">
+    <h2>Key Metrics</h2>
+    {{ charts.summary_metrics|safe }}
+</div>
+{% endif %}
+
+<!-- Score Trends -->
+{% if charts.score_trend %}
+<div class="chart-section">
+    <h2>Score Trends</h2>
+    {{ charts.score_trend|safe }}
+</div>
+{% endif %}
+
+<!-- Provider Comparison -->
+{% if charts.provider_comparison %}
+<div class="chart-section">
+    <h2>Provider Comparison</h2>
+    {{ charts.provider_comparison|safe }}
+</div>
+{% endif %}
+
+<!-- Distribution Charts -->
+<div class="chart-section">
+    <h2>Score Distributions</h2>
+    <div class="chart-grid">
+        {% for key, chart in charts.items() %}
+        {% if key.startswith('distribution_') %}
+        {{ chart|safe }}
+        {% endif %}
+        {% endfor %}
+    </div>
+</div>
+
+<!-- Timeline -->
+{% if charts.timeline %}
+<div class="chart-section">
+    <h2>Processing Timeline</h2>
+    {{ charts.timeline|safe }}
+</div>
+{% endif %}
+
+<!-- Evaluation Summary -->
+{% if has_evaluation_summary and results.evaluation_summary %}
+<div class="chart-section">
+    <h2>Evaluation Summary</h2>
+    {% for provider, verdicts in results.evaluation_summary.items() %}
+    <div style="margin-bottom: 20px;">
+        <h3 style="color: #667eea;">{{ provider|upper }}</h3>
+        <ul style="list-style: none; padding-left: 0;">
+            {% for verdict in verdicts %}
+            <li style="padding: 8px; background: #f8f9fa; margin: 5px 0; border-left: 3px solid #667eea;">
+                {{ verdict }}
+            </li>
+            {% endfor %}
+        </ul>
+    </div>
+    {% endfor %}
+</div>
+{% endif %}
+
+<!-- Raw Data Section -->
+<div class="chart-section">
+    <h2>Detailed Results</h2>
+    <details>
+        <summary style="cursor: pointer; padding: 10px; background: #f8f9fa; border-radius: 5px;">
+            Click to view raw JSON data
+        </summary>
+        <pre
+            style="background: #f8f9fa; padding: 20px; border-radius: 5px; overflow-x: auto; margin-top: 10px;">{{ results.model_dump_json(indent=2) }}</pre>
+    </details>
+</div>
+{% endblock %}

levelapp/workflow/__init__.py

@@ -0,0 +1,6 @@
+from . import registration
+from .base import BaseWorkflow
+from .config import WorkflowConfig
+from .factory import MainFactory
+
+__all__ = ["BaseWorkflow", "WorkflowConfig", "MainFactory"]

levelapp/workflow/base.py

@@ -0,0 +1,192 @@
+import asyncio
+
+from abc import ABC, abstractmethod
+from pydantic import ValidationError
+from functools import partial
+from pathlib import Path
+from typing import Any, Dict
+
+from levelapp.core.base import BaseProcess
+from levelapp.endpoint.client import EndpointConfig
+from levelapp.endpoint.manager import EndpointConfigManager
+from levelapp.simulator.schemas import ScriptsBatch
+from levelapp.simulator.simulator import ConversationSimulator
+from levelapp.workflow.runtime import WorkflowContext
+from levelapp.aspects.loader import DataLoader
+
+
+class BaseWorkflow(ABC):
+    """Abstract base class for evaluation workflows."""
+
+    def __init__(self, name: str, context: WorkflowContext) -> None:
+        self.name = name
+        self.context = context
+        self.process: BaseProcess | None = None
+        self._input_data: Any | None = None
+        self._results: Any | None = None
+        self._initialized: bool = False
+
+    def setup(self) -> None:
+        """Validate and initialize workflow-specific settings."""
+        if self._initialized:
+            return
+
+        self.process = self._setup_process(context=self.context)
+        self._initialized = True
+
+    def load_data(self) -> None:
+        """Load and preprocess input data."""
+        if not self._initialized:
+            raise RuntimeError(f"[{self.name}] Workflow not initialized. Call setup() first.")
+        self._input_data = self._load_input_data(context=self.context)
+
+    def execute(self) -> None:
+        """Run the workflow evaluation steps."""
+        if not self._input_data:
+            raise RuntimeError(f"[{self.name}] No reference data available.")
+
+        if asyncio.iscoroutinefunction(self.process.run):
+            self._results = asyncio.run(self.process.run(**self._input_data))
+        else:
+            self._results = self.process.run(**self._input_data)
+
+    async def aexecute(self) -> None:
+        if not self._input_data:
+            raise RuntimeError(f"[{self.name}] No reference data available.")
+
+        if asyncio.iscoroutinefunction(self.process.run):
+            self._results = await self.process.run(**self._input_data)
+        else:
+            loop = asyncio.get_running_loop()
+            func = partial(self.process.run, **self._input_data)
+            self._results = await loop.run_in_executor(None, func, None)
+
+    def collect_results(self) -> Any:
+        """
+        Return unified results structure.
+
+        Returns:
+            The simulation results.
+        """
+        return self._results
+
+    @abstractmethod
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Abstract method for testing endpoint connection.
+
+        Args:
+            context (Dict[str, Any]): The context (request payload) to test connectivity with.
+
+        Returns:
+            The test connectivity result.
+        """
+
+    @abstractmethod
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Abstract method for setting up the configured process.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def _load_input_data(self, context: WorkflowContext) -> Any:
+        """
+        Abstract method for loading reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context.
+        """
+        raise NotImplementedError
+
+
+class SimulatorWorkflow(BaseWorkflow):
+    def __init__(self, context: WorkflowContext) -> None:
+        super().__init__(name="ConversationSimulator", context=context)
+
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        """
+        Concrete implementation for setting up the simulation workflow.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            ConversationSimulator instance.
+        """
+        simulator = ConversationSimulator()
+        simulator.setup(
+            endpoint_config=context.endpoint,
+            evaluators=context.evaluators,
+            providers=context.providers,
+        )
+
+        return simulator
+
+    async def test_connection(self, context: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Runs a connectivity test of the configured endpoint.
+
+        Args:
+            context (Dict[str, Any]): The request payload to send for testing.
+
+        Returns:
+            The test connectivity result.
+        """
+        endpoint_cm = EndpointConfigManager()
+        endpoint_cm.set_endpoints(endpoints_config=[self.context.endpoint])
+        tester = endpoint_cm.get_tester(endpoint_name=self.context.endpoint.name)
+        results = await tester.test(context=context)
+
+        return results
+
+    def _load_input_data(self, context: WorkflowContext) -> Dict[str, Any]:
+        """
+        Concrete implementation for loading the reference data.
+
+        Args:
+            context (WorkflowContext): The workflow context for the simulation workflow.
+
+        Returns:
+            Dict[str, Any]: The reference data.
+        """
+        loader = DataLoader()
+        if "reference_data" in context.inputs:
+            data_config = context.inputs["reference_data"]
+        else:
+            reference_data_path = context.inputs.get("reference_data_path", "no-path-provided")
+
+            if not reference_data_path:
+                raise RuntimeError(f"[{self.name}] No reference data available.")
+
+            file_path = Path(reference_data_path)
+
+            if not file_path.exists():
+                raise FileNotFoundError(f"[{self.name}] Reference data file not found.")
+
+            data_config = loader.load_raw_data(path=reference_data_path)
+
+        try:
+            scripts_batch = ScriptsBatch.model_validate(data_config)
+
+        except ValidationError as e:
+            raise RuntimeError(f"[{self.name}] Validation error: {e}")
+
+        attempts = context.config.process.evaluation_params.get("attempts", 1)
+        batch_size = context.config.process.evaluation_params.get("batch_size", 5)
+
+        return {"test_batch": scripts_batch, "attempts": attempts, "batch_size": batch_size}
+
+
+class ComparatorWorkflow(BaseWorkflow):
+    def __init__(self, context: WorkflowContext) -> None:
+        super().__init__(name="MetadataComparator", context=context)
+
+    def _setup_process(self, context: WorkflowContext) -> BaseProcess:
+        raise NotImplementedError
+
+    def _load_input_data(self, context: WorkflowContext) -> Any:
+        raise NotImplementedError