edsl 0.1.51__py3-none-any.whl → 0.1.52__py3-none-any.whl

edsl/results/results.py

@@ -44,10 +44,10 @@ from typing import Optional, Callable, Any, Union, List, TYPE_CHECKING
 from bisect import bisect_left
 
 from ..base import Base
+from ..caching import Cache, CacheEntry
 
 if TYPE_CHECKING:
     from ..surveys import Survey
-    from ..caching import Cache
     from ..agents import AgentList
     from ..scenarios import ScenarioList
     from ..results import Result
@@ -707,12 +707,41 @@ class Results(UserList, ResultsOperationsMixin, Base):
             "b_not_a": [other_results[i] for i in indices_other],
         }
 
+    def initialize_cache_from_results(self):
+        cache = Cache(data={})
+
+        for result in self.data:
+            for key in result.data["prompt"]:
+                if key.endswith("_system_prompt"):
+                    question_name = key.removesuffix("_system_prompt")
+                    system_prompt = result.data["prompt"][key].text
+                    user_key = f"{question_name}_user_prompt"
+                    if user_key in result.data["prompt"]:
+                        user_prompt = result.data["prompt"][user_key].text
+                    else:
+                        user_prompt = ""
+
+                    # Get corresponding model response
+                    response_key = f"{question_name}_raw_model_response"
+                    output = result.data["raw_model_response"].get(response_key, "")
+
+                    entry = CacheEntry(
+                        model=result.model.model,
+                        parameters=result.model.parameters,
+                        system_prompt=system_prompt,
+                        user_prompt=user_prompt,
+                        output=json.dumps(output),
+                        iteration=0,
+                    )
+                    cache.data[entry.key] = entry
+
+        self.cache = cache
+
     @property
     def has_unfixed_exceptions(self) -> bool:
         return self.task_history.has_unfixed_exceptions
 
     def __hash__(self) -> int:
-
         return dict_hash(
             self.to_dict(sort=True, add_edsl_version=False, include_cache_info=False)
         )

edsl/tasks/task_history.py

@@ -302,22 +302,59 @@ class TaskHistory(RepresentationMixin):
         js = env.joinpath("report.js").read_text()
         return js
 
+    # @property
+    # def exceptions_table(self) -> dict:
+    #     """Return a dictionary of exceptions organized by type, service, model, and question name."""
+    #     exceptions_table = {}
+    #     for interview in self.total_interviews:
+    #         for question_name, exceptions in interview.exceptions.items():
+    #             for exception in exceptions:
+    #                 key = (
+    #                     exception.exception.__class__.__name__,  # Exception type
+    #                     interview.model._inference_service_,  # Service
+    #                     interview.model.model,  # Model
+    #                     question_name,  # Question name
+    #                 )
+    #                 if key not in exceptions_table:
+    #                     exceptions_table[key] = 0
+    #                 exceptions_table[key] += 1
+    #     return exceptions_table
+
     @property
     def exceptions_table(self) -> dict:
-        """Return a dictionary of exceptions organized by type, service, model, and question name."""
+        """Return a dictionary of unique exceptions organized by type, service, model, and question name."""
         exceptions_table = {}
+        seen_exceptions = set()
+        
         for interview in self.total_interviews:
             for question_name, exceptions in interview.exceptions.items():
                 for exception in exceptions:
-                    key = (
+                    # Create a unique identifier for this exception based on its content
+                    exception_key = (
                         exception.exception.__class__.__name__,  # Exception type
-                        interview.model._inference_service_,  # Service
-                        interview.model.model,  # Model
-                        question_name,  # Question name
+                        interview.model._inference_service_,     # Service
+                        interview.model.model,                   # Model
+                        question_name,                           # Question name
+                        exception.name,                          # Exception name
+                        str(exception.traceback)[:100] if exception.traceback else "",  # Truncated traceback
                     )
-                    if key not in exceptions_table:
-                        exceptions_table[key] = 0
-                    exceptions_table[key] += 1
+                    
+                    # Only count if we haven't seen this exact exception before
+                    if exception_key not in seen_exceptions:
+                        seen_exceptions.add(exception_key)
+                        
+                        # Add to the summary table
+                        table_key = (
+                            exception.exception.__class__.__name__,  # Exception type
+                            interview.model._inference_service_,     # Service
+                            interview.model.model,                   # Model
+                            question_name,                           # Question name
+                        )
+                        
+                        if table_key not in exceptions_table:
+                            exceptions_table[table_key] = 0
+                        exceptions_table[table_key] += 1
+        
         return exceptions_table
 
     @property

edsl/templates/error_reporting/base.html

@@ -5,6 +5,39 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Exceptions Report</title>
     <style>
+    /* Global styles */
+    :root {
+        --primary-color: #3f51b5;
+        --secondary-color: #5c6bc0;
+        --success-color: #4caf50;
+        --error-color: #f44336;
+        --warning-color: #ff9800;
+        --text-color: #333;
+        --light-bg: #f5f7fa;
+        --border-color: #e0e0e0;
+        --header-bg: #f9f9f9;
+        --card-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+        --font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, 'Open Sans', 'Helvetica Neue', sans-serif;
+    }
+    
+    * {
+        box-sizing: border-box;
+    }
+    
+    body {
+        font-family: var(--font-family);
+        background-color: var(--light-bg);
+        color: var(--text-color);
+        line-height: 1.6;
+        margin: 0;
+        padding: 20px;
+    }
+    
+    .container {
+        max-width: 1200px;
+        margin: 0 auto;
+    }
+    
     {{ css }}
     </style>
 
@@ -14,9 +47,9 @@
 
 </head>
 <body>
-    {% include 'overview.html' %}
-    {% include 'exceptions_table.html' %}
-    {% include 'interviews.html' %}
-    {% include 'performance_plot.html' %}
+    <div class="container">
+        {% include 'exceptions_table.html' %}
+        {% include 'interviews.html' %}
+    </div>
 </body>
 </html>

edsl/templates/error_reporting/exceptions_table.html

@@ -1,34 +1,106 @@
+<div class="summary-section">    
+    <div class="table-container">
+        <h2>Exceptions Report</h2>
+        <table class="exceptions-table">
+            <thead>
+                <tr>
+                    <th>Exception Type</th>
+                    <th>Service</th>
+                    <th>Model</th>
+                    <th>Question Name</th>
+                    <th class="count-column">Count</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for (exception_type, service, model, question_name), count in exceptions_table.items() %}
+                    <tr>
+                        <td>{{ exception_type }}</td>
+                        <td>{{ service }}</td>
+                        <td>{{ model }}</td>
+                        <td>{{ question_name }}</td>
+                        <td class="count-cell">{{ count }}</td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    </div>
+    
+    <p class="note">
+        Note: Each unique exception is counted only once. You may encounter repeated exceptions where retries were attempted.
+    </p>
+</div>
+
 <style>
-    th, td {
-        padding: 0 10px; /* This applies the padding uniformly to all td elements */
-    }
-</style>
-
-<table border="1">
-    <thead>
-        <tr>
-            <th style="text-align: left">Exception Type</th>
-            <th style="text-align: left">Service</th>
-            <th style="text-align: left">Model</th>
-            <th style="text-align: left">Question Name</th>
-            <th style="text-align: left">Total</th>
-        </tr>
-    </thead>
-    <tbody>
-        {% for (exception_type, service, model, question_name), count in exceptions_table.items() %}
-            <tr>
-                <td>{{ exception_type }}</td>
-                <td>{{ service }}</td>
-                <td>{{ model }}</td>
-                <td>{{ question_name }}</td>
-                <td>{{ count }}</td>
-            </tr>
-        {% endfor %}
-    </tbody>
-</table>
-<p>
-    Note: You may encounter repeated exceptions where retries were attempted. 
-</p>
-<p>
-    See details about each exception, including code for reproducing it (click to expand).
-</p>
+/* Summary section styles */
+.summary-section {
+    background-color: white;
+    border-radius: 8px;
+    margin-bottom: 24px;
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+    overflow: hidden;
+    border: 1px solid #e0e0e0;
+    padding: 0 0 16px 0;
+}
+
+.section-header {
+    background-color: #f9f9f9;
+    padding: 12px 16px;
+    border-bottom: 1px solid #e0e0e0;
+}
+
+.section-header h2 {
+    margin: 0;
+    font-size: 18px;
+    font-weight: 500;
+    color: #3f51b5;
+}
+
+.table-container {
+    padding: 16px;
+    overflow-x: auto;
+}
+
+/* Table styles */
+.exceptions-table {
+    width: 100%;
+    border-collapse: collapse;
+    margin-bottom: 16px;
+}
+
+.exceptions-table th {
+    background-color: #f5f5f5;
+    color: #333;
+    font-weight: 500;
+    text-align: left;
+    padding: 12px;
+    border-bottom: 2px solid #e0e0e0;
+}
+
+.exceptions-table td {
+    padding: 10px 12px;
+    border-bottom: 1px solid #e0e0e0;
+    color: #333;
+}
+
+.exceptions-table tr:hover {
+    background-color: #f9f9f9;
+}
+
+.count-column {
+    width: 80px;
+    text-align: center;
+}
+
+.count-cell {
+    text-align: center;
+    font-weight: 500;
+}
+
+/* Note styles */
+.note {
+    font-size: 14px;
+    color: #666;
+    margin: 0 16px;
+    line-height: 1.5;
+}
+</style>

edsl/templates/error_reporting/interview_details.html

@@ -1,130 +1,134 @@
-<style>
-    td {
-        padding: 0 10px; /* This applies the padding uniformly to all td elements */
-    }
-    .toggle-btn {
-        background-color: #4CAF50;
-        color: white;
-        border: none;
-        padding: 10px 20px;
-        text-align: center;
-        text-decoration: none;
-        display: inline-block;
-        font-size: 16px;
-        margin: 4px 2px;
-        cursor: pointer;
-        border-radius: 8px;
-        white-space: nowrap;
-    }
-    .toggle-btn span.collapse {
-        display: none;
-    }
-    .exception-content {
-        max-width: 100%; /* Adjust this value based on your layout */
-        overflow-x: auto; /* Enables horizontal scrolling */
-    }
-</style>
+<div class="summary-section">
+    <div class="section-header">
+        <h2>Exception Details</h2>
+    </div>
+    <div class="p-4">
+        <div class="question">Question name: {{ question }}</div>
 
-<div class="question">Question name: {{ question }}</div>
+        {% set seen_exceptions = [] %}
 
-{% for exception_message in exceptions %}
-<div class="exception-detail">
-    <div class="exception-header">
-        <span class="exception-exception">Exception: {{ exception_message.name }}</span>
-        <button id="toggleBtn" class="toggle-btn" onclick="toggleButton(this)" aria-expanded="false">
-            <span class="expand"> ▼ </span>
-        </button>             
-    </div>
-    <div class="exception-content">
-        <table border="1">
-            <tr>
-                <td>Interview ID (index in results)</td>
-                <td>{{ index }}</td>
-            </tr>
-            <tr>
-                <td>Question name</td>
-                <td>{{ question }}</td>
-            </tr>
-            <tr>
-                <td>Question type</td>
-                <td>{{ exception_message.question_type }}</td>
-            </tr>
-            <tr>
-                <td>Human-readable question</td>
-                <td>{{ interview.survey._get_question_by_name(question).html(
-                    scenario = interview.scenario, 
-                    agent = interview.agent,
-                    answers = exception_message.answers
-                    ) 
-                }}</td>
-            </tr>
-            <tr>
-                <td>User Prompt</td>
-                <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
-            </tr>
-            <tr>
-                <td>Scenario</td>
-                <td>{{ interview.scenario.__repr__() }}</td>
-            </tr>
-            <tr>
-                <td>Agent</td>
-                <td>{{ interview.agent.__repr__() }}</td>
-            </tr>
-            <tr>
-                <td>System Prompt</td>
-                <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
-            </tr>
-            <tr>
-                <td>Inference service</td>
-                <td>{{ interview.model._inference_service_ }}</td>
-            </tr>
-            <tr>
-                <td>Model name</td>
-                <td>{{ interview.model.model }}</td>
-            </tr>
-            <tr>
-                <td>Model parameters</td>
-                <td>{{ interview.model.__repr__() }}</td>
-            </tr>
-            <tr>
-                <td>Raw model response</td>
-                <td><pre>{{ exception_message.raw_model_response }}</pre>
-                </td>
-            </tr>
-            <tr>
-                <td>Generated token string (at {{ exception_message.key_sequence }}) in raw response</td>
-                <td><pre>{{ exception_message.generated_token_string }}</pre>
-                </td>
-            </tr>
-            <tr>
-            <td>Code likely to reproduce the error</td>
-            <td>
-                <textarea id="codeToCopy" rows="10" cols="90">{{ exception_message.code_to_reproduce }}</textarea>
-                <button onclick="copyCode()">Copy</button>
-            </td>
-            </tr>
+        {% for exception_message in exceptions %}
+            {% set exception_key = exception_message.name ~ '|' ~ 
+                                exception_message.question_type|default('') ~ '|' ~ 
+                                (exception_message.traceback|default(''))[0:100] %}
+            
+            {% if exception_key not in seen_exceptions %}
+                {% set _ = seen_exceptions.append(exception_key) %}
+                
+                <div class="exception-detail">
+                    <div class="exception-header">
+                        <span class="exception-exception">
+                            {{ exception_message.exception.__class__.__name__ }}
+                        </span>
+                        <span class="chevron"></span>
+                    </div>
+                    <div class="exception-content">
+                        <!-- Error Summary Section -->
+                        <div>
+                            <div class="section-header error-header">
+                                <h3>Error Summary</h3>
+                            </div>
+                            
+                            <table class="details-table">
+                                <tbody>
+                                    <tr>
+                                        <th>Exception</th>
+                                        <td>{{ exception_message.name }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Model</th>
+                                        <td>{{ interview.model.model }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Question</th>
+                                        <td>{{ question }} ({{ exception_message.question_type|default('Unknown type') }})</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Time</th>
+                                        <td>{{ exception_message.time }}</td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                        </div>
 
-        </table>
-    
-        {% if exception_message.exception.__class__.__name__ == 'QuestionAnswerValidationError' %}        
-        <h3>Answer validation details</h3>
-        <table border="1">
-            {% for field, (open_tag, close_tag, value) in exception_message.exception.to_html_dict().items() %}
-            <tr>
-                <td>{{ field }}</td>
-                <td><{{ open_tag }}> {{ value | escape }} <{{ close_tag }}></td>
-            </tr>
-            {% endfor %}
-        </table>
-    {% endif %}
-        <br><br>
-        <div class="exception-time">Time: {{ exception_message.time }}</div>          
-        <div class="exception-traceback">Traceback: 
-            <text>
-            <pre>{{ exception_message.traceback }}</pre>
-            </text>
-        </div>
-    </div>
-</div>
+                        <!-- Details Section -->
+                        <div>
+                            <div class="section-header">
+                                <h3>Details</h3>
+                            </div>
+                            <table class="details-table">
+                                <tbody>
+                                    <tr>
+                                        <th>Interview ID</th>
+                                        <td>{{ index }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Question name</th>
+                                        <td>{{ question }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Question type</th>
+                                        <td>{{ exception_message.question_type|default('Unknown') }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>User Prompt</th>
+                                        <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
+                                    </tr>
+                                    <tr>
+                                        <th>Scenario</th>
+                                        <td>{{ interview.scenario.__repr__() }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Agent</th>
+                                        <td>{{ interview.agent.__repr__() }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>System Prompt</th>
+                                        <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
+                                    </tr>
+                                    <tr>
+                                        <th>Inference service</th>
+                                        <td>{{ interview.model._inference_service_ }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Model name</th>
+                                        <td>{{ interview.model.model }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Model parameters</th>
+                                        <td>{{ interview.model.__repr__() }}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Raw model response</th>
+                                        <td><pre>{{ exception_message.raw_model_response }}</pre></td>
+                                    </tr>
+                                    <tr>
+                                        <th>Generated token string</th>
+                                        <td><pre>{{ exception_message.generated_token_string }}</pre></td>
+                                    </tr>
+                                </tbody>
+                            </table>
+                        </div>
 
-{% endfor %}
+                        <!-- Code to reproduce section -->
+                        <div>
+                            <div class="section-header code-header">
+                                <h3>Code to reproduce error</h3>
+                                <button class="copy-button">Copy</button>
+                            </div>
+                            <pre class="code-block"><code>{{ exception_message.code_to_reproduce }}</code></pre>
+                        </div>
+
+                        <!-- Traceback section -->
+                        <div>
+                            <div class="section-header">
+                                <h3>Traceback</h3>
+                            </div>
+                            <pre class="traceback">{{ exception_message.traceback }}</pre>
+                        </div>
+                    </div>
+                </div>
+            {% endif %}
+        {% endfor %}
+    </div>
+</div>

edsl/templates/error_reporting/overview.html

@@ -1,26 +1,22 @@
-<style>
-    td {
-        padding: 0 10px; /* This applies the padding uniformly to all td elements */
-    }
-</style>
+<div class="overview-section">
+    <h2>Exceptions Report</h2>
+    <p>
+        This report summarizes exceptions encountered in the job that was run.
+    </p>
 
-<h3>Exceptions Report</h3>
-<p>
-    This report summarizes exceptions encountered in the job that was run.
-</p>
-
-<table border="1">
-    <tbody>
-        <tr>
-            <td>Total interviews</td>
-            <td>{{ interviews|length }}</td>
-        </tr>
-        <tr>
-            <td>Interviews with exceptions</td>
-            <td>{{ num_exceptions }}</td>
-        </tr>
-    </tbody>
-</table>
-<p>
-    An interview is the result of one survey, taken by one agent, with one model and one scenario (if any).
-</p>
+    <table class="summary-table">
+        <tbody>
+            <tr>
+                <td><strong>Total interviews</strong></td>
+                <td>{{ interviews|length }}</td>
+            </tr>
+            <tr>
+                <td><strong>Interviews with exceptions</strong></td>
+                <td>{{ num_exceptions }}</td>
+            </tr>
+        </tbody>
+    </table>
+    <p>
+        An interview is the result of one survey, taken by one agent, with one model and one scenario (if any).
+    </p>
+</div>