edsl 0.1.31.dev4__py3-none-any.whl → 0.1.33__py3-none-any.whl

edsl/templates/error_reporting/interview_details.html

@@ -0,0 +1,116 @@
+<div class="question">question_name: {{ question }}</div>
+
+
+<h2>Exception details</h2>
+
+{% for exception_message in exceptions %}
+<div class="exception-detail">
+        <div class="exception-header">
+        <span class="exception-exception">Exception: {{ exception_message.name }}</span>
+        <button class="toggle-btn">▼</button>
+        </div>
+        <div class="exception-content">
+        <table border="1">
+            <tr>
+                <th>Key</th>
+                <th>Value</th>
+            </tr>
+            <tr>
+                <td>Interview ID (index in results)</td>
+                <td>{{ index }}</td>
+            </tr>
+            <tr>
+                <td>Question name (question_name)</td>
+                <td>{{ question }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Question type (question_type)</td>
+                <td>{{ exception_message.question_type }}</td>
+            </tr>
+ 
+            <tr>
+                <td>Human-readable question</td>
+                <td>{{ interview.survey.get_question(question).html(
+                    scenario = interview.scenario, 
+                    agent = interview.agent,
+                    answers = exception_message.answers) 
+                
+                }}</td>
+            </tr>
+            <tr>
+                <td>Scenario</td>
+                <td>{{ interview.scenario._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>Agent</td>
+                <td>{{ interview.agent._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>Model name</td>
+                <td>{{ interview.model.model }}</td>
+            </tr>
+            <tr>
+                <td>Inference service</td>
+                <td>{{ interview.model._inference_service_ }}</td>
+            </tr>
+            <tr>
+                <td>Model parameters</td>
+                <td>{{ interview.model._repr_html_() }}</td>
+            </tr>
+            <tr>
+                <td>User Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['user_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>System Prompt</td>
+                <td><pre>{{ exception_message.rendered_prompts['system_prompt'] }}</pre></td>
+            </tr>
+            <tr>
+                <td>Raw model response</td>
+                <td><pre>{{ exception_message.raw_model_response }}</pre>
+                </td>
+            </tr>
+            <tr>
+                <td>Generated token string (at {{ exception_message.key_sequence }}) in raw response</td>
+                <td><pre>{{ exception_message.generated_token_string }}</pre>
+                </td>
+            </tr>
+            <tr>
+            <td>Code to (likely) reproduce the error</td>
+            <td>
+                <textarea id="codeToCopy" rows="10" cols="90">{{ exception_message.code_to_reproduce }}</textarea>
+                <button onclick="copyCode()">Copy</button>
+            </td>
+            </tr>
+
+        </table>
+
+        
+            {% if exception_message.exception.__class__.__name__ == 'QuestionAnswerValidationError' %}        
+            <h3>Answer validation details</h3>
+            <table border="1">
+                <tr>
+                    <th>Field</th>
+                    <th>Value</th>
+                </tr>
+                {% for field, (explanation, open_tag, close_tag, value) in exception_message.exception.to_html_dict().items() %}
+                
+                <tr>
+                    <td>{{ field }}: ({{ explanation }})</td>
+                    <td><{{open_tag}}> {{ value | escape }} <{{close_tag}}></td>
+                </tr>
+                {% endfor %}
+            </table>
+        {% endif %}
+
+            <div class="exception-time">Time: {{ exception_message.time }}</div>          
+            <div class="exception-traceback">Traceback: 
+                <text>
+                <pre>{{ exception_message.traceback }}</pre>
+                </text>
+            </div>
+        </div>
+    </div>
+
+{% endfor %}

edsl/templates/error_reporting/interviews.html

@@ -0,0 +1,10 @@
+{% for index, interview in interviews.items() %}
+    {% if interview.exceptions != {} %}
+       <div class="interview">Interview: {{ index }} </div>
+       Model: {{ interview.model.model }}
+    <h1>Failing questions</h1>
+    {% endif %}
+    {% for question, exceptions in interview.exceptions.items() %}
+        {% include 'interview_details.html' %}
+    {% endfor %}            
+{% endfor %}

edsl/templates/error_reporting/overview.html

@@ -0,0 +1,5 @@
+<h1>Overview</h1>
+<p>There were {{ interviews|length }} total interview(s). An 'interview' is the result of one survey, taken by one agent, with one model, with one scenario.</p>    
+The number of interviews with any exceptions was {{ num_exceptions }}.</p>
+<p>For advice on dealing with exceptions on Expected Parrot, 
+see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>

edsl/templates/error_reporting/performance_plot.html

@@ -0,0 +1,2 @@
+<h1>Performance Plot</h1>
+{{ performance_plot_html }}

edsl/templates/error_reporting/report.css

@@ -0,0 +1,74 @@
+body {
+    font-family: Arial, sans-serif;
+    line-height: 1.6;
+    background-color: #f9f9f9;
+    color: #333;
+    margin: 20px;
+}
+
+.interview {
+    font-size: 1.5em;
+    margin-bottom: 10px;
+    padding: 10px;
+    background-color: #e3f2fd;
+    border-left: 5px solid #2196f3;
+}
+
+.question {
+    font-size: 1.2em;
+    margin-bottom: 10px;
+    padding: 10px;
+    background-color: #fff9c4;
+    border-left: 5px solid #ffeb3b;
+}
+
+.exception-detail {
+    margin-bottom: 10px;
+    background-color: #ffebee;
+    border-left: 5px solid #f44336;
+}
+
+.exception-header {
+    padding: 10px;
+    cursor: pointer;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+}
+
+.exception-content {
+    padding: 10px;
+    display: none;
+}
+
+.exception-content.show {
+    display: block;
+}
+
+.question-detail {
+    border: 3px solid black;
+    padding: 10px;
+}
+
+.exception-exception {
+    font-weight: bold;
+    color: #d32f2f;
+}
+
+.exception-time,
+.exception-traceback {
+    font-style: italic;
+    color: #555;
+}
+
+.toggle-btn {
+    background: none;
+    border: none;
+    font-size: 1.2em;
+    cursor: pointer;
+    transition: transform 0.3s;
+}
+
+.toggle-btn.rotated {
+    transform: rotate(180deg);
+}

edsl/templates/error_reporting/report.html

@@ -0,0 +1,118 @@
+<!DOCTYPE html>
+        <html lang="en">
+        <head>
+        <meta charset="UTF-8">
+        <meta name="viewport" content="width=device-width, initial-scale=1.0">
+        <title>Exception Details</title>
+        <style>
+        {{ css }}
+        </style>
+        </head>
+        <body>
+            <h1>Overview</h1>
+            <p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
+            <p>The models used were: {{ models_used }}.</p>
+            <p>For documentation on dealing with exceptions on Expected Parrot, 
+            see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
+
+            <h2>Exceptions by Type</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Exception Type</th>
+                        <th>Number</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for exception_type, exceptions in exceptions_by_type.items() %}
+                        <tr>
+                            <td>{{ exception_type }}</td>
+                            <td>{{ exceptions }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            <h2>Exceptions by Model</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Model</th>
+                        <th>Number</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for model, exceptions in exceptions_by_model.items() %}
+                        <tr>
+                            <td>{{ model }}</td>
+                            <td>{{ exceptions }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            <h2>Exceptions by Question Name</h2>
+            <table>
+                <thead>
+                    <tr>
+                        <th>Question Name</th>
+                        <th>Number of Exceptions</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for question_name, exception_count in exceptions_by_question_name.items() %}
+                        <tr>
+                            <td>{{ question_name }}</td>
+                            <td>{{ exception_count }}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table>
+
+
+            {% for index, interview in interviews.items() %}
+                {% if interview.exceptions != {} %}
+                   <div class="interview">Interview: {{ index }} </div>
+                    <h1>Failing questions</h1>
+                {% endif %}
+                {% for question, exceptions in interview.exceptions.items() %}
+                    <div class="question">question_name: {{ question }}</div>
+
+                    <h2>Question</h2>
+                    <div class="question-detail"> 
+                            {{ interview.survey.get_question(question).html() }}
+                    </div>        
+
+                    <h2>Scenario</h2>                            
+                    <div class="scenario"> 
+                            {{ interview.scenario._repr_html_() }}
+                    </div>        
+
+                    <h2>Agent</h2>
+                    <div class="agent">
+                            {{ interview.agent._repr_html_() }}
+                    </div>
+
+                    <h2>Model</h2>
+                    <div class="model">
+                            {{ interview.model._repr_html_() }}
+                    </div>
+                            
+                    <h2>Exception details</h2>
+
+                    {% for exception_message in exceptions %}
+                        <div class="exception-detail">
+                            <div class="exception-exception">Exception: {{ exception_message.exception }}</div>
+                            <div class="exception-time">Time: {{ exception_message.time }}</div>
+                            <div class="exception-traceback">Traceback: <pre>{{ exception_message.traceback }} </pre></div>
+                        </div>
+                    {% endfor %}            
+                {% endfor %}            
+            {% endfor %}
+                            
+        <h1>Performance Plot</h1>
+        {{ performance_plot_html }}
+        </body>
+        </html>

edsl/templates/error_reporting/report.js

@@ -0,0 +1,25 @@
+document.addEventListener('DOMContentLoaded', function() {
+    const collapsibleSections = document.querySelectorAll('.exception-detail, .raw-model-response');
+
+    collapsibleSections.forEach(section => {
+        const header = section.querySelector('.exception-header, .response-header');
+        const content = section.querySelector('.exception-content, .response-content');
+        const toggleBtn = section.querySelector('.toggle-btn');
+
+        header.addEventListener('click', function() {
+            content.classList.toggle('show');
+            toggleBtn.classList.toggle('rotated');
+        });
+    });
+
+});
+
+function copyCode() {
+    const textarea = document.getElementById('codeToCopy');
+    textarea.select();
+    textarea.setSelectionRange(0, 99999); // For mobile devices
+    document.execCommand("copy");
+
+    // Optionally, you can display an alert or change the button text to indicate success
+    alert("Code copied to clipboard!");
+}

edsl/utilities/utilities.py

@@ -20,6 +20,14 @@ from html import escape
 from typing import Callable, Union
 
 
+class CustomEncoder(json.JSONEncoder):
+    def default(self, obj):
+        try:
+            return json.JSONEncoder.default(self, obj)
+        except TypeError:
+            return str(obj)
+
+
 def time_it(func):
     @wraps(func)
     def wrapper(*args, **kwargs):
@@ -61,6 +69,37 @@ def extract_json_from_string(text):
     return None
 
 
+def fix_partial_correct_response(text: str) -> dict:
+    # Find the start position of the key "answer"
+    answer_key_start = text.find('"answer"')
+
+    if answer_key_start == -1:
+        return {"error": "No 'answer' key found in the text"}
+
+    # Define regex to find the complete JSON object starting with "answer"
+    json_pattern = r'(\{[^\{\}]*"answer"[^\{\}]*\})'
+    match = re.search(json_pattern, text)
+
+    if not match:
+        return {"error": "No valid JSON object found"}
+
+    # Extract the matched JSON object
+    json_object = match.group(0)
+
+    # Find the start and stop positions of the JSON object in the original text
+    start_pos = text.find(json_object)
+    stop_pos = start_pos + len(json_object)
+
+    # Parse the JSON object to validate it
+    try:
+        parsed_json = json.loads(json_object)
+    except json.JSONDecodeError:
+        return {"error": "Failed to parse JSON object"}
+
+    # Return the result as a dictionary with positions
+    return {"start": start_pos, "stop": stop_pos, "extracted_json": json_object}
+
+
 def clean_json(bad_json_str):
     """
     Clean JSON string by replacing single quotes with double quotes
@@ -93,7 +132,7 @@ def data_to_html(data, replace_new_lines=False):
     from pygments.formatters import HtmlFormatter
     from IPython.display import HTML
 
-    json_str = json.dumps(data, indent=4)
+    json_str = json.dumps(data, indent=4, cls=CustomEncoder)
     formatted_json = highlight(
         json_str,
         JsonLexer(),

{edsl-0.1.31.dev4.dist-info → edsl-0.1.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: edsl
-Version: 0.1.31.dev4
+Version: 0.1.33
 Summary: Create and analyze LLM-based surveys
 Home-page: https://www.expectedparrot.com/
 License: MIT
@@ -18,15 +18,20 @@ Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: aiohttp (>=3.9.1,<4.0.0)
 Requires-Dist: anthropic (>=0.23.1,<0.24.0)
+Requires-Dist: azure-ai-inference (>=1.0.0b3,<2.0.0)
 Requires-Dist: black[jupyter] (>=24.4.2,<25.0.0)
+Requires-Dist: boto3 (>=1.34.161,<2.0.0)
 Requires-Dist: groq (>=0.9.0,<0.10.0)
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
+Requires-Dist: json-repair (>=0.28.4,<0.29.0)
 Requires-Dist: jupyter (>=1.0.0,<2.0.0)
 Requires-Dist: markdown2 (>=2.4.11,<3.0.0)
 Requires-Dist: matplotlib (>=3.8,<3.9)
+Requires-Dist: mistralai (>=1.0.2,<2.0.0)
 Requires-Dist: nest-asyncio (>=1.5.9,<2.0.0)
 Requires-Dist: numpy (>=1.22,<2.0)
 Requires-Dist: openai (>=1.4.0,<2.0.0)
+Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.1.4,<3.0.0)
 Requires-Dist: pydot (>=2.0.0,<3.0.0)
 Requires-Dist: pygments (>=2.17.2,<3.0.0)
@@ -40,6 +45,7 @@ Requires-Dist: setuptools (<72.0)
 Requires-Dist: simpleeval (>=0.9.13,<0.10.0)
 Requires-Dist: sqlalchemy (>=2.0.23,<3.0.0)
 Requires-Dist: tenacity (>=8.2.3,<9.0.0)
+Requires-Dist: urllib3 (>=1.25.4,<1.27)
 Project-URL: Documentation, https://docs.expectedparrot.com
 Description-Content-Type: text/markdown
 
@@ -64,7 +70,7 @@ A quick example:
 
 ```python
 # Import a question type
-from edsl.questions import QuestionMultipleChoice
+from edsl import QuestionMultipleChoice
 
 # Construct a question using the question type template
 q = QuestionMultipleChoice(