edsl 0.1.54__py3-none-any.whl → 0.1.56__py3-none-any.whl

edsl/notebooks/notebook.py

@@ -2,6 +2,10 @@
 
 from __future__ import annotations
 import json
+import subprocess
+import tempfile
+import os
+import shutil
 from typing import Dict, List, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -17,12 +21,56 @@ class Notebook(Base):
     """
 
     default_name = "notebook"
+    
+    @staticmethod
+    def _lint_code(code: str) -> str:
+        """
+        Lint Python code using ruff.
+        
+        :param code: The Python code to lint
+        :return: The linted code
+        """
+        try:
+            # Check if ruff is installed
+            if shutil.which("ruff") is None:
+                # If ruff is not installed, return original code
+                return code
+                
+            with tempfile.NamedTemporaryFile(mode='w+', suffix='.py', delete=False) as temp_file:
+                temp_file.write(code)
+                temp_file_path = temp_file.name
+            
+            # Run ruff to format the code
+            try:
+                result = subprocess.run(
+                    ["ruff", "format", temp_file_path], 
+                    check=True, 
+                    stdout=subprocess.PIPE, 
+                    stderr=subprocess.PIPE
+                )
+                
+                # Read the formatted code
+                with open(temp_file_path, 'r') as f:
+                    linted_code = f.read()
+                
+                return linted_code
+            except subprocess.CalledProcessError:
+                # If ruff fails, return the original code
+                return code
+            except FileNotFoundError:
+                # If ruff is not installed, return the original code
+                return code
+        finally:
+            # Clean up temporary file
+            if 'temp_file_path' in locals() and os.path.exists(temp_file_path):
+                os.unlink(temp_file_path)
 
     def __init__(
         self,
         path: Optional[str] = None,
         data: Optional[Dict] = None,
         name: Optional[str] = None,
+        lint: bool = True,
     ):
         """
         Initialize a new Notebook.
@@ -32,6 +80,7 @@ class Notebook(Base):
         :param path: A filepath from which to load the notebook.
         If no path is provided, assume this code is run in a notebook and try to load the current notebook from file.
         :param name: A name for the Notebook.
+        :param lint: Whether to lint Python code cells using ruff. Defaults to True.
         """
         import nbformat
 
@@ -54,6 +103,16 @@ class Notebook(Base):
             raise NotebookEnvironmentError(
                 "Cannot create a notebook from within itself in this development environment"
             )
+            
+        # Store the lint parameter
+        self.lint = lint
+        
+        # Apply linting to code cells if enabled
+        if self.lint and self.data and "cells" in self.data:
+            for cell in self.data["cells"]:
+                if cell.get("cell_type") == "code" and "source" in cell:
+                    # Only lint Python code cells
+                    cell["source"] = self._lint_code(cell["source"])
 
         # TODO: perhaps add sanity check function
         # 1. could check if the notebook is a valid notebook
@@ -63,7 +122,7 @@ class Notebook(Base):
         self.name = name or self.default_name
 
     @classmethod
-    def from_script(cls, path: str, name: Optional[str] = None) -> "Notebook":
+    def from_script(cls, path: str, name: Optional[str] = None, lint: bool = True) -> "Notebook":
         import nbformat
 
         # Read the script file
@@ -78,12 +137,12 @@ class Notebook(Base):
         nb.cells.append(first_cell)
 
         # Create a Notebook instance with the notebook data
-        notebook_instance = cls(nb)
+        notebook_instance = cls(data=nb, name=name, lint=lint)
 
         return notebook_instance
 
     @classmethod
-    def from_current_script(cls) -> "Notebook":
+    def from_current_script(cls, lint: bool = True) -> "Notebook":
         import inspect
         import os
 
@@ -93,7 +152,7 @@ class Notebook(Base):
         current_file_path = os.path.abspath(caller_frame[1].filename)
 
         # Use from_script to create the notebook
-        return cls.from_script(current_file_path)
+        return cls.from_script(current_file_path, lint=lint)
 
     def __eq__(self, other):
         """
@@ -114,7 +173,7 @@ class Notebook(Base):
         """
         Serialize to a dictionary.
         """
-        d = {"name": self.name, "data": self.data}
+        d = {"name": self.name, "data": self.data, "lint": self.lint}
         if add_edsl_version:
             from .. import __version__
 
@@ -124,11 +183,17 @@ class Notebook(Base):
 
     @classmethod
     @remove_edsl_version
-    def from_dict(cls, d: Dict) -> "Notebook":
+    def from_dict(cls, d: Dict, lint: bool = None) -> "Notebook":
         """
         Convert a dictionary representation of a Notebook to a Notebook object.
+        
+        :param d: Dictionary containing notebook data and name
+        :param lint: Whether to lint Python code cells. If None, uses the value from the dictionary or defaults to True.
+        :return: A new Notebook instance
         """
-        return cls(data=d["data"], name=d["name"])
+        # Use the lint parameter from the dictionary if none is provided, otherwise default to True
+        notebook_lint = lint if lint is not None else d.get("lint", True)
+        return cls(data=d["data"], name=d["name"], lint=notebook_lint)
 
     def to_file(self, path: str):
         """
@@ -205,11 +270,13 @@ class Notebook(Base):
         return table
 
     @classmethod
-    def example(cls, randomize: bool = False) -> Notebook:
+    def example(cls, randomize: bool = False, lint: bool = True) -> Notebook:
         """
         Returns an example Notebook instance.
 
         :param randomize: If True, adds a random string one of the cells' output.
+        :param lint: Whether to lint Python code cells. Defaults to True.
+        :return: An example Notebook instance
         """
         addition = "" if not randomize else str(uuid4())
         cells = [
@@ -238,7 +305,7 @@ class Notebook(Base):
             "nbformat_minor": 4,
             "cells": cells,
         }
-        return cls(data=data)
+        return cls(data=data, lint=lint)
 
     def code(self) -> List[str]:
         """
@@ -246,7 +313,7 @@ class Notebook(Base):
         """
         lines = []
         lines.append("from edsl import Notebook")  # Keep as absolute for code generation
-        lines.append(f'nb = Notebook(data={self.data}, name="""{self.name}""")')
+        lines.append(f'nb = Notebook(data={self.data}, name="""{self.name}""", lint={self.lint})')
         return lines
 
     def to_latex(self, filename: str):

edsl/questions/VALIDATION_README.md

@@ -0,0 +1,134 @@
+# EDSL Validation Logging System
+
+This system logs validation failures that occur during question answering and provides tools to analyze these failures to improve the "fix" methods for various question types.
+
+## Background
+
+When a language model's response to a question fails validation (e.g., the response doesn't match the expected format or constraints), EDSL throws a `QuestionAnswerValidationError`. To make these validations more robust, we've added a system to log these failures and analyze common patterns.
+
+## Features
+
+- **Validation Logging**: Automatically logs validation failures to a local file
+- **Log Analysis**: Tools to analyze validation failures by question type and error message
+- **Fix Method Suggestions**: Generates suggestions for improving fix methods based on common failure patterns
+- **CLI Interface**: Command-line tools for managing and analyzing validation logs
+
+## Usage
+
+### Command Line Interface
+
+The validation logging system is integrated with the EDSL CLI:
+
+```bash
+# Show recent validation failure logs
+edsl validation logs
+
+# Show recent logs filtered by question type
+edsl validation logs --type QuestionMultipleChoice
+
+# Save logs to a file
+edsl validation logs --output validation_logs.json
+
+# Clear all validation logs
+edsl validation clear
+
+# Show validation failure statistics
+edsl validation stats
+
+# Get suggestions for improving fix methods
+edsl validation suggest
+
+# Filter suggestions for a specific question type
+edsl validation suggest --type QuestionMultipleChoice
+
+# Generate a comprehensive JSON report
+edsl validation report
+
+# Generate an HTML report and open it in browser
+edsl validation html-report
+
+# Generate HTML report without opening browser
+edsl validation html-report --no-open
+```
+
+You can also use the `make` command to generate reports:
+
+```bash
+# Generate and open HTML validation report
+make validation-report
+
+# Show validation statistics
+make validation-stats
+```
+
+### Programmatic Usage
+
+You can also use the validation logging system programmatically:
+
+```python
+from edsl.questions import (
+    log_validation_failure,
+    get_validation_failure_logs,
+    clear_validation_logs,
+    get_validation_failure_stats,
+    suggest_fix_improvements,
+    export_improvements_report,
+    generate_html_report,
+    generate_and_open_report
+)
+
+# Get recent validation failure logs
+logs = get_validation_failure_logs(n=10)
+
+# Get validation failure statistics
+stats = get_validation_failure_stats()
+
+# Get suggestions for improving fix methods
+suggestions = suggest_fix_improvements()
+
+# Generate a JSON report
+report_path = export_improvements_report()
+
+# Generate an HTML report
+html_report_path = generate_html_report()
+
+# Generate and open HTML report in browser
+generate_and_open_report()
+```
+
+## Implementation Details
+
+The validation logging system consists of the following components:
+
+1. **Validation Logger**: Logs validation failures to a local file
+2. **Validation Analysis**: Analyzes logs to identify patterns and suggest improvements
+3. **HTML Report Generator**: Creates user-friendly HTML reports with visualizations
+4. **CLI Integration**: Provides command-line tools for working with validation logs
+
+### Log Format
+
+Validation failure logs include the following information:
+
+- Timestamp
+- Question type and name
+- Error message
+- Invalid data that failed validation
+- Model schema used for validation
+- Question details (if available)
+- Stack trace
+
+### Storage Location
+
+Logs are stored in the default EDSL log directory:
+
+- Linux/macOS: `~/.edsl/logs/validation_failures.log`
+- Windows: `%USERPROFILE%\.edsl\logs\validation_failures.log`
+
+## Future Improvements
+
+Potential future improvements to the validation logging system:
+
+1. Integration with coop for cloud storage and analysis of validation failures
+2. Machine learning to automatically suggest fix method improvements
+3. Automated tests using common validation failure patterns
+4. A web-based dashboard for visualizing validation failure statistics

edsl/questions/__init__.py

@@ -34,6 +34,7 @@ Derived Question Types:
 - QuestionLinearScale: Linear scale with customizable range and labels
 - QuestionYesNo: Simple binary yes/no response
 - QuestionTopK: Selection of top K items from a list of options
+- QuestionMultipleChoiceWithOther: Multiple choice with option to specify "Other" custom response
 
 Technical Architecture:
 ---------------------
@@ -124,9 +125,19 @@ from .question_likert_five import QuestionLikertFive
 from .question_linear_scale import QuestionLinearScale
 from .question_yes_no import QuestionYesNo
 from .question_top_k import QuestionTopK
+from .question_multiple_choice_with_other import QuestionMultipleChoiceWithOther
 
 from .exceptions import QuestionScenarioRenderError
 
+# Import validation modules
+from .validation_logger import log_validation_failure, get_validation_failure_logs, clear_validation_logs
+from .validation_analysis import (
+    get_validation_failure_stats, 
+    suggest_fix_improvements, 
+    export_improvements_report
+)
+from .validation_html_report import generate_html_report, generate_and_open_report
+
 __all__ = [
     # Exceptions
     "QuestionScenarioRenderError",
@@ -156,4 +167,16 @@ __all__ = [
     "QuestionTopK",
     "QuestionLikertFive",
     "QuestionYesNo",
-]
+    "QuestionMultipleChoiceWithOther",
+    "QuestionMultipleChoiceWithOther",
+    
+    # Validation utilities
+    "log_validation_failure",
+    "get_validation_failure_logs",
+    "clear_validation_logs",
+    "get_validation_failure_stats",
+    "suggest_fix_improvements",
+    "export_improvements_report",
+    "generate_html_report",
+    "generate_and_open_report",
+]

edsl/questions/exceptions.py

@@ -72,6 +72,27 @@ class QuestionAnswerValidationError(QuestionErrors):
         self.data = data
         self.model = model
         super().__init__(self.message)
+        
+        # Log validation failure for analysis
+        try:
+            from .validation_logger import log_validation_failure
+            
+            # Get question type and name if available
+            question_type = getattr(model, "question_type", "unknown")
+            question_name = getattr(model, "question_name", "unknown")
+            
+            # Log the validation failure
+            log_validation_failure(
+                question_type=question_type,
+                question_name=question_name,
+                error_message=str(message),
+                invalid_data=data,
+                model_schema=model.model_json_schema(),
+                question_dict=getattr(model, "to_dict", lambda: None)(),
+            )
+        except Exception:
+            # Silently ignore logging errors to not disrupt normal operation
+            pass
 
     def __str__(self):
         if isinstance(self.message, ValidationError):

edsl/questions/question_dict.py

@@ -76,7 +76,6 @@ def create_dict_response(
     field_definitions = {}
     if len(value_types) == 0:
         value_types = ["str"] * len(answer_keys)  # Default to str if no types provided
-
     for key, t_str in zip(answer_keys, value_types):
         python_type = _parse_type_string(t_str)
         field_definitions[key] = (python_type, Field(...))
@@ -193,27 +192,177 @@ class DictResponseValidator(ResponseValidatorABC):
             'not a dictionary'
         """
         # First try to separate dictionary from trailing comment if they're on the same line
+        original_response = response
         if isinstance(response, str):
             # Try to find where the dictionary ends and comment begins
             try:
-                dict_match = re.match(r"(\{.*?\})(.*)", response.strip())
-                if dict_match:
-                    dict_str, comment = dict_match.groups()
-                    try:
-                        answer_dict = ast.literal_eval(dict_str)
-                        response = {
-                            "answer": answer_dict,
-                            "comment": comment.strip() if comment.strip() else None,
-                        }
-                    except (ValueError, SyntaxError):
-                        pass
-            except Exception:
-                pass
+                # Find the first opening brace
+                response_str = response.strip()
+                if response_str.startswith("{"):
+                    # Count braces to find proper JSON ending
+                    brace_count = 0
+                    dict_end_pos = None
+
+                    for i, char in enumerate(response_str):
+                        if char == "{":
+                            brace_count += 1
+                        elif char == "}":
+                            brace_count -= 1
+                            if brace_count == 0:
+                                dict_end_pos = i + 1
+                                break
+
+                    if dict_end_pos is not None:
+                        dict_str = response_str[:dict_end_pos]
+                        comment = response_str[dict_end_pos:].strip()
+
+                        try:
+                            answer_dict = ast.literal_eval(dict_str)
+                            response = {
+                                "answer": answer_dict,
+                                "comment": comment if comment else None,
+                            }
+                            if verbose:
+                                print(
+                                    f"Successfully split answer from comment. Comment length: {len(comment) if comment else 0}"
+                                )
+                        except (ValueError, SyntaxError) as e:
+                            if verbose:
+                                print(f"Failed to parse dictionary: {e}")
+            except Exception as e:
+                if verbose:
+                    print(f"Exception during dictionary parsing: {e}")
 
         # Continue with existing fix logic
         if "answer" not in response or not isinstance(response["answer"], dict):
             if verbose:
                 print("Cannot fix response: 'answer' field missing or not a dictionary")
+
+            # Special case: if we have the original string response, try a more direct parsing approach
+            if isinstance(
+                original_response, str
+            ) and original_response.strip().startswith("{"):
+                try:
+                    # Try to parse the JSON part directly, skipping nested comments
+                    response_str = original_response.strip()
+                    import json
+
+                    # Find where the dict ends by tracking nested braces
+                    brace_count = 0
+                    dict_end_pos = None
+
+                    for i, char in enumerate(response_str):
+                        if char == "{":
+                            brace_count += 1
+                        elif char == "}":
+                            brace_count -= 1
+                            if brace_count == 0:
+                                dict_end_pos = i + 1
+                                break
+
+                    if dict_end_pos is not None:
+                        dict_str = response_str[:dict_end_pos]
+                        comment = response_str[dict_end_pos:].strip()
+
+                        # Try parsing with JSON first (faster but stricter)
+                        try:
+                            dict_str = dict_str.replace(
+                                "'", '"'
+                            )  # Convert Python quotes to JSON quotes
+                            dict_str = dict_str.replace("False", "false").replace(
+                                "True", "true"
+                            )  # Fix booleans
+                            answer_dict = json.loads(dict_str)
+                        except json.JSONDecodeError:
+                            # Fall back to ast.literal_eval (safer)
+                            try:
+                                answer_dict = ast.literal_eval(dict_str)
+                            except (ValueError, SyntaxError):
+                                if verbose:
+                                    print("Could not parse the dictionary part")
+                                return original_response
+
+                        # Now fix types
+                        fixed_answer = {}
+                        for key, type_str in zip(
+                            self.answer_keys, getattr(self, "value_types", [])
+                        ):
+                            if key in answer_dict:
+                                value = answer_dict[key]
+                                # Convert types
+                                if type_str == "int" and not isinstance(value, int):
+                                    try:
+                                        fixed_answer[key] = int(value)
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from {type(value).__name__} to int"
+                                            )
+                                    except (ValueError, TypeError):
+                                        fixed_answer[key] = value
+
+                                elif type_str == "float" and not isinstance(
+                                    value, float
+                                ):
+                                    try:
+                                        fixed_answer[key] = float(value)
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from {type(value).__name__} to float"
+                                            )
+                                    except (ValueError, TypeError):
+                                        fixed_answer[key] = value
+
+                                elif (
+                                    type_str.startswith("list[") or type_str == "list"
+                                ) and not isinstance(value, list):
+                                    # Convert string to list by splitting
+                                    if isinstance(value, str):
+                                        items = [
+                                            item.strip() for item in value.split(",")
+                                        ]
+                                        fixed_answer[key] = items
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from string to list: {items}"
+                                            )
+                                    else:
+                                        fixed_answer[key] = value
+                                else:
+                                    fixed_answer[key] = value
+                            else:
+                                # Key not in answer, set a default
+                                if type_str == "int":
+                                    fixed_answer[key] = 0
+                                elif type_str == "float":
+                                    fixed_answer[key] = 0.0
+                                elif type_str.startswith("list") or type_str == "list":
+                                    fixed_answer[key] = []
+                                else:
+                                    fixed_answer[key] = ""
+
+                        # Construct final fixed response
+                        fixed_response = {
+                            "answer": fixed_answer,
+                            "comment": comment if comment else None,
+                            "generated_tokens": None,
+                        }
+
+                        if verbose:
+                            print(f"Directly fixed response with type conversion")
+
+                        try:
+                            # Try to validate
+                            self.response_model.model_validate(fixed_response)
+                            if verbose:
+                                print("Successfully validated fixed response")
+                            return fixed_response
+                        except Exception as e:
+                            if verbose:
+                                print(f"Validation of direct fix failed: {e}")
+                except Exception as e:
+                    if verbose:
+                        print(f"Error during direct parsing: {e}")
+
             return response
 
         answer_dict = response["answer"]
@@ -246,7 +395,9 @@ class DictResponseValidator(ResponseValidatorABC):
                     except (ValueError, TypeError):
                         pass
 
-                elif type_str.startswith("list[") and not isinstance(value, list):
+                elif (
+                    type_str.startswith("list[") or type_str == "list"
+                ) and not isinstance(value, list):
                     # Try to convert string to list by splitting
                     if isinstance(value, str):
                         items = [item.strip() for item in value.split(",")]
@@ -267,7 +418,8 @@ class DictResponseValidator(ResponseValidatorABC):
         fixed_response = {
             "answer": fixed_answer,
             "comment": response.get("comment"),
-            "generated_tokens": response.get("generated_tokens"),
+            "generated_tokens": response.get("generated_tokens")
+            or response,  # Ensure generated_tokens is captured
         }
 
         try:
@@ -279,6 +431,39 @@ class DictResponseValidator(ResponseValidatorABC):
         except Exception as e:
             if verbose:
                 print(f"Validation failed for fixed answer: {e}")
+
+            # If still failing, try one more time with default values for missing keys
+            if hasattr(self, "answer_keys") and hasattr(self, "value_types"):
+                for key, type_str in zip(
+                    self.answer_keys, getattr(self, "value_types", [])
+                ):
+                    if key not in fixed_answer:
+                        if type_str == "int":
+                            fixed_answer[key] = 0
+                        elif type_str == "float":
+                            fixed_answer[key] = 0.0
+                        elif type_str.startswith("list") or type_str == "list":
+                            fixed_answer[key] = []
+                        else:
+                            fixed_answer[key] = ""
+
+                # Try again with all keys
+                fixed_response = {
+                    "answer": fixed_answer,
+                    "comment": response.get("comment"),
+                    "generated_tokens": response.get("generated_tokens"),
+                }
+
+                try:
+                    # Validate the fixed answer
+                    self.response_model.model_validate(fixed_response)
+                    if verbose:
+                        print("Successfully fixed response with defaults")
+                    return fixed_response
+                except Exception as e:
+                    if verbose:
+                        print(f"Validation still failed after adding defaults: {e}")
+
             return response
 
     valid_examples = [