edsl 0.1.54__py3-none-any.whl → 0.1.55__py3-none-any.whl

edsl/questions/VALIDATION_README.md

@@ -0,0 +1,134 @@
+# EDSL Validation Logging System
+
+This system logs validation failures that occur during question answering and provides tools to analyze these failures to improve the "fix" methods for various question types.
+
+## Background
+
+When a language model's response to a question fails validation (e.g., the response doesn't match the expected format or constraints), EDSL throws a `QuestionAnswerValidationError`. To make these validations more robust, we've added a system to log these failures and analyze common patterns.
+
+## Features
+
+- **Validation Logging**: Automatically logs validation failures to a local file
+- **Log Analysis**: Tools to analyze validation failures by question type and error message
+- **Fix Method Suggestions**: Generates suggestions for improving fix methods based on common failure patterns
+- **CLI Interface**: Command-line tools for managing and analyzing validation logs
+
+## Usage
+
+### Command Line Interface
+
+The validation logging system is integrated with the EDSL CLI:
+
+```bash
+# Show recent validation failure logs
+edsl validation logs
+
+# Show recent logs filtered by question type
+edsl validation logs --type QuestionMultipleChoice
+
+# Save logs to a file
+edsl validation logs --output validation_logs.json
+
+# Clear all validation logs
+edsl validation clear
+
+# Show validation failure statistics
+edsl validation stats
+
+# Get suggestions for improving fix methods
+edsl validation suggest
+
+# Filter suggestions for a specific question type
+edsl validation suggest --type QuestionMultipleChoice
+
+# Generate a comprehensive JSON report
+edsl validation report
+
+# Generate an HTML report and open it in browser
+edsl validation html-report
+
+# Generate HTML report without opening browser
+edsl validation html-report --no-open
+```
+
+You can also use the `make` command to generate reports:
+
+```bash
+# Generate and open HTML validation report
+make validation-report
+
+# Show validation statistics
+make validation-stats
+```
+
+### Programmatic Usage
+
+You can also use the validation logging system programmatically:
+
+```python
+from edsl.questions import (
+    log_validation_failure,
+    get_validation_failure_logs,
+    clear_validation_logs,
+    get_validation_failure_stats,
+    suggest_fix_improvements,
+    export_improvements_report,
+    generate_html_report,
+    generate_and_open_report
+)
+
+# Get recent validation failure logs
+logs = get_validation_failure_logs(n=10)
+
+# Get validation failure statistics
+stats = get_validation_failure_stats()
+
+# Get suggestions for improving fix methods
+suggestions = suggest_fix_improvements()
+
+# Generate a JSON report
+report_path = export_improvements_report()
+
+# Generate an HTML report
+html_report_path = generate_html_report()
+
+# Generate and open HTML report in browser
+generate_and_open_report()
+```
+
+## Implementation Details
+
+The validation logging system consists of the following components:
+
+1. **Validation Logger**: Logs validation failures to a local file
+2. **Validation Analysis**: Analyzes logs to identify patterns and suggest improvements
+3. **HTML Report Generator**: Creates user-friendly HTML reports with visualizations
+4. **CLI Integration**: Provides command-line tools for working with validation logs
+
+### Log Format
+
+Validation failure logs include the following information:
+
+- Timestamp
+- Question type and name
+- Error message
+- Invalid data that failed validation
+- Model schema used for validation
+- Question details (if available)
+- Stack trace
+
+### Storage Location
+
+Logs are stored in the default EDSL log directory:
+
+- Linux/macOS: `~/.edsl/logs/validation_failures.log`
+- Windows: `%USERPROFILE%\.edsl\logs\validation_failures.log`
+
+## Future Improvements
+
+Potential future improvements to the validation logging system:
+
+1. Integration with coop for cloud storage and analysis of validation failures
+2. Machine learning to automatically suggest fix method improvements
+3. Automated tests using common validation failure patterns
+4. A web-based dashboard for visualizing validation failure statistics

edsl/questions/__init__.py

@@ -34,6 +34,7 @@ Derived Question Types:
 - QuestionLinearScale: Linear scale with customizable range and labels
 - QuestionYesNo: Simple binary yes/no response
 - QuestionTopK: Selection of top K items from a list of options
+- QuestionMultipleChoiceWithOther: Multiple choice with option to specify "Other" custom response
 
 Technical Architecture:
 ---------------------
@@ -124,9 +125,19 @@ from .question_likert_five import QuestionLikertFive
 from .question_linear_scale import QuestionLinearScale
 from .question_yes_no import QuestionYesNo
 from .question_top_k import QuestionTopK
+from .question_multiple_choice_with_other import QuestionMultipleChoiceWithOther
 
 from .exceptions import QuestionScenarioRenderError
 
+# Import validation modules
+from .validation_logger import log_validation_failure, get_validation_failure_logs, clear_validation_logs
+from .validation_analysis import (
+    get_validation_failure_stats, 
+    suggest_fix_improvements, 
+    export_improvements_report
+)
+from .validation_html_report import generate_html_report, generate_and_open_report
+
 __all__ = [
     # Exceptions
     "QuestionScenarioRenderError",
@@ -156,4 +167,16 @@ __all__ = [
     "QuestionTopK",
     "QuestionLikertFive",
     "QuestionYesNo",
-]
+    "QuestionMultipleChoiceWithOther",
+    "QuestionMultipleChoiceWithOther",
+    
+    # Validation utilities
+    "log_validation_failure",
+    "get_validation_failure_logs",
+    "clear_validation_logs",
+    "get_validation_failure_stats",
+    "suggest_fix_improvements",
+    "export_improvements_report",
+    "generate_html_report",
+    "generate_and_open_report",
+]

edsl/questions/exceptions.py

@@ -72,6 +72,27 @@ class QuestionAnswerValidationError(QuestionErrors):
         self.data = data
         self.model = model
         super().__init__(self.message)
+        
+        # Log validation failure for analysis
+        try:
+            from .validation_logger import log_validation_failure
+            
+            # Get question type and name if available
+            question_type = getattr(model, "question_type", "unknown")
+            question_name = getattr(model, "question_name", "unknown")
+            
+            # Log the validation failure
+            log_validation_failure(
+                question_type=question_type,
+                question_name=question_name,
+                error_message=str(message),
+                invalid_data=data,
+                model_schema=model.model_json_schema(),
+                question_dict=getattr(model, "to_dict", lambda: None)(),
+            )
+        except Exception:
+            # Silently ignore logging errors to not disrupt normal operation
+            pass
 
     def __str__(self):
         if isinstance(self.message, ValidationError):

edsl/questions/question_dict.py

@@ -76,7 +76,6 @@ def create_dict_response(
     field_definitions = {}
     if len(value_types) == 0:
         value_types = ["str"] * len(answer_keys)  # Default to str if no types provided
-
     for key, t_str in zip(answer_keys, value_types):
         python_type = _parse_type_string(t_str)
         field_definitions[key] = (python_type, Field(...))
@@ -193,27 +192,177 @@ class DictResponseValidator(ResponseValidatorABC):
             'not a dictionary'
         """
         # First try to separate dictionary from trailing comment if they're on the same line
+        original_response = response
         if isinstance(response, str):
             # Try to find where the dictionary ends and comment begins
             try:
-                dict_match = re.match(r"(\{.*?\})(.*)", response.strip())
-                if dict_match:
-                    dict_str, comment = dict_match.groups()
-                    try:
-                        answer_dict = ast.literal_eval(dict_str)
-                        response = {
-                            "answer": answer_dict,
-                            "comment": comment.strip() if comment.strip() else None,
-                        }
-                    except (ValueError, SyntaxError):
-                        pass
-            except Exception:
-                pass
+                # Find the first opening brace
+                response_str = response.strip()
+                if response_str.startswith("{"):
+                    # Count braces to find proper JSON ending
+                    brace_count = 0
+                    dict_end_pos = None
+
+                    for i, char in enumerate(response_str):
+                        if char == "{":
+                            brace_count += 1
+                        elif char == "}":
+                            brace_count -= 1
+                            if brace_count == 0:
+                                dict_end_pos = i + 1
+                                break
+
+                    if dict_end_pos is not None:
+                        dict_str = response_str[:dict_end_pos]
+                        comment = response_str[dict_end_pos:].strip()
+
+                        try:
+                            answer_dict = ast.literal_eval(dict_str)
+                            response = {
+                                "answer": answer_dict,
+                                "comment": comment if comment else None,
+                            }
+                            if verbose:
+                                print(
+                                    f"Successfully split answer from comment. Comment length: {len(comment) if comment else 0}"
+                                )
+                        except (ValueError, SyntaxError) as e:
+                            if verbose:
+                                print(f"Failed to parse dictionary: {e}")
+            except Exception as e:
+                if verbose:
+                    print(f"Exception during dictionary parsing: {e}")
 
         # Continue with existing fix logic
         if "answer" not in response or not isinstance(response["answer"], dict):
             if verbose:
                 print("Cannot fix response: 'answer' field missing or not a dictionary")
+
+            # Special case: if we have the original string response, try a more direct parsing approach
+            if isinstance(
+                original_response, str
+            ) and original_response.strip().startswith("{"):
+                try:
+                    # Try to parse the JSON part directly, skipping nested comments
+                    response_str = original_response.strip()
+                    import json
+
+                    # Find where the dict ends by tracking nested braces
+                    brace_count = 0
+                    dict_end_pos = None
+
+                    for i, char in enumerate(response_str):
+                        if char == "{":
+                            brace_count += 1
+                        elif char == "}":
+                            brace_count -= 1
+                            if brace_count == 0:
+                                dict_end_pos = i + 1
+                                break
+
+                    if dict_end_pos is not None:
+                        dict_str = response_str[:dict_end_pos]
+                        comment = response_str[dict_end_pos:].strip()
+
+                        # Try parsing with JSON first (faster but stricter)
+                        try:
+                            dict_str = dict_str.replace(
+                                "'", '"'
+                            )  # Convert Python quotes to JSON quotes
+                            dict_str = dict_str.replace("False", "false").replace(
+                                "True", "true"
+                            )  # Fix booleans
+                            answer_dict = json.loads(dict_str)
+                        except json.JSONDecodeError:
+                            # Fall back to ast.literal_eval (safer)
+                            try:
+                                answer_dict = ast.literal_eval(dict_str)
+                            except (ValueError, SyntaxError):
+                                if verbose:
+                                    print("Could not parse the dictionary part")
+                                return original_response
+
+                        # Now fix types
+                        fixed_answer = {}
+                        for key, type_str in zip(
+                            self.answer_keys, getattr(self, "value_types", [])
+                        ):
+                            if key in answer_dict:
+                                value = answer_dict[key]
+                                # Convert types
+                                if type_str == "int" and not isinstance(value, int):
+                                    try:
+                                        fixed_answer[key] = int(value)
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from {type(value).__name__} to int"
+                                            )
+                                    except (ValueError, TypeError):
+                                        fixed_answer[key] = value
+
+                                elif type_str == "float" and not isinstance(
+                                    value, float
+                                ):
+                                    try:
+                                        fixed_answer[key] = float(value)
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from {type(value).__name__} to float"
+                                            )
+                                    except (ValueError, TypeError):
+                                        fixed_answer[key] = value
+
+                                elif (
+                                    type_str.startswith("list[") or type_str == "list"
+                                ) and not isinstance(value, list):
+                                    # Convert string to list by splitting
+                                    if isinstance(value, str):
+                                        items = [
+                                            item.strip() for item in value.split(",")
+                                        ]
+                                        fixed_answer[key] = items
+                                        if verbose:
+                                            print(
+                                                f"Converted '{key}' from string to list: {items}"
+                                            )
+                                    else:
+                                        fixed_answer[key] = value
+                                else:
+                                    fixed_answer[key] = value
+                            else:
+                                # Key not in answer, set a default
+                                if type_str == "int":
+                                    fixed_answer[key] = 0
+                                elif type_str == "float":
+                                    fixed_answer[key] = 0.0
+                                elif type_str.startswith("list") or type_str == "list":
+                                    fixed_answer[key] = []
+                                else:
+                                    fixed_answer[key] = ""
+
+                        # Construct final fixed response
+                        fixed_response = {
+                            "answer": fixed_answer,
+                            "comment": comment if comment else None,
+                            "generated_tokens": None,
+                        }
+
+                        if verbose:
+                            print(f"Directly fixed response with type conversion")
+
+                        try:
+                            # Try to validate
+                            self.response_model.model_validate(fixed_response)
+                            if verbose:
+                                print("Successfully validated fixed response")
+                            return fixed_response
+                        except Exception as e:
+                            if verbose:
+                                print(f"Validation of direct fix failed: {e}")
+                except Exception as e:
+                    if verbose:
+                        print(f"Error during direct parsing: {e}")
+
             return response
 
         answer_dict = response["answer"]
@@ -246,7 +395,9 @@ class DictResponseValidator(ResponseValidatorABC):
                     except (ValueError, TypeError):
                         pass
 
-                elif type_str.startswith("list[") and not isinstance(value, list):
+                elif (
+                    type_str.startswith("list[") or type_str == "list"
+                ) and not isinstance(value, list):
                     # Try to convert string to list by splitting
                     if isinstance(value, str):
                         items = [item.strip() for item in value.split(",")]
@@ -267,7 +418,8 @@ class DictResponseValidator(ResponseValidatorABC):
         fixed_response = {
             "answer": fixed_answer,
             "comment": response.get("comment"),
-            "generated_tokens": response.get("generated_tokens"),
+            "generated_tokens": response.get("generated_tokens")
+            or response,  # Ensure generated_tokens is captured
         }
 
         try:
@@ -279,6 +431,39 @@ class DictResponseValidator(ResponseValidatorABC):
         except Exception as e:
             if verbose:
                 print(f"Validation failed for fixed answer: {e}")
+
+            # If still failing, try one more time with default values for missing keys
+            if hasattr(self, "answer_keys") and hasattr(self, "value_types"):
+                for key, type_str in zip(
+                    self.answer_keys, getattr(self, "value_types", [])
+                ):
+                    if key not in fixed_answer:
+                        if type_str == "int":
+                            fixed_answer[key] = 0
+                        elif type_str == "float":
+                            fixed_answer[key] = 0.0
+                        elif type_str.startswith("list") or type_str == "list":
+                            fixed_answer[key] = []
+                        else:
+                            fixed_answer[key] = ""
+
+                # Try again with all keys
+                fixed_response = {
+                    "answer": fixed_answer,
+                    "comment": response.get("comment"),
+                    "generated_tokens": response.get("generated_tokens"),
+                }
+
+                try:
+                    # Validate the fixed answer
+                    self.response_model.model_validate(fixed_response)
+                    if verbose:
+                        print("Successfully fixed response with defaults")
+                    return fixed_response
+                except Exception as e:
+                    if verbose:
+                        print(f"Validation still failed after adding defaults: {e}")
+
             return response
 
     valid_examples = [