edsl 0.1.53__py3-none-any.whl → 0.1.55__py3-none-any.whl

edsl/questions/VALIDATION_README.md

@@ -0,0 +1,134 @@
+# EDSL Validation Logging System
+
+This system logs validation failures that occur during question answering and provides tools to analyze these failures to improve the "fix" methods for various question types.
+
+## Background
+
+When a language model's response to a question fails validation (e.g., the response doesn't match the expected format or constraints), EDSL throws a `QuestionAnswerValidationError`. To make these validations more robust, we've added a system to log these failures and analyze common patterns.
+
+## Features
+
+- **Validation Logging**: Automatically logs validation failures to a local file
+- **Log Analysis**: Tools to analyze validation failures by question type and error message
+- **Fix Method Suggestions**: Generates suggestions for improving fix methods based on common failure patterns
+- **CLI Interface**: Command-line tools for managing and analyzing validation logs
+
+## Usage
+
+### Command Line Interface
+
+The validation logging system is integrated with the EDSL CLI:
+
+```bash
+# Show recent validation failure logs
+edsl validation logs
+
+# Show recent logs filtered by question type
+edsl validation logs --type QuestionMultipleChoice
+
+# Save logs to a file
+edsl validation logs --output validation_logs.json
+
+# Clear all validation logs
+edsl validation clear
+
+# Show validation failure statistics
+edsl validation stats
+
+# Get suggestions for improving fix methods
+edsl validation suggest
+
+# Filter suggestions for a specific question type
+edsl validation suggest --type QuestionMultipleChoice
+
+# Generate a comprehensive JSON report
+edsl validation report
+
+# Generate an HTML report and open it in browser
+edsl validation html-report
+
+# Generate HTML report without opening browser
+edsl validation html-report --no-open
+```
+
+You can also use the `make` command to generate reports:
+
+```bash
+# Generate and open HTML validation report
+make validation-report
+
+# Show validation statistics
+make validation-stats
+```
+
+### Programmatic Usage
+
+You can also use the validation logging system programmatically:
+
+```python
+from edsl.questions import (
+    log_validation_failure,
+    get_validation_failure_logs,
+    clear_validation_logs,
+    get_validation_failure_stats,
+    suggest_fix_improvements,
+    export_improvements_report,
+    generate_html_report,
+    generate_and_open_report
+)
+
+# Get recent validation failure logs
+logs = get_validation_failure_logs(n=10)
+
+# Get validation failure statistics
+stats = get_validation_failure_stats()
+
+# Get suggestions for improving fix methods
+suggestions = suggest_fix_improvements()
+
+# Generate a JSON report
+report_path = export_improvements_report()
+
+# Generate an HTML report
+html_report_path = generate_html_report()
+
+# Generate and open HTML report in browser
+generate_and_open_report()
+```
+
+## Implementation Details
+
+The validation logging system consists of the following components:
+
+1. **Validation Logger**: Logs validation failures to a local file
+2. **Validation Analysis**: Analyzes logs to identify patterns and suggest improvements
+3. **HTML Report Generator**: Creates user-friendly HTML reports with visualizations
+4. **CLI Integration**: Provides command-line tools for working with validation logs
+
+### Log Format
+
+Validation failure logs include the following information:
+
+- Timestamp
+- Question type and name
+- Error message
+- Invalid data that failed validation
+- Model schema used for validation
+- Question details (if available)
+- Stack trace
+
+### Storage Location
+
+Logs are stored in the default EDSL log directory:
+
+- Linux/macOS: `~/.edsl/logs/validation_failures.log`
+- Windows: `%USERPROFILE%\.edsl\logs\validation_failures.log`
+
+## Future Improvements
+
+Potential future improvements to the validation logging system:
+
+1. Integration with coop for cloud storage and analysis of validation failures
+2. Machine learning to automatically suggest fix method improvements
+3. Automated tests using common validation failure patterns
+4. A web-based dashboard for visualizing validation failure statistics

edsl/questions/__init__.py

@@ -34,6 +34,7 @@ Derived Question Types:
 - QuestionLinearScale: Linear scale with customizable range and labels
 - QuestionYesNo: Simple binary yes/no response
 - QuestionTopK: Selection of top K items from a list of options
+- QuestionMultipleChoiceWithOther: Multiple choice with option to specify "Other" custom response
 
 Technical Architecture:
 ---------------------
@@ -124,9 +125,19 @@ from .question_likert_five import QuestionLikertFive
 from .question_linear_scale import QuestionLinearScale
 from .question_yes_no import QuestionYesNo
 from .question_top_k import QuestionTopK
+from .question_multiple_choice_with_other import QuestionMultipleChoiceWithOther
 
 from .exceptions import QuestionScenarioRenderError
 
+# Import validation modules
+from .validation_logger import log_validation_failure, get_validation_failure_logs, clear_validation_logs
+from .validation_analysis import (
+    get_validation_failure_stats, 
+    suggest_fix_improvements, 
+    export_improvements_report
+)
+from .validation_html_report import generate_html_report, generate_and_open_report
+
 __all__ = [
     # Exceptions
     "QuestionScenarioRenderError",
@@ -156,4 +167,16 @@ __all__ = [
     "QuestionTopK",
     "QuestionLikertFive",
     "QuestionYesNo",
-]
+    "QuestionMultipleChoiceWithOther",
+    "QuestionMultipleChoiceWithOther",
+    
+    # Validation utilities
+    "log_validation_failure",
+    "get_validation_failure_logs",
+    "clear_validation_logs",
+    "get_validation_failure_stats",
+    "suggest_fix_improvements",
+    "export_improvements_report",
+    "generate_html_report",
+    "generate_and_open_report",
+]

edsl/questions/exceptions.py

@@ -72,6 +72,27 @@ class QuestionAnswerValidationError(QuestionErrors):
         self.data = data
         self.model = model
         super().__init__(self.message)
+        
+        # Log validation failure for analysis
+        try:
+            from .validation_logger import log_validation_failure
+            
+            # Get question type and name if available
+            question_type = getattr(model, "question_type", "unknown")
+            question_name = getattr(model, "question_name", "unknown")
+            
+            # Log the validation failure
+            log_validation_failure(
+                question_type=question_type,
+                question_name=question_name,
+                error_message=str(message),
+                invalid_data=data,
+                model_schema=model.model_json_schema(),
+                question_dict=getattr(model, "to_dict", lambda: None)(),
+            )
+        except Exception:
+            # Silently ignore logging errors to not disrupt normal operation
+            pass
 
     def __str__(self):
         if isinstance(self.message, ValidationError):