edsl 0.1.54__py3-none-any.whl → 0.1.56__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__init__.py +8 -1
- edsl/__init__original.py +134 -0
- edsl/__version__.py +1 -1
- edsl/agents/agent.py +29 -0
- edsl/agents/agent_list.py +36 -1
- edsl/base/base_class.py +281 -151
- edsl/base/data_transfer_models.py +15 -4
- edsl/buckets/__init__.py +8 -3
- edsl/buckets/bucket_collection.py +9 -3
- edsl/buckets/model_buckets.py +4 -2
- edsl/buckets/token_bucket.py +2 -2
- edsl/buckets/token_bucket_client.py +5 -3
- edsl/caching/cache.py +131 -62
- edsl/caching/cache_entry.py +70 -58
- edsl/caching/sql_dict.py +17 -0
- edsl/cli.py +99 -0
- edsl/config/config_class.py +16 -0
- edsl/conversation/__init__.py +31 -0
- edsl/coop/coop.py +276 -242
- edsl/coop/coop_jobs_objects.py +59 -0
- edsl/coop/coop_objects.py +29 -0
- edsl/coop/coop_regular_objects.py +26 -0
- edsl/coop/utils.py +24 -19
- edsl/dataset/dataset.py +338 -101
- edsl/dataset/dataset_operations_mixin.py +216 -180
- edsl/db_list/sqlite_list.py +349 -0
- edsl/inference_services/__init__.py +40 -5
- edsl/inference_services/exceptions.py +11 -0
- edsl/inference_services/services/anthropic_service.py +5 -2
- edsl/inference_services/services/aws_bedrock.py +6 -2
- edsl/inference_services/services/azure_ai.py +6 -2
- edsl/inference_services/services/google_service.py +7 -3
- edsl/inference_services/services/mistral_ai_service.py +6 -2
- edsl/inference_services/services/open_ai_service.py +6 -2
- edsl/inference_services/services/perplexity_service.py +6 -2
- edsl/inference_services/services/test_service.py +94 -5
- edsl/interviews/answering_function.py +167 -59
- edsl/interviews/interview.py +124 -72
- edsl/interviews/interview_task_manager.py +10 -0
- edsl/interviews/request_token_estimator.py +8 -0
- edsl/invigilators/invigilators.py +35 -13
- edsl/jobs/async_interview_runner.py +146 -104
- edsl/jobs/data_structures.py +6 -4
- edsl/jobs/decorators.py +61 -0
- edsl/jobs/fetch_invigilator.py +61 -18
- edsl/jobs/html_table_job_logger.py +14 -2
- edsl/jobs/jobs.py +180 -104
- edsl/jobs/jobs_component_constructor.py +2 -2
- edsl/jobs/jobs_interview_constructor.py +2 -0
- edsl/jobs/jobs_pricing_estimation.py +154 -113
- edsl/jobs/jobs_remote_inference_logger.py +4 -0
- edsl/jobs/jobs_runner_status.py +30 -25
- edsl/jobs/progress_bar_manager.py +79 -0
- edsl/jobs/remote_inference.py +35 -1
- edsl/key_management/key_lookup_builder.py +6 -1
- edsl/language_models/language_model.py +110 -12
- edsl/language_models/model.py +10 -3
- edsl/language_models/price_manager.py +176 -71
- edsl/language_models/registry.py +5 -0
- edsl/notebooks/notebook.py +77 -10
- edsl/questions/VALIDATION_README.md +134 -0
- edsl/questions/__init__.py +24 -1
- edsl/questions/exceptions.py +21 -0
- edsl/questions/question_dict.py +201 -16
- edsl/questions/question_multiple_choice_with_other.py +624 -0
- edsl/questions/question_registry.py +2 -1
- edsl/questions/templates/multiple_choice_with_other/__init__.py +0 -0
- edsl/questions/templates/multiple_choice_with_other/answering_instructions.jinja +15 -0
- edsl/questions/templates/multiple_choice_with_other/question_presentation.jinja +17 -0
- edsl/questions/validation_analysis.py +185 -0
- edsl/questions/validation_cli.py +131 -0
- edsl/questions/validation_html_report.py +404 -0
- edsl/questions/validation_logger.py +136 -0
- edsl/results/result.py +115 -46
- edsl/results/results.py +702 -171
- edsl/scenarios/construct_download_link.py +16 -3
- edsl/scenarios/directory_scanner.py +226 -226
- edsl/scenarios/file_methods.py +5 -0
- edsl/scenarios/file_store.py +150 -9
- edsl/scenarios/handlers/__init__.py +5 -1
- edsl/scenarios/handlers/mp4_file_store.py +104 -0
- edsl/scenarios/handlers/webm_file_store.py +104 -0
- edsl/scenarios/scenario.py +120 -101
- edsl/scenarios/scenario_list.py +800 -727
- edsl/scenarios/scenario_list_gc_test.py +146 -0
- edsl/scenarios/scenario_list_memory_test.py +214 -0
- edsl/scenarios/scenario_list_source_refactor.md +35 -0
- edsl/scenarios/scenario_selector.py +5 -4
- edsl/scenarios/scenario_source.py +1990 -0
- edsl/scenarios/tests/test_scenario_list_sources.py +52 -0
- edsl/surveys/survey.py +22 -0
- edsl/tasks/__init__.py +4 -2
- edsl/tasks/task_history.py +198 -36
- edsl/tests/scenarios/test_ScenarioSource.py +51 -0
- edsl/tests/scenarios/test_scenario_list_sources.py +51 -0
- edsl/utilities/__init__.py +2 -1
- edsl/utilities/decorators.py +121 -0
- edsl/utilities/memory_debugger.py +1010 -0
- {edsl-0.1.54.dist-info → edsl-0.1.56.dist-info}/METADATA +51 -76
- {edsl-0.1.54.dist-info → edsl-0.1.56.dist-info}/RECORD +103 -79
- edsl/jobs/jobs_runner_asyncio.py +0 -281
- edsl/language_models/unused/fake_openai_service.py +0 -60
- {edsl-0.1.54.dist-info → edsl-0.1.56.dist-info}/LICENSE +0 -0
- {edsl-0.1.54.dist-info → edsl-0.1.56.dist-info}/WHEEL +0 -0
- {edsl-0.1.54.dist-info → edsl-0.1.56.dist-info}/entry_points.txt +0 -0
|
@@ -0,0 +1,404 @@
|
|
|
1
|
+
"""Generate an HTML report for validation failures.
|
|
2
|
+
|
|
3
|
+
This module provides functionality to create an HTML report of validation failures,
|
|
4
|
+
including statistics, suggestions for improvements, and examples of common failures.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Dict, List, Optional
|
|
12
|
+
|
|
13
|
+
from ..config import CONFIG
|
|
14
|
+
from .validation_analysis import (
|
|
15
|
+
get_validation_failure_stats,
|
|
16
|
+
suggest_fix_improvements,
|
|
17
|
+
export_improvements_report
|
|
18
|
+
)
|
|
19
|
+
from .validation_logger import get_validation_failure_logs
|
|
20
|
+
|
|
21
|
+
HTML_TEMPLATE = """
|
|
22
|
+
<!DOCTYPE html>
|
|
23
|
+
<html lang="en">
|
|
24
|
+
<head>
|
|
25
|
+
<meta charset="UTF-8">
|
|
26
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
27
|
+
<title>EDSL Validation Failures Report</title>
|
|
28
|
+
<style>
|
|
29
|
+
body {
|
|
30
|
+
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
31
|
+
line-height: 1.6;
|
|
32
|
+
color: #333;
|
|
33
|
+
max-width: 1200px;
|
|
34
|
+
margin: 0 auto;
|
|
35
|
+
padding: 20px;
|
|
36
|
+
}
|
|
37
|
+
h1, h2, h3, h4 {
|
|
38
|
+
color: #2c3e50;
|
|
39
|
+
}
|
|
40
|
+
.header {
|
|
41
|
+
border-bottom: 1px solid #eee;
|
|
42
|
+
padding-bottom: 10px;
|
|
43
|
+
margin-bottom: 20px;
|
|
44
|
+
display: flex;
|
|
45
|
+
justify-content: space-between;
|
|
46
|
+
align-items: center;
|
|
47
|
+
}
|
|
48
|
+
.timestamp {
|
|
49
|
+
color: #7f8c8d;
|
|
50
|
+
font-size: 0.9em;
|
|
51
|
+
}
|
|
52
|
+
.summary {
|
|
53
|
+
background-color: #f8f9fa;
|
|
54
|
+
border-radius: 5px;
|
|
55
|
+
padding: 15px;
|
|
56
|
+
margin-bottom: 20px;
|
|
57
|
+
}
|
|
58
|
+
.stats-container, .suggestions-container, .examples-container {
|
|
59
|
+
margin-bottom: 30px;
|
|
60
|
+
}
|
|
61
|
+
table {
|
|
62
|
+
width: 100%;
|
|
63
|
+
border-collapse: collapse;
|
|
64
|
+
margin-bottom: 20px;
|
|
65
|
+
}
|
|
66
|
+
th, td {
|
|
67
|
+
padding: 12px 15px;
|
|
68
|
+
text-align: left;
|
|
69
|
+
border-bottom: 1px solid #ddd;
|
|
70
|
+
}
|
|
71
|
+
th {
|
|
72
|
+
background-color: #f8f9fa;
|
|
73
|
+
font-weight: 600;
|
|
74
|
+
}
|
|
75
|
+
tr:hover {
|
|
76
|
+
background-color: #f5f5f5;
|
|
77
|
+
}
|
|
78
|
+
.suggestion {
|
|
79
|
+
background-color: #e3f2fd;
|
|
80
|
+
border-left: 4px solid #2196f3;
|
|
81
|
+
padding: 10px 15px;
|
|
82
|
+
margin-bottom: 10px;
|
|
83
|
+
border-radius: 0 4px 4px 0;
|
|
84
|
+
}
|
|
85
|
+
.card {
|
|
86
|
+
border: 1px solid #ddd;
|
|
87
|
+
border-radius: 4px;
|
|
88
|
+
padding: 15px;
|
|
89
|
+
margin-bottom: 20px;
|
|
90
|
+
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
|
91
|
+
}
|
|
92
|
+
.card-header {
|
|
93
|
+
font-weight: 600;
|
|
94
|
+
margin-bottom: 10px;
|
|
95
|
+
padding-bottom: 10px;
|
|
96
|
+
border-bottom: 1px solid #eee;
|
|
97
|
+
}
|
|
98
|
+
.example {
|
|
99
|
+
background-color: #fff8e1;
|
|
100
|
+
border-left: 4px solid #ffc107;
|
|
101
|
+
padding: 10px 15px;
|
|
102
|
+
margin-bottom: 10px;
|
|
103
|
+
border-radius: 0 4px 4px 0;
|
|
104
|
+
overflow-x: auto;
|
|
105
|
+
}
|
|
106
|
+
pre {
|
|
107
|
+
background-color: #f5f5f5;
|
|
108
|
+
padding: 10px;
|
|
109
|
+
border-radius: 4px;
|
|
110
|
+
overflow-x: auto;
|
|
111
|
+
}
|
|
112
|
+
code {
|
|
113
|
+
font-family: SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace;
|
|
114
|
+
font-size: 0.9em;
|
|
115
|
+
}
|
|
116
|
+
.badge {
|
|
117
|
+
display: inline-block;
|
|
118
|
+
padding: 3px 7px;
|
|
119
|
+
font-size: 0.75em;
|
|
120
|
+
font-weight: 600;
|
|
121
|
+
line-height: 1;
|
|
122
|
+
text-align: center;
|
|
123
|
+
white-space: nowrap;
|
|
124
|
+
vertical-align: baseline;
|
|
125
|
+
border-radius: 10px;
|
|
126
|
+
background-color: #e9ecef;
|
|
127
|
+
margin-right: 5px;
|
|
128
|
+
}
|
|
129
|
+
.badge-warning {
|
|
130
|
+
background-color: #fff3cd;
|
|
131
|
+
color: #856404;
|
|
132
|
+
}
|
|
133
|
+
.badge-primary {
|
|
134
|
+
background-color: #cfe2ff;
|
|
135
|
+
color: #084298;
|
|
136
|
+
}
|
|
137
|
+
.badge-success {
|
|
138
|
+
background-color: #d1e7dd;
|
|
139
|
+
color: #0f5132;
|
|
140
|
+
}
|
|
141
|
+
.fix-method {
|
|
142
|
+
background-color: #e8f5e9;
|
|
143
|
+
border-left: 4px solid #4caf50;
|
|
144
|
+
padding: 10px 15px;
|
|
145
|
+
margin: 10px 0;
|
|
146
|
+
border-radius: 0 4px 4px 0;
|
|
147
|
+
}
|
|
148
|
+
</style>
|
|
149
|
+
</head>
|
|
150
|
+
<body>
|
|
151
|
+
<div class="header">
|
|
152
|
+
<h1>EDSL Validation Failures Report</h1>
|
|
153
|
+
<span class="timestamp">Generated on {{timestamp}}</span>
|
|
154
|
+
</div>
|
|
155
|
+
|
|
156
|
+
<div class="summary">
|
|
157
|
+
<h2>Summary</h2>
|
|
158
|
+
<p>This report analyzes validation failures that occurred when question answers didn't meet the expected format or constraints.
|
|
159
|
+
It provides statistics, improvement suggestions for fix methods, and examples of common failures.</p>
|
|
160
|
+
<p><strong>Total validation failures:</strong> {{total_failures}}</p>
|
|
161
|
+
<p><strong>Question types with failures:</strong> {{question_types_count}}</p>
|
|
162
|
+
</div>
|
|
163
|
+
|
|
164
|
+
<div class="stats-container">
|
|
165
|
+
<h2>Validation Failure Statistics</h2>
|
|
166
|
+
|
|
167
|
+
<div class="card">
|
|
168
|
+
<div class="card-header">Failures by Question Type</div>
|
|
169
|
+
<table>
|
|
170
|
+
<thead>
|
|
171
|
+
<tr>
|
|
172
|
+
<th>Question Type</th>
|
|
173
|
+
<th>Failure Count</th>
|
|
174
|
+
<th>Percentage</th>
|
|
175
|
+
</tr>
|
|
176
|
+
</thead>
|
|
177
|
+
<tbody>
|
|
178
|
+
{{type_stats_rows}}
|
|
179
|
+
</tbody>
|
|
180
|
+
</table>
|
|
181
|
+
</div>
|
|
182
|
+
|
|
183
|
+
<div class="card">
|
|
184
|
+
<div class="card-header">Top Error Messages</div>
|
|
185
|
+
<table>
|
|
186
|
+
<thead>
|
|
187
|
+
<tr>
|
|
188
|
+
<th>Error Message</th>
|
|
189
|
+
<th>Occurrence Count</th>
|
|
190
|
+
</tr>
|
|
191
|
+
</thead>
|
|
192
|
+
<tbody>
|
|
193
|
+
{{error_stats_rows}}
|
|
194
|
+
</tbody>
|
|
195
|
+
</table>
|
|
196
|
+
</div>
|
|
197
|
+
</div>
|
|
198
|
+
|
|
199
|
+
<div class="suggestions-container">
|
|
200
|
+
<h2>Fix Method Improvement Suggestions</h2>
|
|
201
|
+
{{suggestions_content}}
|
|
202
|
+
</div>
|
|
203
|
+
|
|
204
|
+
<div class="examples-container">
|
|
205
|
+
<h2>Example Validation Failures</h2>
|
|
206
|
+
{{examples_content}}
|
|
207
|
+
</div>
|
|
208
|
+
</body>
|
|
209
|
+
</html>
|
|
210
|
+
"""
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _generate_type_stats_rows(stats: Dict) -> str:
|
|
214
|
+
"""Generate HTML table rows for question type statistics."""
|
|
215
|
+
type_stats = stats.get("by_question_type", {})
|
|
216
|
+
total_failures = sum(type_stats.values())
|
|
217
|
+
|
|
218
|
+
rows = []
|
|
219
|
+
for question_type, count in sorted(type_stats.items(), key=lambda x: x[1], reverse=True):
|
|
220
|
+
percentage = (count / total_failures) * 100 if total_failures > 0 else 0
|
|
221
|
+
row = (
|
|
222
|
+
f"<tr>"
|
|
223
|
+
f"<td>{question_type}</td>"
|
|
224
|
+
f"<td>{count}</td>"
|
|
225
|
+
f"<td>{percentage:.1f}%</td>"
|
|
226
|
+
f"</tr>"
|
|
227
|
+
)
|
|
228
|
+
rows.append(row)
|
|
229
|
+
|
|
230
|
+
return "\n".join(rows)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _generate_error_stats_rows(stats: Dict) -> str:
|
|
234
|
+
"""Generate HTML table rows for error message statistics."""
|
|
235
|
+
error_counts = {}
|
|
236
|
+
|
|
237
|
+
# Aggregate error counts across all question types
|
|
238
|
+
for question_type, errors in stats.get("by_error_message", {}).items():
|
|
239
|
+
for error_msg, count in errors.items():
|
|
240
|
+
error_counts[error_msg] = error_counts.get(error_msg, 0) + count
|
|
241
|
+
|
|
242
|
+
# Sort by count (descending)
|
|
243
|
+
sorted_errors = sorted(error_counts.items(), key=lambda x: x[1], reverse=True)
|
|
244
|
+
|
|
245
|
+
rows = []
|
|
246
|
+
for error_msg, count in sorted_errors[:10]: # Show top 10 errors
|
|
247
|
+
shortened_msg = error_msg[:100] + "..." if len(error_msg) > 100 else error_msg
|
|
248
|
+
row = (
|
|
249
|
+
f"<tr>"
|
|
250
|
+
f"<td>{shortened_msg}</td>"
|
|
251
|
+
f"<td>{count}</td>"
|
|
252
|
+
f"</tr>"
|
|
253
|
+
)
|
|
254
|
+
rows.append(row)
|
|
255
|
+
|
|
256
|
+
return "\n".join(rows)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _generate_suggestions_content(suggestions: Dict) -> str:
|
|
260
|
+
"""Generate HTML content for fix method suggestions."""
|
|
261
|
+
if not suggestions:
|
|
262
|
+
return "<p>No suggestions available. Log more validation failures to generate improvement suggestions.</p>"
|
|
263
|
+
|
|
264
|
+
content = []
|
|
265
|
+
|
|
266
|
+
for question_type, question_suggestions in suggestions.items():
|
|
267
|
+
content.append(f"<div class='card'>")
|
|
268
|
+
content.append(f"<div class='card-header'>{question_type}</div>")
|
|
269
|
+
|
|
270
|
+
for suggestion in question_suggestions:
|
|
271
|
+
error_msg = suggestion.get("error_message", "")
|
|
272
|
+
occurrence_count = suggestion.get("occurrence_count", 0)
|
|
273
|
+
suggestion_text = suggestion.get("suggestion", "")
|
|
274
|
+
|
|
275
|
+
content.append(
|
|
276
|
+
f"<div class='suggestion'>"
|
|
277
|
+
f"<p><strong>Error:</strong> {error_msg}</p>"
|
|
278
|
+
f"<p><strong>Occurrences:</strong> {occurrence_count}</p>"
|
|
279
|
+
f"<div class='fix-method'>"
|
|
280
|
+
f"<p><strong>Suggested improvement:</strong></p>"
|
|
281
|
+
f"<p>{suggestion_text}</p>"
|
|
282
|
+
f"</div>"
|
|
283
|
+
f"</div>"
|
|
284
|
+
)
|
|
285
|
+
|
|
286
|
+
content.append("</div>")
|
|
287
|
+
|
|
288
|
+
return "\n".join(content)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _generate_examples_content(logs: List[Dict]) -> str:
|
|
292
|
+
"""Generate HTML content for example validation failures."""
|
|
293
|
+
if not logs:
|
|
294
|
+
return "<p>No validation failure examples available.</p>"
|
|
295
|
+
|
|
296
|
+
content = []
|
|
297
|
+
|
|
298
|
+
# Group logs by question type
|
|
299
|
+
logs_by_type = {}
|
|
300
|
+
for log in logs:
|
|
301
|
+
question_type = log.get("question_type", "unknown")
|
|
302
|
+
if question_type not in logs_by_type:
|
|
303
|
+
logs_by_type[question_type] = []
|
|
304
|
+
logs_by_type[question_type].append(log)
|
|
305
|
+
|
|
306
|
+
# For each question type, show the most recent example
|
|
307
|
+
for question_type, type_logs in logs_by_type.items():
|
|
308
|
+
# Sort by timestamp (newest first)
|
|
309
|
+
sorted_logs = sorted(type_logs, key=lambda x: x.get("timestamp", ""), reverse=True)
|
|
310
|
+
example_log = sorted_logs[0]
|
|
311
|
+
|
|
312
|
+
error_message = example_log.get("error_message", "")
|
|
313
|
+
invalid_data = example_log.get("invalid_data", {})
|
|
314
|
+
model_schema = example_log.get("model_schema", {})
|
|
315
|
+
|
|
316
|
+
content.append(f"<div class='card'>")
|
|
317
|
+
content.append(f"<div class='card-header'>{question_type}</div>")
|
|
318
|
+
|
|
319
|
+
content.append(
|
|
320
|
+
f"<div class='example'>"
|
|
321
|
+
f"<p><strong>Error:</strong> {error_message}</p>"
|
|
322
|
+
f"<p><strong>Invalid Data:</strong></p>"
|
|
323
|
+
f"<pre><code>{json.dumps(invalid_data, indent=2)}</code></pre>"
|
|
324
|
+
f"<p><strong>Expected Schema:</strong></p>"
|
|
325
|
+
f"<pre><code>{json.dumps(model_schema, indent=2)}</code></pre>"
|
|
326
|
+
f"</div>"
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
content.append("</div>")
|
|
330
|
+
|
|
331
|
+
return "\n".join(content)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def generate_html_report(output_path: Optional[Path] = None) -> Path:
|
|
335
|
+
"""
|
|
336
|
+
Generate an HTML report of validation failures.
|
|
337
|
+
|
|
338
|
+
Args:
|
|
339
|
+
output_path: Optional custom path for the report
|
|
340
|
+
|
|
341
|
+
Returns:
|
|
342
|
+
Path to the generated HTML report
|
|
343
|
+
"""
|
|
344
|
+
# Determine output path
|
|
345
|
+
if output_path is None:
|
|
346
|
+
default_log_dir = Path.home() / ".edsl" / "logs"
|
|
347
|
+
try:
|
|
348
|
+
report_dir = Path(CONFIG.get("EDSL_LOG_DIR"))
|
|
349
|
+
except Exception:
|
|
350
|
+
# If EDSL_LOG_DIR is not defined, use default
|
|
351
|
+
report_dir = default_log_dir
|
|
352
|
+
os.makedirs(report_dir, exist_ok=True)
|
|
353
|
+
output_path = report_dir / "validation_report.html"
|
|
354
|
+
|
|
355
|
+
# Get validation data
|
|
356
|
+
logs = get_validation_failure_logs(n=100) # Get up to 100 recent logs
|
|
357
|
+
stats = get_validation_failure_stats()
|
|
358
|
+
suggestions = suggest_fix_improvements()
|
|
359
|
+
|
|
360
|
+
# Calculate summary statistics
|
|
361
|
+
total_failures = sum(stats.get("by_question_type", {}).values())
|
|
362
|
+
question_types_count = len(stats.get("by_question_type", {}))
|
|
363
|
+
|
|
364
|
+
# Generate report content
|
|
365
|
+
type_stats_rows = _generate_type_stats_rows(stats)
|
|
366
|
+
error_stats_rows = _generate_error_stats_rows(stats)
|
|
367
|
+
suggestions_content = _generate_suggestions_content(suggestions)
|
|
368
|
+
examples_content = _generate_examples_content(logs)
|
|
369
|
+
|
|
370
|
+
# Format timestamp
|
|
371
|
+
timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
|
372
|
+
|
|
373
|
+
# Fill the template
|
|
374
|
+
html_content = HTML_TEMPLATE.replace("{{timestamp}}", timestamp)
|
|
375
|
+
html_content = html_content.replace("{{total_failures}}", str(total_failures))
|
|
376
|
+
html_content = html_content.replace("{{question_types_count}}", str(question_types_count))
|
|
377
|
+
html_content = html_content.replace("{{type_stats_rows}}", type_stats_rows)
|
|
378
|
+
html_content = html_content.replace("{{error_stats_rows}}", error_stats_rows)
|
|
379
|
+
html_content = html_content.replace("{{suggestions_content}}", suggestions_content)
|
|
380
|
+
html_content = html_content.replace("{{examples_content}}", examples_content)
|
|
381
|
+
|
|
382
|
+
# Write the report
|
|
383
|
+
with open(output_path, "w") as f:
|
|
384
|
+
f.write(html_content)
|
|
385
|
+
|
|
386
|
+
return output_path
|
|
387
|
+
|
|
388
|
+
|
|
389
|
+
def generate_and_open_report() -> None:
|
|
390
|
+
"""Generate a validation report and open it in the default browser."""
|
|
391
|
+
report_path = generate_html_report()
|
|
392
|
+
print(f"Report generated at: {report_path}")
|
|
393
|
+
|
|
394
|
+
# Try to open the report in a browser
|
|
395
|
+
try:
|
|
396
|
+
import webbrowser
|
|
397
|
+
webbrowser.open(f"file://{report_path}")
|
|
398
|
+
except Exception as e:
|
|
399
|
+
print(f"Could not open browser: {e}")
|
|
400
|
+
print(f"Report is available at: {report_path}")
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
if __name__ == "__main__":
|
|
404
|
+
generate_and_open_report()
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
"""Logger for validation failures in questions.
|
|
2
|
+
|
|
3
|
+
This module provides functionality to log validation failures that occur when
|
|
4
|
+
question answers don't meet the expected format or constraints. The logs can be
|
|
5
|
+
used to improve the "fix" methods for questions.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import datetime
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import traceback
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any, Dict, Optional
|
|
15
|
+
|
|
16
|
+
from ..config import CONFIG
|
|
17
|
+
|
|
18
|
+
# Set up logging
|
|
19
|
+
logger = logging.getLogger("validation_failures")
|
|
20
|
+
logger.setLevel(logging.INFO)
|
|
21
|
+
|
|
22
|
+
# Determine log directory path
|
|
23
|
+
DEFAULT_LOG_DIR = Path.home() / ".edsl" / "logs"
|
|
24
|
+
try:
|
|
25
|
+
LOG_DIR = Path(CONFIG.get("EDSL_LOG_DIR"))
|
|
26
|
+
except Exception:
|
|
27
|
+
# If EDSL_LOG_DIR is not defined, use default
|
|
28
|
+
LOG_DIR = DEFAULT_LOG_DIR
|
|
29
|
+
VALIDATION_LOG_FILE = LOG_DIR / "validation_failures.log"
|
|
30
|
+
|
|
31
|
+
# Create log directory if it doesn't exist
|
|
32
|
+
os.makedirs(LOG_DIR, exist_ok=True)
|
|
33
|
+
|
|
34
|
+
# Create file handler
|
|
35
|
+
file_handler = logging.FileHandler(VALIDATION_LOG_FILE)
|
|
36
|
+
file_handler.setLevel(logging.INFO)
|
|
37
|
+
|
|
38
|
+
# Create formatter
|
|
39
|
+
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
|
40
|
+
file_handler.setFormatter(formatter)
|
|
41
|
+
|
|
42
|
+
# Add handler to logger
|
|
43
|
+
logger.addHandler(file_handler)
|
|
44
|
+
|
|
45
|
+
# Touch the log file to make sure it exists
|
|
46
|
+
if not os.path.exists(VALIDATION_LOG_FILE):
|
|
47
|
+
with open(VALIDATION_LOG_FILE, 'a'):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def log_validation_failure(
|
|
52
|
+
question_type: str,
|
|
53
|
+
question_name: str,
|
|
54
|
+
error_message: str,
|
|
55
|
+
invalid_data: Dict[str, Any],
|
|
56
|
+
model_schema: Dict[str, Any],
|
|
57
|
+
question_dict: Optional[Dict[str, Any]] = None,
|
|
58
|
+
) -> None:
|
|
59
|
+
"""
|
|
60
|
+
Log a validation failure to the validation failures log file.
|
|
61
|
+
|
|
62
|
+
Args:
|
|
63
|
+
question_type: The type of question that had a validation failure
|
|
64
|
+
question_name: The name of the question
|
|
65
|
+
error_message: The validation error message
|
|
66
|
+
invalid_data: The data that failed validation
|
|
67
|
+
model_schema: The schema of the model used for validation
|
|
68
|
+
question_dict: Optional dictionary representation of the question
|
|
69
|
+
"""
|
|
70
|
+
log_entry = {
|
|
71
|
+
"timestamp": datetime.datetime.now().isoformat(),
|
|
72
|
+
"question_type": question_type,
|
|
73
|
+
"question_name": question_name,
|
|
74
|
+
"error_message": error_message,
|
|
75
|
+
"invalid_data": invalid_data,
|
|
76
|
+
"model_schema": model_schema,
|
|
77
|
+
"question_dict": question_dict,
|
|
78
|
+
"traceback": traceback.format_exc(),
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
# Log as JSON for easier parsing
|
|
82
|
+
logger.info(json.dumps(log_entry))
|
|
83
|
+
|
|
84
|
+
# Write directly to the file as well to ensure it's written
|
|
85
|
+
with open(VALIDATION_LOG_FILE, "a") as f:
|
|
86
|
+
f.write(f"{datetime.datetime.now().isoformat()} - validation_failures - INFO - {json.dumps(log_entry)}\n")
|
|
87
|
+
f.flush()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def get_validation_failure_logs(n: int = 10) -> list:
|
|
91
|
+
"""
|
|
92
|
+
Get the latest n validation failure logs.
|
|
93
|
+
|
|
94
|
+
Args:
|
|
95
|
+
n: Number of logs to return (default: 10)
|
|
96
|
+
|
|
97
|
+
Returns:
|
|
98
|
+
List of validation failure log entries as dictionaries
|
|
99
|
+
"""
|
|
100
|
+
logs = []
|
|
101
|
+
|
|
102
|
+
# Check if log file exists
|
|
103
|
+
if not os.path.exists(VALIDATION_LOG_FILE):
|
|
104
|
+
return logs
|
|
105
|
+
|
|
106
|
+
with open(VALIDATION_LOG_FILE, "r") as f:
|
|
107
|
+
for line in f:
|
|
108
|
+
try:
|
|
109
|
+
# Skip non-JSON lines (like logger initialization)
|
|
110
|
+
if not line.strip():
|
|
111
|
+
continue
|
|
112
|
+
|
|
113
|
+
# Handle both the Python logging format and our direct write format
|
|
114
|
+
parts = line.strip().split(" - ")
|
|
115
|
+
if len(parts) >= 4:
|
|
116
|
+
# Regular log line format: timestamp - name - level - message
|
|
117
|
+
json_part = parts[-1]
|
|
118
|
+
try:
|
|
119
|
+
log_entry = json.loads(json_part)
|
|
120
|
+
logs.append(log_entry)
|
|
121
|
+
except json.JSONDecodeError:
|
|
122
|
+
# Skip malformed JSON
|
|
123
|
+
continue
|
|
124
|
+
except (IndexError, ValueError):
|
|
125
|
+
# Skip malformed lines
|
|
126
|
+
continue
|
|
127
|
+
|
|
128
|
+
# Return most recent logs first
|
|
129
|
+
return sorted(logs, key=lambda x: x.get("timestamp", ""), reverse=True)[:n]
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def clear_validation_logs() -> None:
|
|
133
|
+
"""Clear all validation failure logs."""
|
|
134
|
+
if os.path.exists(VALIDATION_LOG_FILE):
|
|
135
|
+
with open(VALIDATION_LOG_FILE, "w") as f:
|
|
136
|
+
f.write("")
|