edsl 0.1.50__py3-none-any.whl → 0.1.52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__init__.py +45 -34
- edsl/__version__.py +1 -1
- edsl/base/base_exception.py +2 -2
- edsl/buckets/bucket_collection.py +1 -1
- edsl/buckets/exceptions.py +32 -0
- edsl/buckets/token_bucket_api.py +26 -10
- edsl/caching/cache.py +5 -2
- edsl/caching/remote_cache_sync.py +5 -5
- edsl/caching/sql_dict.py +12 -11
- edsl/config/__init__.py +1 -1
- edsl/config/config_class.py +4 -2
- edsl/conversation/Conversation.py +9 -5
- edsl/conversation/car_buying.py +1 -3
- edsl/conversation/mug_negotiation.py +2 -6
- edsl/coop/__init__.py +11 -8
- edsl/coop/coop.py +15 -13
- edsl/coop/coop_functions.py +1 -1
- edsl/coop/ep_key_handling.py +1 -1
- edsl/coop/price_fetcher.py +2 -2
- edsl/coop/utils.py +2 -2
- edsl/dataset/dataset.py +144 -63
- edsl/dataset/dataset_operations_mixin.py +14 -6
- edsl/dataset/dataset_tree.py +3 -3
- edsl/dataset/display/table_renderers.py +6 -3
- edsl/dataset/file_exports.py +4 -4
- edsl/dataset/r/ggplot.py +3 -3
- edsl/inference_services/available_model_fetcher.py +2 -2
- edsl/inference_services/data_structures.py +5 -5
- edsl/inference_services/inference_service_abc.py +1 -1
- edsl/inference_services/inference_services_collection.py +1 -1
- edsl/inference_services/service_availability.py +3 -3
- edsl/inference_services/services/azure_ai.py +3 -3
- edsl/inference_services/services/google_service.py +1 -1
- edsl/inference_services/services/test_service.py +1 -1
- edsl/instructions/change_instruction.py +5 -4
- edsl/instructions/instruction.py +1 -0
- edsl/instructions/instruction_collection.py +5 -4
- edsl/instructions/instruction_handler.py +10 -8
- edsl/interviews/answering_function.py +20 -21
- edsl/interviews/exception_tracking.py +3 -2
- edsl/interviews/interview.py +1 -1
- edsl/interviews/interview_status_dictionary.py +1 -1
- edsl/interviews/interview_task_manager.py +7 -4
- edsl/interviews/request_token_estimator.py +3 -2
- edsl/interviews/statistics.py +2 -2
- edsl/invigilators/invigilators.py +34 -6
- edsl/jobs/__init__.py +39 -2
- edsl/jobs/async_interview_runner.py +1 -1
- edsl/jobs/check_survey_scenario_compatibility.py +5 -5
- edsl/jobs/data_structures.py +2 -2
- edsl/jobs/html_table_job_logger.py +494 -257
- edsl/jobs/jobs.py +2 -2
- edsl/jobs/jobs_checks.py +5 -5
- edsl/jobs/jobs_component_constructor.py +2 -2
- edsl/jobs/jobs_pricing_estimation.py +1 -1
- edsl/jobs/jobs_runner_asyncio.py +2 -2
- edsl/jobs/jobs_status_enums.py +1 -0
- edsl/jobs/remote_inference.py +47 -13
- edsl/jobs/results_exceptions_handler.py +2 -2
- edsl/language_models/language_model.py +151 -145
- edsl/notebooks/__init__.py +24 -1
- edsl/notebooks/exceptions.py +82 -0
- edsl/notebooks/notebook.py +7 -3
- edsl/notebooks/notebook_to_latex.py +1 -1
- edsl/prompts/__init__.py +23 -2
- edsl/prompts/prompt.py +1 -1
- edsl/questions/__init__.py +4 -4
- edsl/questions/answer_validator_mixin.py +0 -5
- edsl/questions/compose_questions.py +2 -2
- edsl/questions/descriptors.py +1 -1
- edsl/questions/question_base.py +32 -3
- edsl/questions/question_base_prompts_mixin.py +4 -4
- edsl/questions/question_budget.py +503 -102
- edsl/questions/question_check_box.py +658 -156
- edsl/questions/question_dict.py +176 -2
- edsl/questions/question_extract.py +401 -61
- edsl/questions/question_free_text.py +77 -9
- edsl/questions/question_functional.py +118 -9
- edsl/questions/{derived/question_likert_five.py → question_likert_five.py} +2 -2
- edsl/questions/{derived/question_linear_scale.py → question_linear_scale.py} +3 -4
- edsl/questions/question_list.py +246 -26
- edsl/questions/question_matrix.py +586 -73
- edsl/questions/question_multiple_choice.py +213 -47
- edsl/questions/question_numerical.py +360 -29
- edsl/questions/question_rank.py +401 -124
- edsl/questions/question_registry.py +3 -3
- edsl/questions/{derived/question_top_k.py → question_top_k.py} +3 -3
- edsl/questions/{derived/question_yes_no.py → question_yes_no.py} +3 -4
- edsl/questions/register_questions_meta.py +2 -1
- edsl/questions/response_validator_abc.py +6 -2
- edsl/questions/response_validator_factory.py +10 -12
- edsl/results/report.py +1 -1
- edsl/results/result.py +7 -4
- edsl/results/results.py +500 -271
- edsl/results/results_selector.py +2 -2
- edsl/scenarios/construct_download_link.py +3 -3
- edsl/scenarios/scenario.py +1 -2
- edsl/scenarios/scenario_list.py +41 -23
- edsl/surveys/survey_css.py +3 -3
- edsl/surveys/survey_simulator.py +2 -1
- edsl/tasks/__init__.py +22 -2
- edsl/tasks/exceptions.py +72 -0
- edsl/tasks/task_history.py +48 -11
- edsl/templates/error_reporting/base.html +37 -4
- edsl/templates/error_reporting/exceptions_table.html +105 -33
- edsl/templates/error_reporting/interview_details.html +130 -126
- edsl/templates/error_reporting/overview.html +21 -25
- edsl/templates/error_reporting/report.css +215 -46
- edsl/templates/error_reporting/report.js +122 -20
- edsl/tokens/__init__.py +27 -1
- edsl/tokens/exceptions.py +37 -0
- edsl/tokens/interview_token_usage.py +3 -2
- edsl/tokens/token_usage.py +4 -3
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/METADATA +1 -1
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/RECORD +118 -116
- edsl/questions/derived/__init__.py +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/LICENSE +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/WHEEL +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/entry_points.txt +0 -0
@@ -1,8 +1,7 @@
|
|
1
1
|
"""
|
2
2
|
question_matrix.py
|
3
3
|
|
4
|
-
|
5
|
-
that automatically raises ValidationError for invalid matrix answers.
|
4
|
+
Module implementing the matrix question type with Pydantic validation
|
6
5
|
"""
|
7
6
|
|
8
7
|
from __future__ import annotations
|
@@ -13,12 +12,13 @@ from typing import (
|
|
13
12
|
List,
|
14
13
|
Any,
|
15
14
|
Type,
|
16
|
-
get_args,
|
17
15
|
Literal
|
18
16
|
)
|
19
17
|
import random
|
18
|
+
import json
|
19
|
+
import re
|
20
20
|
|
21
|
-
from pydantic import BaseModel, Field, create_model, ValidationError
|
21
|
+
from pydantic import BaseModel, Field, create_model, ValidationError, model_validator
|
22
22
|
from jinja2 import Template
|
23
23
|
|
24
24
|
from .question_base import QuestionBase
|
@@ -32,40 +32,102 @@ from .decorators import inject_exception
|
|
32
32
|
|
33
33
|
from .exceptions import (
|
34
34
|
QuestionCreationValidationError,
|
35
|
-
QuestionAnswerValidationError,
|
35
|
+
QuestionAnswerValidationError,
|
36
36
|
)
|
37
37
|
|
38
38
|
|
39
|
+
class MatrixResponseBase(BaseModel):
|
40
|
+
"""
|
41
|
+
Base model for matrix question responses.
|
42
|
+
|
43
|
+
Attributes:
|
44
|
+
answer: A dictionary mapping each item to a selected option
|
45
|
+
comment: Optional comment about the selections
|
46
|
+
generated_tokens: Optional token usage data
|
47
|
+
|
48
|
+
Examples:
|
49
|
+
>>> # Valid response with two items
|
50
|
+
>>> model = MatrixResponseBase(answer={"Item1": 1, "Item2": 2})
|
51
|
+
>>> model.answer
|
52
|
+
{'Item1': 1, 'Item2': 2}
|
53
|
+
|
54
|
+
>>> # Valid response with a comment
|
55
|
+
>>> model = MatrixResponseBase(
|
56
|
+
... answer={"Item1": "Yes", "Item2": "No"},
|
57
|
+
... comment="This is my reasoning"
|
58
|
+
... )
|
59
|
+
>>> model.comment
|
60
|
+
'This is my reasoning'
|
61
|
+
"""
|
62
|
+
answer: Dict[str, Any]
|
63
|
+
comment: Optional[str] = None
|
64
|
+
generated_tokens: Optional[Any] = None
|
65
|
+
|
66
|
+
|
39
67
|
def create_matrix_response(
|
40
68
|
question_items: List[str],
|
41
69
|
question_options: List[Union[int, str, float]],
|
42
70
|
permissive: bool = False,
|
43
71
|
) -> Type[BaseModel]:
|
44
72
|
"""
|
45
|
-
Create a dynamic Pydantic model for matrix questions.
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
73
|
+
Create a dynamic Pydantic model for matrix questions with appropriate validation.
|
74
|
+
|
75
|
+
Args:
|
76
|
+
question_items: List of items that need responses
|
77
|
+
question_options: List of allowed options for each item
|
78
|
+
permissive: If True, allows any values and additional items
|
79
|
+
|
80
|
+
Returns:
|
81
|
+
A Pydantic model class for validating matrix responses
|
82
|
+
|
83
|
+
Examples:
|
84
|
+
>>> # Create a model for a 2x3 matrix
|
85
|
+
>>> Model = create_matrix_response(
|
86
|
+
... ["Item1", "Item2"],
|
87
|
+
... [1, 2, 3]
|
88
|
+
... )
|
89
|
+
>>> # Valid response
|
90
|
+
>>> response = Model(answer={"Item1": 1, "Item2": 2})
|
91
|
+
>>> isinstance(response.answer, BaseModel)
|
92
|
+
True
|
93
|
+
>>> response.answer.Item1
|
94
|
+
1
|
95
|
+
>>> response.answer.Item2
|
96
|
+
2
|
97
|
+
|
98
|
+
>>> # Invalid: missing an item
|
99
|
+
>>> try:
|
100
|
+
... Model(answer={"Item1": 1})
|
101
|
+
... except Exception:
|
102
|
+
... print("Validation error occurred")
|
103
|
+
Validation error occurred
|
104
|
+
|
105
|
+
>>> # Invalid: invalid option value
|
106
|
+
>>> try:
|
107
|
+
... Model(answer={"Item1": 4, "Item2": 2})
|
108
|
+
... except Exception:
|
109
|
+
... print("Validation error occurred")
|
110
|
+
Validation error occurred
|
50
111
|
"""
|
51
|
-
|
112
|
+
# Convert question_options to a tuple for Literal type
|
113
|
+
option_tuple = tuple(question_options)
|
114
|
+
|
52
115
|
# If non-permissive, build a Literal for each valid option
|
53
116
|
# e.g. Literal[1,2,3] or Literal["Yes","No"] or a mix
|
54
117
|
if not permissive:
|
55
118
|
# If question_options is empty (edge case), fall back to 'Any'
|
56
119
|
if question_options:
|
57
|
-
AllowedOptions = Literal[
|
120
|
+
AllowedOptions = Literal[option_tuple] # type: ignore
|
58
121
|
else:
|
59
122
|
AllowedOptions = Any
|
60
123
|
else:
|
61
124
|
# Permissive => let each item be anything
|
62
125
|
AllowedOptions = Any
|
63
126
|
|
64
|
-
# Build field definitions for
|
65
|
-
# question_item is a required field with type AllowedOptions
|
127
|
+
# Build field definitions for the answer submodel
|
66
128
|
field_definitions = {}
|
67
129
|
for item in question_items:
|
68
|
-
field_definitions[item] = (AllowedOptions, Field(...)) # required
|
130
|
+
field_definitions[item] = (AllowedOptions, Field(...)) # required field
|
69
131
|
|
70
132
|
# Dynamically create the submodel
|
71
133
|
MatrixAnswerSubModel = create_model(
|
@@ -74,22 +136,96 @@ def create_matrix_response(
|
|
74
136
|
**field_definitions
|
75
137
|
)
|
76
138
|
|
77
|
-
#
|
78
|
-
class MatrixResponse(
|
79
|
-
|
80
|
-
|
81
|
-
|
139
|
+
# Create the full response model with custom validation
|
140
|
+
class MatrixResponse(MatrixResponseBase):
|
141
|
+
"""
|
142
|
+
Model for matrix question responses with validation for specific items and options.
|
143
|
+
"""
|
144
|
+
answer: MatrixAnswerSubModel # Use the dynamically created submodel
|
145
|
+
|
146
|
+
@model_validator(mode='after')
|
147
|
+
def validate_matrix_constraints(self):
|
148
|
+
"""
|
149
|
+
Validates that:
|
150
|
+
1. All required items have responses
|
151
|
+
2. All responses are valid options
|
152
|
+
3. No unexpected items are included (unless permissive)
|
153
|
+
"""
|
154
|
+
matrix_answer = self.answer.model_dump()
|
155
|
+
|
156
|
+
# Check that all required items have responses
|
157
|
+
missing_items = [item for item in question_items if item not in matrix_answer]
|
158
|
+
if missing_items and not permissive:
|
159
|
+
missing_str = ", ".join(missing_items)
|
160
|
+
validation_error = ValidationError.from_exception_data(
|
161
|
+
title='MatrixResponse',
|
162
|
+
line_errors=[{
|
163
|
+
'type': 'value_error',
|
164
|
+
'loc': ('answer',),
|
165
|
+
'msg': f'Missing responses for items: {missing_str}',
|
166
|
+
'input': matrix_answer,
|
167
|
+
'ctx': {'missing_items': missing_items}
|
168
|
+
}]
|
169
|
+
)
|
170
|
+
raise QuestionAnswerValidationError(
|
171
|
+
message=f"Missing responses for items: {missing_str}",
|
172
|
+
data=self.model_dump(),
|
173
|
+
model=self.__class__,
|
174
|
+
pydantic_error=validation_error
|
175
|
+
)
|
176
|
+
|
177
|
+
# Check that all responses are valid options
|
178
|
+
if not permissive:
|
179
|
+
invalid_items = {}
|
180
|
+
for item, value in matrix_answer.items():
|
181
|
+
if value not in option_tuple:
|
182
|
+
invalid_items[item] = value
|
183
|
+
|
184
|
+
if invalid_items:
|
185
|
+
items_str = ", ".join(f"{k}: {v}" for k, v in invalid_items.items())
|
186
|
+
validation_error = ValidationError.from_exception_data(
|
187
|
+
title='MatrixResponse',
|
188
|
+
line_errors=[{
|
189
|
+
'type': 'value_error',
|
190
|
+
'loc': ('answer',),
|
191
|
+
'msg': f'Invalid options selected: {items_str}',
|
192
|
+
'input': matrix_answer,
|
193
|
+
'ctx': {'invalid_items': invalid_items, 'allowed_options': option_tuple}
|
194
|
+
}]
|
195
|
+
)
|
196
|
+
raise QuestionAnswerValidationError(
|
197
|
+
message=f"Invalid options selected: {items_str}. Allowed options: {option_tuple}",
|
198
|
+
data=self.model_dump(),
|
199
|
+
model=self.__class__,
|
200
|
+
pydantic_error=validation_error
|
201
|
+
)
|
202
|
+
|
203
|
+
return self
|
82
204
|
|
83
205
|
class Config:
|
84
|
-
# If permissive=
|
85
|
-
# If permissive=True, allow them.
|
206
|
+
# If permissive=True, allow extra fields in the answer dict
|
86
207
|
extra = "allow" if permissive else "forbid"
|
208
|
+
|
209
|
+
@staticmethod
|
210
|
+
def json_schema_extra(schema: dict, model: BaseModel) -> None:
|
211
|
+
# Add the options to the schema for better documentation
|
212
|
+
if "properties" in schema and "answer" in schema["properties"]:
|
213
|
+
schema["properties"]["answer"]["description"] = "Matrix responses for each item"
|
214
|
+
if "properties" in schema["properties"]["answer"]:
|
215
|
+
for _, prop in schema["properties"]["answer"]["properties"].items():
|
216
|
+
prop["enum"] = list(question_options)
|
87
217
|
|
88
218
|
return MatrixResponse
|
89
219
|
|
90
220
|
|
91
221
|
class MatrixResponseValidator(ResponseValidatorABC):
|
92
|
-
"""
|
222
|
+
"""
|
223
|
+
Validator for matrix question responses that attempts to fix invalid responses.
|
224
|
+
|
225
|
+
This validator tries multiple approaches to recover valid matrix responses from
|
226
|
+
malformed inputs, including JSON parsing, remapping numeric keys, and extracting
|
227
|
+
structured data from text.
|
228
|
+
"""
|
93
229
|
required_params = ["question_items", "question_options", "permissive"]
|
94
230
|
|
95
231
|
valid_examples = [
|
@@ -98,8 +234,17 @@ class MatrixResponseValidator(ResponseValidatorABC):
|
|
98
234
|
{
|
99
235
|
"question_items": ["Item1", "Item2"],
|
100
236
|
"question_options": [1, 2, 3],
|
237
|
+
"permissive": False
|
101
238
|
},
|
102
|
-
)
|
239
|
+
),
|
240
|
+
(
|
241
|
+
{"answer": {"Item1": "Yes", "Item2": "No"}},
|
242
|
+
{
|
243
|
+
"question_items": ["Item1", "Item2"],
|
244
|
+
"question_options": ["Yes", "No", "Maybe"],
|
245
|
+
"permissive": False
|
246
|
+
},
|
247
|
+
),
|
103
248
|
]
|
104
249
|
|
105
250
|
invalid_examples = [
|
@@ -108,14 +253,16 @@ class MatrixResponseValidator(ResponseValidatorABC):
|
|
108
253
|
{
|
109
254
|
"question_items": ["Item1", "Item2"],
|
110
255
|
"question_options": [1, 2, 3],
|
256
|
+
"permissive": False
|
111
257
|
},
|
112
|
-
"Missing responses for
|
258
|
+
"Missing responses for items",
|
113
259
|
),
|
114
260
|
(
|
115
261
|
{"answer": {"Item1": 4, "Item2": 5}},
|
116
262
|
{
|
117
263
|
"question_items": ["Item1", "Item2"],
|
118
264
|
"question_options": [1, 2, 3],
|
265
|
+
"permissive": False
|
119
266
|
},
|
120
267
|
"Invalid options selected",
|
121
268
|
),
|
@@ -123,39 +270,373 @@ class MatrixResponseValidator(ResponseValidatorABC):
|
|
123
270
|
|
124
271
|
def fix(self, response, verbose=False):
|
125
272
|
"""
|
126
|
-
|
127
|
-
|
273
|
+
Attempts to fix an invalid matrix response by trying multiple parsing strategies.
|
274
|
+
|
275
|
+
Args:
|
276
|
+
response: The invalid response to fix
|
277
|
+
verbose: Whether to print verbose debugging information
|
278
|
+
|
279
|
+
Returns:
|
280
|
+
A fixed response dict if fixable, otherwise the original response
|
128
281
|
"""
|
129
282
|
if verbose:
|
130
283
|
print(f"Fixing matrix response: {response}")
|
131
|
-
|
132
|
-
# If
|
133
|
-
if "
|
284
|
+
|
285
|
+
# If response doesn't have an answer field, nothing to do
|
286
|
+
if "answer" not in response:
|
287
|
+
if verbose:
|
288
|
+
print("Response has no answer field, cannot fix")
|
289
|
+
return response
|
290
|
+
|
291
|
+
# Strategy 1: If we have generated_tokens, try to parse them as JSON
|
292
|
+
if "generated_tokens" in response and response["generated_tokens"]:
|
134
293
|
try:
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
294
|
+
# Try to parse generated_tokens as JSON
|
295
|
+
tokens_text = str(response["generated_tokens"])
|
296
|
+
json_match = re.search(r'\{.*\}', tokens_text, re.DOTALL)
|
297
|
+
|
298
|
+
if json_match:
|
299
|
+
json_str = json_match.group(0)
|
300
|
+
fixed = json.loads(json_str)
|
301
|
+
|
302
|
+
if isinstance(fixed, dict):
|
303
|
+
# Map numeric keys to question items if needed
|
304
|
+
if all(str(k).isdigit() for k in fixed.keys()):
|
305
|
+
if verbose:
|
306
|
+
print(f"JSON extraction found numeric keys: {fixed}")
|
307
|
+
print(f"Question items: {self.question_items}")
|
308
|
+
print(f"Question options: {self.question_options}")
|
309
|
+
|
310
|
+
# Special handling for case when numeric keys directly represent option indices
|
311
|
+
# This is the case we're trying to fix: {"0": 1, "1": 3, "2": 0} maps to options at those indices
|
312
|
+
direct_mapped_answer = {}
|
313
|
+
if verbose:
|
314
|
+
print(f"Attempting to map numeric key/value format in JSON: {fixed}")
|
315
|
+
|
316
|
+
for idx, item in enumerate(self.question_items):
|
317
|
+
if str(idx) in fixed:
|
318
|
+
# Get the option index directly from the value
|
319
|
+
option_idx = fixed[str(idx)]
|
320
|
+
|
321
|
+
# Convert to int if needed
|
322
|
+
if isinstance(option_idx, str) and option_idx.isdigit():
|
323
|
+
option_idx = int(option_idx)
|
324
|
+
|
325
|
+
if verbose:
|
326
|
+
print(f"Item {item} at index {idx} maps to value {option_idx}")
|
327
|
+
|
328
|
+
if isinstance(option_idx, (int, float)) and 0 <= option_idx < len(self.question_options):
|
329
|
+
direct_mapped_answer[item] = self.question_options[option_idx]
|
330
|
+
if verbose:
|
331
|
+
print(f"Mapped option_idx {option_idx} to {self.question_options[option_idx]}")
|
332
|
+
|
333
|
+
if direct_mapped_answer and len(direct_mapped_answer) == len(self.question_items):
|
334
|
+
proposed_data = {
|
335
|
+
"answer": direct_mapped_answer,
|
336
|
+
"comment": response.get("comment"),
|
337
|
+
"generated_tokens": response.get("generated_tokens")
|
338
|
+
}
|
339
|
+
if verbose:
|
340
|
+
print(f"Created direct option mapping from JSON: {proposed_data}")
|
341
|
+
try:
|
342
|
+
self.response_model(**proposed_data)
|
343
|
+
if verbose:
|
344
|
+
print(f"Successfully fixed with direct option mapping from JSON: {proposed_data}")
|
345
|
+
return proposed_data
|
346
|
+
except Exception as e:
|
347
|
+
if verbose:
|
348
|
+
print(f"Direct option mapping from JSON failed validation: {e}")
|
349
|
+
|
350
|
+
# Try the standard approach as well
|
351
|
+
mapped_answer = {}
|
352
|
+
for idx, item in enumerate(self.question_items):
|
353
|
+
if str(idx) in fixed:
|
354
|
+
# Get the value (column index) from the response
|
355
|
+
value_idx = fixed[str(idx)]
|
356
|
+
|
357
|
+
# Convert to int if it's a digit string
|
358
|
+
if isinstance(value_idx, str) and value_idx.isdigit():
|
359
|
+
value_idx = int(value_idx)
|
360
|
+
|
361
|
+
# Convert column index to actual option value
|
362
|
+
if isinstance(value_idx, (int, float)) and 0 <= value_idx < len(self.question_options):
|
363
|
+
option_value = self.question_options[value_idx]
|
364
|
+
mapped_answer[item] = option_value
|
365
|
+
else:
|
366
|
+
# If the value is already a valid option, use it directly
|
367
|
+
if value_idx in self.question_options:
|
368
|
+
mapped_answer[item] = value_idx
|
369
|
+
else:
|
370
|
+
# Last resort - try to use it as a direct value even if not in options
|
371
|
+
mapped_answer[item] = value_idx
|
372
|
+
|
373
|
+
if mapped_answer and (len(mapped_answer) == len(self.question_items) or self.permissive):
|
374
|
+
proposed_data = {
|
375
|
+
"answer": mapped_answer,
|
376
|
+
"comment": response.get("comment"),
|
377
|
+
"generated_tokens": response.get("generated_tokens")
|
378
|
+
}
|
379
|
+
try:
|
380
|
+
# Validate the fixed response
|
381
|
+
self.response_model(**proposed_data)
|
382
|
+
if verbose:
|
383
|
+
print(f"Successfully fixed by parsing JSON: {proposed_data}")
|
384
|
+
return proposed_data
|
385
|
+
except Exception as e:
|
386
|
+
if verbose:
|
387
|
+
print(f"Fixed response failed validation: {e}")
|
388
|
+
|
389
|
+
# Try again with string values for all options
|
390
|
+
text_mapped_answer = {}
|
391
|
+
for item_name, option_value in mapped_answer.items():
|
392
|
+
text_mapped_answer[item_name] = str(option_value)
|
393
|
+
|
394
|
+
proposed_data = {
|
395
|
+
"answer": text_mapped_answer,
|
396
|
+
"comment": response.get("comment"),
|
397
|
+
"generated_tokens": response.get("generated_tokens")
|
398
|
+
}
|
399
|
+
try:
|
400
|
+
self.response_model(**proposed_data)
|
401
|
+
if verbose:
|
402
|
+
print(f"Successfully fixed with string conversion from JSON: {proposed_data}")
|
403
|
+
return proposed_data
|
404
|
+
except Exception as e:
|
405
|
+
if verbose:
|
406
|
+
print(f"String conversion from JSON failed validation: {e}")
|
407
|
+
else:
|
408
|
+
# The JSON already has string keys, use directly
|
409
|
+
proposed_data = {
|
410
|
+
"answer": fixed,
|
411
|
+
"comment": response.get("comment"),
|
412
|
+
"generated_tokens": response.get("generated_tokens")
|
413
|
+
}
|
414
|
+
try:
|
415
|
+
self.response_model(**proposed_data)
|
416
|
+
if verbose:
|
417
|
+
print(f"Successfully fixed by direct JSON: {proposed_data}")
|
418
|
+
return proposed_data
|
419
|
+
except Exception as e:
|
420
|
+
if verbose:
|
421
|
+
print(f"Fixed response failed validation: {e}")
|
422
|
+
|
423
|
+
# If validation failed, check if we need to map string keys to item names
|
424
|
+
# This handles cases where the model responded with something like {"Row 0": 1, "Row 1": 2}
|
425
|
+
# instead of using the exact item names
|
426
|
+
item_map = {}
|
427
|
+
for item in self.question_items:
|
428
|
+
# Create various forms of the item name that might appear in responses
|
429
|
+
item_variants = [
|
430
|
+
item.lower(),
|
431
|
+
item.upper(),
|
432
|
+
item.strip(),
|
433
|
+
f"Row {self.question_items.index(item)}",
|
434
|
+
f"Item {self.question_items.index(item)}",
|
435
|
+
f"{self.question_items.index(item)}"
|
436
|
+
]
|
437
|
+
for key in fixed.keys():
|
438
|
+
if isinstance(key, str):
|
439
|
+
key_lower = key.lower().strip()
|
440
|
+
if key_lower in item_variants or item.lower() in key_lower:
|
441
|
+
item_map[key] = item
|
442
|
+
|
443
|
+
if item_map:
|
444
|
+
mapped_answer = {}
|
445
|
+
for key, value in fixed.items():
|
446
|
+
if key in item_map:
|
447
|
+
# Handle both numeric indices and direct values
|
448
|
+
if isinstance(value, (int, float)) and 0 <= value < len(self.question_options):
|
449
|
+
mapped_answer[item_map[key]] = self.question_options[value]
|
450
|
+
else:
|
451
|
+
mapped_answer[item_map[key]] = value
|
452
|
+
|
453
|
+
if mapped_answer:
|
454
|
+
proposed_data = {
|
455
|
+
"answer": mapped_answer,
|
456
|
+
"comment": response.get("comment"),
|
457
|
+
"generated_tokens": response.get("generated_tokens")
|
458
|
+
}
|
459
|
+
try:
|
460
|
+
self.response_model(**proposed_data)
|
461
|
+
if verbose:
|
462
|
+
print(f"Successfully fixed by mapping item names: {proposed_data}")
|
463
|
+
return proposed_data
|
464
|
+
except Exception as e:
|
465
|
+
if verbose:
|
466
|
+
print(f"Item-mapped response failed validation: {e}")
|
467
|
+
except (ValueError, KeyError, TypeError, json.JSONDecodeError) as e:
|
468
|
+
if verbose:
|
469
|
+
print(f"JSON parsing failed: {e}")
|
470
|
+
# Continue to other strategies
|
471
|
+
|
472
|
+
# Strategy 2: If answer uses numeric keys, map them to question items
|
473
|
+
if isinstance(response.get("answer"), dict):
|
474
|
+
answer_dict = response["answer"]
|
475
|
+
|
476
|
+
if all(str(k).isdigit() for k in answer_dict.keys()):
|
477
|
+
if verbose:
|
478
|
+
print(f"Processing answer with numeric keys: {answer_dict}")
|
479
|
+
print(f"Question items: {self.question_items}")
|
480
|
+
print(f"Question options: {self.question_options}")
|
481
|
+
|
152
482
|
mapped_answer = {}
|
153
483
|
for idx, item in enumerate(self.question_items):
|
154
|
-
if str(idx) in
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
484
|
+
if str(idx) in answer_dict:
|
485
|
+
# Get the value (column index) from the response
|
486
|
+
value_idx = answer_dict[str(idx)]
|
487
|
+
|
488
|
+
# Convert to int if it's a digit string
|
489
|
+
if isinstance(value_idx, str) and value_idx.isdigit():
|
490
|
+
value_idx = int(value_idx)
|
491
|
+
|
492
|
+
if verbose:
|
493
|
+
print(f"Processing item {item} at index {idx}, value_idx={value_idx}")
|
494
|
+
|
495
|
+
# Convert column index to actual option value
|
496
|
+
if isinstance(value_idx, (int, float)) and 0 <= value_idx < len(self.question_options):
|
497
|
+
option_value = self.question_options[value_idx]
|
498
|
+
mapped_answer[item] = option_value
|
499
|
+
if verbose:
|
500
|
+
print(f"Mapped column index {value_idx} to option '{option_value}'")
|
501
|
+
else:
|
502
|
+
# If the value is already a valid option, use it directly
|
503
|
+
if value_idx in self.question_options:
|
504
|
+
mapped_answer[item] = value_idx
|
505
|
+
if verbose:
|
506
|
+
print(f"Used direct option value '{value_idx}'")
|
507
|
+
else:
|
508
|
+
# Last resort - try to use it as a direct value even if not in options
|
509
|
+
# (this helps with permissive mode)
|
510
|
+
mapped_answer[item] = value_idx
|
511
|
+
if verbose:
|
512
|
+
print(f"Used non-option value '{value_idx}' as direct value")
|
513
|
+
|
514
|
+
if mapped_answer and len(mapped_answer) == len(self.question_items):
|
515
|
+
if verbose:
|
516
|
+
print(f"Created complete mapped answer: {mapped_answer}")
|
517
|
+
|
518
|
+
proposed_data = {
|
519
|
+
"answer": mapped_answer,
|
520
|
+
"comment": response.get("comment"),
|
521
|
+
"generated_tokens": response.get("generated_tokens")
|
522
|
+
}
|
523
|
+
try:
|
524
|
+
self.response_model(**proposed_data)
|
525
|
+
if verbose:
|
526
|
+
print(f"Successfully fixed by mapping numeric keys: {proposed_data}")
|
527
|
+
return proposed_data
|
528
|
+
except Exception as e:
|
529
|
+
if verbose:
|
530
|
+
print(f"Fixed response failed validation: {e}")
|
531
|
+
|
532
|
+
# Try again with string values for the options
|
533
|
+
text_mapped_answer = {}
|
534
|
+
for item_name, option_value in mapped_answer.items():
|
535
|
+
text_mapped_answer[item_name] = str(option_value)
|
536
|
+
|
537
|
+
proposed_data = {
|
538
|
+
"answer": text_mapped_answer,
|
539
|
+
"comment": response.get("comment"),
|
540
|
+
"generated_tokens": response.get("generated_tokens")
|
541
|
+
}
|
542
|
+
try:
|
543
|
+
self.response_model(**proposed_data)
|
544
|
+
if verbose:
|
545
|
+
print(f"Successfully fixed with string conversion: {proposed_data}")
|
546
|
+
return proposed_data
|
547
|
+
except Exception as e:
|
548
|
+
if verbose:
|
549
|
+
print(f"String conversion failed validation: {e}")
|
550
|
+
|
551
|
+
# Special handling for case when numeric keys directly represent option indices
|
552
|
+
# This is the case we're trying to fix: {"0": 1, "1": 3, "2": 0} maps to options at those indices
|
553
|
+
direct_mapped_answer = {}
|
554
|
+
if verbose:
|
555
|
+
print(f"Attempting to map numeric key/value format in answer: {answer_dict}")
|
556
|
+
|
557
|
+
for idx, item in enumerate(self.question_items):
|
558
|
+
if str(idx) in answer_dict:
|
559
|
+
# Get the option index directly from the value
|
560
|
+
option_idx = answer_dict[str(idx)]
|
561
|
+
|
562
|
+
# Convert to int if needed
|
563
|
+
if isinstance(option_idx, str) and option_idx.isdigit():
|
564
|
+
option_idx = int(option_idx)
|
565
|
+
|
566
|
+
if verbose:
|
567
|
+
print(f"Item {item} at index {idx} maps to value {option_idx}")
|
568
|
+
|
569
|
+
if isinstance(option_idx, (int, float)) and 0 <= option_idx < len(self.question_options):
|
570
|
+
direct_mapped_answer[item] = self.question_options[option_idx]
|
571
|
+
if verbose:
|
572
|
+
print(f"Mapped option_idx {option_idx} to {self.question_options[option_idx]}")
|
573
|
+
|
574
|
+
if direct_mapped_answer and len(direct_mapped_answer) == len(self.question_items):
|
575
|
+
proposed_data = {
|
576
|
+
"answer": direct_mapped_answer,
|
577
|
+
"comment": response.get("comment"),
|
578
|
+
"generated_tokens": response.get("generated_tokens")
|
579
|
+
}
|
580
|
+
if verbose:
|
581
|
+
print(f"Created direct option mapping: {proposed_data}")
|
582
|
+
try:
|
583
|
+
self.response_model(**proposed_data)
|
584
|
+
if verbose:
|
585
|
+
print(f"Successfully fixed with direct option mapping: {proposed_data}")
|
586
|
+
return proposed_data
|
587
|
+
except Exception as e:
|
588
|
+
if verbose:
|
589
|
+
print(f"Direct option mapping failed validation: {e}")
|
590
|
+
|
591
|
+
# Strategy 3: If answer is a string, try to extract a structured response
|
592
|
+
if isinstance(response.get("answer"), str):
|
593
|
+
answer_text = response["answer"]
|
594
|
+
|
595
|
+
# Try to extract item-option pairs using regex
|
596
|
+
pairs = re.findall(r'([^:,]+):\s*([^,]+)', answer_text)
|
597
|
+
if pairs:
|
598
|
+
extracted = {}
|
599
|
+
for item, option in pairs:
|
600
|
+
item = item.strip()
|
601
|
+
option = option.strip()
|
602
|
+
|
603
|
+
# Match the item name with the closest question item
|
604
|
+
best_match = None
|
605
|
+
for q_item in self.question_items:
|
606
|
+
if q_item.lower() in item.lower():
|
607
|
+
best_match = q_item
|
608
|
+
break
|
609
|
+
|
610
|
+
if best_match:
|
611
|
+
# Try to match the option with question options
|
612
|
+
matched_option = None
|
613
|
+
for q_option in self.question_options:
|
614
|
+
q_option_str = str(q_option)
|
615
|
+
if q_option_str == option or q_option_str in option:
|
616
|
+
matched_option = q_option
|
617
|
+
break
|
618
|
+
|
619
|
+
if matched_option is not None:
|
620
|
+
extracted[best_match] = matched_option
|
621
|
+
|
622
|
+
if extracted and (len(extracted) == len(self.question_items) or self.permissive):
|
623
|
+
proposed_data = {
|
624
|
+
"answer": extracted,
|
625
|
+
"comment": response.get("comment"),
|
626
|
+
"generated_tokens": response.get("generated_tokens")
|
627
|
+
}
|
628
|
+
try:
|
629
|
+
self.response_model(**proposed_data)
|
630
|
+
if verbose:
|
631
|
+
print(f"Successfully fixed by extracting pairs: {proposed_data}")
|
632
|
+
return proposed_data
|
633
|
+
except Exception as e:
|
634
|
+
if verbose:
|
635
|
+
print(f"Fixed response failed validation: {e}")
|
636
|
+
|
637
|
+
# If we got here, we couldn't fix the response
|
638
|
+
if verbose:
|
639
|
+
print("Could not fix matrix response, returning original")
|
159
640
|
return response
|
160
641
|
|
161
642
|
|
@@ -163,10 +644,23 @@ class QuestionMatrix(QuestionBase):
|
|
163
644
|
"""
|
164
645
|
A question that presents a matrix/grid where multiple items are rated
|
165
646
|
or selected from the same set of options.
|
166
|
-
|
167
|
-
This
|
168
|
-
|
169
|
-
|
647
|
+
|
648
|
+
This question type allows respondents to provide an answer for each row
|
649
|
+
in a grid, selecting from the same set of options for each row. It's often
|
650
|
+
used for Likert scales, ratings grids, or any scenario where multiple items
|
651
|
+
need to be rated using the same scale.
|
652
|
+
|
653
|
+
Examples:
|
654
|
+
>>> # Create a happiness rating matrix
|
655
|
+
>>> question = QuestionMatrix(
|
656
|
+
... question_name="happiness_matrix",
|
657
|
+
... question_text="Rate your happiness with each aspect:",
|
658
|
+
... question_items=["Work", "Family", "Social life"],
|
659
|
+
... question_options=[1, 2, 3, 4, 5],
|
660
|
+
... option_labels={1: "Very unhappy", 3: "Neutral", 5: "Very happy"}
|
661
|
+
... )
|
662
|
+
>>> # The response is a dict matching each item to a rating
|
663
|
+
>>> response = {"answer": {"Work": 4, "Family": 5, "Social life": 3}}
|
170
664
|
"""
|
171
665
|
|
172
666
|
question_type = "matrix"
|
@@ -196,12 +690,12 @@ class QuestionMatrix(QuestionBase):
|
|
196
690
|
Args:
|
197
691
|
question_name: The name of the question
|
198
692
|
question_text: The text of the question
|
199
|
-
question_items: List of items to be rated or answered
|
200
|
-
question_options: Possible answer options
|
693
|
+
question_items: List of items to be rated or answered (rows)
|
694
|
+
question_options: Possible answer options for each item (columns)
|
201
695
|
option_labels: Optional mapping of options to labels (e.g. {1: "Sad", 5: "Happy"})
|
202
696
|
include_comment: Whether to include a comment field
|
203
|
-
answering_instructions: Custom instructions
|
204
|
-
question_presentation: Custom presentation
|
697
|
+
answering_instructions: Custom instructions template
|
698
|
+
question_presentation: Custom presentation template
|
205
699
|
permissive: Whether to allow any values & extra items instead of strictly checking
|
206
700
|
"""
|
207
701
|
self.question_name = question_name
|
@@ -224,7 +718,10 @@ class QuestionMatrix(QuestionBase):
|
|
224
718
|
|
225
719
|
def create_response_model(self) -> Type[BaseModel]:
|
226
720
|
"""
|
227
|
-
Returns the pydantic model
|
721
|
+
Returns the pydantic model for validating responses to this question.
|
722
|
+
|
723
|
+
The model is dynamically created based on the question's configuration,
|
724
|
+
including allowed items, options, and permissiveness.
|
228
725
|
"""
|
229
726
|
return create_matrix_response(
|
230
727
|
self.question_items,
|
@@ -232,9 +729,29 @@ class QuestionMatrix(QuestionBase):
|
|
232
729
|
self.permissive
|
233
730
|
)
|
234
731
|
|
732
|
+
def _simulate_answer(self) -> dict:
|
733
|
+
"""
|
734
|
+
Simulate a random valid answer for testing purposes.
|
735
|
+
|
736
|
+
Returns:
|
737
|
+
A valid simulated response with random selections
|
738
|
+
"""
|
739
|
+
return {
|
740
|
+
"answer": {
|
741
|
+
item: random.choice(self.question_options)
|
742
|
+
for item in self.question_items
|
743
|
+
},
|
744
|
+
"comment": "Sample matrix response"
|
745
|
+
}
|
746
|
+
|
235
747
|
@property
|
236
748
|
def question_html_content(self) -> str:
|
237
|
-
"""
|
749
|
+
"""
|
750
|
+
Generate an HTML representation of the matrix question.
|
751
|
+
|
752
|
+
Returns:
|
753
|
+
HTML content string for rendering the question
|
754
|
+
"""
|
238
755
|
template = Template(
|
239
756
|
"""
|
240
757
|
<table class="matrix-question">
|
@@ -276,7 +793,12 @@ class QuestionMatrix(QuestionBase):
|
|
276
793
|
@classmethod
|
277
794
|
@inject_exception
|
278
795
|
def example(cls) -> QuestionMatrix:
|
279
|
-
"""
|
796
|
+
"""
|
797
|
+
Return an example matrix question.
|
798
|
+
|
799
|
+
Returns:
|
800
|
+
An example QuestionMatrix instance for happiness ratings by family size
|
801
|
+
"""
|
280
802
|
return cls(
|
281
803
|
question_name="child_happiness",
|
282
804
|
question_text="How happy would you be with different numbers of children?",
|
@@ -288,13 +810,4 @@ class QuestionMatrix(QuestionBase):
|
|
288
810
|
],
|
289
811
|
question_options=[1, 2, 3, 4, 5],
|
290
812
|
option_labels={1: "Very sad", 3: "Neutral", 5: "Extremely happy"},
|
291
|
-
)
|
292
|
-
|
293
|
-
def _simulate_answer(self) -> dict:
|
294
|
-
"""Simulate a random valid answer."""
|
295
|
-
return {
|
296
|
-
"answer": {
|
297
|
-
item: random.choice(self.question_options)
|
298
|
-
for item in self.question_items
|
299
|
-
}
|
300
|
-
}
|
813
|
+
)
|