edsl 0.1.50__py3-none-any.whl → 0.1.52__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. edsl/__init__.py +45 -34
  2. edsl/__version__.py +1 -1
  3. edsl/base/base_exception.py +2 -2
  4. edsl/buckets/bucket_collection.py +1 -1
  5. edsl/buckets/exceptions.py +32 -0
  6. edsl/buckets/token_bucket_api.py +26 -10
  7. edsl/caching/cache.py +5 -2
  8. edsl/caching/remote_cache_sync.py +5 -5
  9. edsl/caching/sql_dict.py +12 -11
  10. edsl/config/__init__.py +1 -1
  11. edsl/config/config_class.py +4 -2
  12. edsl/conversation/Conversation.py +9 -5
  13. edsl/conversation/car_buying.py +1 -3
  14. edsl/conversation/mug_negotiation.py +2 -6
  15. edsl/coop/__init__.py +11 -8
  16. edsl/coop/coop.py +15 -13
  17. edsl/coop/coop_functions.py +1 -1
  18. edsl/coop/ep_key_handling.py +1 -1
  19. edsl/coop/price_fetcher.py +2 -2
  20. edsl/coop/utils.py +2 -2
  21. edsl/dataset/dataset.py +144 -63
  22. edsl/dataset/dataset_operations_mixin.py +14 -6
  23. edsl/dataset/dataset_tree.py +3 -3
  24. edsl/dataset/display/table_renderers.py +6 -3
  25. edsl/dataset/file_exports.py +4 -4
  26. edsl/dataset/r/ggplot.py +3 -3
  27. edsl/inference_services/available_model_fetcher.py +2 -2
  28. edsl/inference_services/data_structures.py +5 -5
  29. edsl/inference_services/inference_service_abc.py +1 -1
  30. edsl/inference_services/inference_services_collection.py +1 -1
  31. edsl/inference_services/service_availability.py +3 -3
  32. edsl/inference_services/services/azure_ai.py +3 -3
  33. edsl/inference_services/services/google_service.py +1 -1
  34. edsl/inference_services/services/test_service.py +1 -1
  35. edsl/instructions/change_instruction.py +5 -4
  36. edsl/instructions/instruction.py +1 -0
  37. edsl/instructions/instruction_collection.py +5 -4
  38. edsl/instructions/instruction_handler.py +10 -8
  39. edsl/interviews/answering_function.py +20 -21
  40. edsl/interviews/exception_tracking.py +3 -2
  41. edsl/interviews/interview.py +1 -1
  42. edsl/interviews/interview_status_dictionary.py +1 -1
  43. edsl/interviews/interview_task_manager.py +7 -4
  44. edsl/interviews/request_token_estimator.py +3 -2
  45. edsl/interviews/statistics.py +2 -2
  46. edsl/invigilators/invigilators.py +34 -6
  47. edsl/jobs/__init__.py +39 -2
  48. edsl/jobs/async_interview_runner.py +1 -1
  49. edsl/jobs/check_survey_scenario_compatibility.py +5 -5
  50. edsl/jobs/data_structures.py +2 -2
  51. edsl/jobs/html_table_job_logger.py +494 -257
  52. edsl/jobs/jobs.py +2 -2
  53. edsl/jobs/jobs_checks.py +5 -5
  54. edsl/jobs/jobs_component_constructor.py +2 -2
  55. edsl/jobs/jobs_pricing_estimation.py +1 -1
  56. edsl/jobs/jobs_runner_asyncio.py +2 -2
  57. edsl/jobs/jobs_status_enums.py +1 -0
  58. edsl/jobs/remote_inference.py +47 -13
  59. edsl/jobs/results_exceptions_handler.py +2 -2
  60. edsl/language_models/language_model.py +151 -145
  61. edsl/notebooks/__init__.py +24 -1
  62. edsl/notebooks/exceptions.py +82 -0
  63. edsl/notebooks/notebook.py +7 -3
  64. edsl/notebooks/notebook_to_latex.py +1 -1
  65. edsl/prompts/__init__.py +23 -2
  66. edsl/prompts/prompt.py +1 -1
  67. edsl/questions/__init__.py +4 -4
  68. edsl/questions/answer_validator_mixin.py +0 -5
  69. edsl/questions/compose_questions.py +2 -2
  70. edsl/questions/descriptors.py +1 -1
  71. edsl/questions/question_base.py +32 -3
  72. edsl/questions/question_base_prompts_mixin.py +4 -4
  73. edsl/questions/question_budget.py +503 -102
  74. edsl/questions/question_check_box.py +658 -156
  75. edsl/questions/question_dict.py +176 -2
  76. edsl/questions/question_extract.py +401 -61
  77. edsl/questions/question_free_text.py +77 -9
  78. edsl/questions/question_functional.py +118 -9
  79. edsl/questions/{derived/question_likert_five.py → question_likert_five.py} +2 -2
  80. edsl/questions/{derived/question_linear_scale.py → question_linear_scale.py} +3 -4
  81. edsl/questions/question_list.py +246 -26
  82. edsl/questions/question_matrix.py +586 -73
  83. edsl/questions/question_multiple_choice.py +213 -47
  84. edsl/questions/question_numerical.py +360 -29
  85. edsl/questions/question_rank.py +401 -124
  86. edsl/questions/question_registry.py +3 -3
  87. edsl/questions/{derived/question_top_k.py → question_top_k.py} +3 -3
  88. edsl/questions/{derived/question_yes_no.py → question_yes_no.py} +3 -4
  89. edsl/questions/register_questions_meta.py +2 -1
  90. edsl/questions/response_validator_abc.py +6 -2
  91. edsl/questions/response_validator_factory.py +10 -12
  92. edsl/results/report.py +1 -1
  93. edsl/results/result.py +7 -4
  94. edsl/results/results.py +500 -271
  95. edsl/results/results_selector.py +2 -2
  96. edsl/scenarios/construct_download_link.py +3 -3
  97. edsl/scenarios/scenario.py +1 -2
  98. edsl/scenarios/scenario_list.py +41 -23
  99. edsl/surveys/survey_css.py +3 -3
  100. edsl/surveys/survey_simulator.py +2 -1
  101. edsl/tasks/__init__.py +22 -2
  102. edsl/tasks/exceptions.py +72 -0
  103. edsl/tasks/task_history.py +48 -11
  104. edsl/templates/error_reporting/base.html +37 -4
  105. edsl/templates/error_reporting/exceptions_table.html +105 -33
  106. edsl/templates/error_reporting/interview_details.html +130 -126
  107. edsl/templates/error_reporting/overview.html +21 -25
  108. edsl/templates/error_reporting/report.css +215 -46
  109. edsl/templates/error_reporting/report.js +122 -20
  110. edsl/tokens/__init__.py +27 -1
  111. edsl/tokens/exceptions.py +37 -0
  112. edsl/tokens/interview_token_usage.py +3 -2
  113. edsl/tokens/token_usage.py +4 -3
  114. {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/METADATA +1 -1
  115. {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/RECORD +118 -116
  116. edsl/questions/derived/__init__.py +0 -0
  117. {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/LICENSE +0 -0
  118. {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/WHEEL +0 -0
  119. {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/entry_points.txt +0 -0
@@ -1,8 +1,7 @@
1
1
  """
2
2
  question_matrix.py
3
3
 
4
- Drop-in replacement for `QuestionMatrix` with a dynamic Pydantic approach
5
- that automatically raises ValidationError for invalid matrix answers.
4
+ Module implementing the matrix question type with Pydantic validation
6
5
  """
7
6
 
8
7
  from __future__ import annotations
@@ -13,12 +12,13 @@ from typing import (
13
12
  List,
14
13
  Any,
15
14
  Type,
16
- get_args,
17
15
  Literal
18
16
  )
19
17
  import random
18
+ import json
19
+ import re
20
20
 
21
- from pydantic import BaseModel, Field, create_model, ValidationError
21
+ from pydantic import BaseModel, Field, create_model, ValidationError, model_validator
22
22
  from jinja2 import Template
23
23
 
24
24
  from .question_base import QuestionBase
@@ -32,40 +32,102 @@ from .decorators import inject_exception
32
32
 
33
33
  from .exceptions import (
34
34
  QuestionCreationValidationError,
35
- QuestionAnswerValidationError, # If you still want to raise custom exceptions
35
+ QuestionAnswerValidationError,
36
36
  )
37
37
 
38
38
 
39
+ class MatrixResponseBase(BaseModel):
40
+ """
41
+ Base model for matrix question responses.
42
+
43
+ Attributes:
44
+ answer: A dictionary mapping each item to a selected option
45
+ comment: Optional comment about the selections
46
+ generated_tokens: Optional token usage data
47
+
48
+ Examples:
49
+ >>> # Valid response with two items
50
+ >>> model = MatrixResponseBase(answer={"Item1": 1, "Item2": 2})
51
+ >>> model.answer
52
+ {'Item1': 1, 'Item2': 2}
53
+
54
+ >>> # Valid response with a comment
55
+ >>> model = MatrixResponseBase(
56
+ ... answer={"Item1": "Yes", "Item2": "No"},
57
+ ... comment="This is my reasoning"
58
+ ... )
59
+ >>> model.comment
60
+ 'This is my reasoning'
61
+ """
62
+ answer: Dict[str, Any]
63
+ comment: Optional[str] = None
64
+ generated_tokens: Optional[Any] = None
65
+
66
+
39
67
  def create_matrix_response(
40
68
  question_items: List[str],
41
69
  question_options: List[Union[int, str, float]],
42
70
  permissive: bool = False,
43
71
  ) -> Type[BaseModel]:
44
72
  """
45
- Create a dynamic Pydantic model for matrix questions.
46
-
47
- If `permissive=False`, each item is a required field with a `Literal[...]` type
48
- so that only the given question_options are allowed.
49
- If `permissive=True`, each item can have any value, and extra items are allowed.
73
+ Create a dynamic Pydantic model for matrix questions with appropriate validation.
74
+
75
+ Args:
76
+ question_items: List of items that need responses
77
+ question_options: List of allowed options for each item
78
+ permissive: If True, allows any values and additional items
79
+
80
+ Returns:
81
+ A Pydantic model class for validating matrix responses
82
+
83
+ Examples:
84
+ >>> # Create a model for a 2x3 matrix
85
+ >>> Model = create_matrix_response(
86
+ ... ["Item1", "Item2"],
87
+ ... [1, 2, 3]
88
+ ... )
89
+ >>> # Valid response
90
+ >>> response = Model(answer={"Item1": 1, "Item2": 2})
91
+ >>> isinstance(response.answer, BaseModel)
92
+ True
93
+ >>> response.answer.Item1
94
+ 1
95
+ >>> response.answer.Item2
96
+ 2
97
+
98
+ >>> # Invalid: missing an item
99
+ >>> try:
100
+ ... Model(answer={"Item1": 1})
101
+ ... except Exception:
102
+ ... print("Validation error occurred")
103
+ Validation error occurred
104
+
105
+ >>> # Invalid: invalid option value
106
+ >>> try:
107
+ ... Model(answer={"Item1": 4, "Item2": 2})
108
+ ... except Exception:
109
+ ... print("Validation error occurred")
110
+ Validation error occurred
50
111
  """
51
-
112
+ # Convert question_options to a tuple for Literal type
113
+ option_tuple = tuple(question_options)
114
+
52
115
  # If non-permissive, build a Literal for each valid option
53
116
  # e.g. Literal[1,2,3] or Literal["Yes","No"] or a mix
54
117
  if not permissive:
55
118
  # If question_options is empty (edge case), fall back to 'Any'
56
119
  if question_options:
57
- AllowedOptions = Literal[tuple(question_options)] # type: ignore
120
+ AllowedOptions = Literal[option_tuple] # type: ignore
58
121
  else:
59
122
  AllowedOptions = Any
60
123
  else:
61
124
  # Permissive => let each item be anything
62
125
  AllowedOptions = Any
63
126
 
64
- # Build field definitions for an "AnswerSubModel", where each
65
- # question_item is a required field with type AllowedOptions
127
+ # Build field definitions for the answer submodel
66
128
  field_definitions = {}
67
129
  for item in question_items:
68
- field_definitions[item] = (AllowedOptions, Field(...)) # required
130
+ field_definitions[item] = (AllowedOptions, Field(...)) # required field
69
131
 
70
132
  # Dynamically create the submodel
71
133
  MatrixAnswerSubModel = create_model(
@@ -74,22 +136,96 @@ def create_matrix_response(
74
136
  **field_definitions
75
137
  )
76
138
 
77
- # Build the top-level model with `answer` + optional `comment`
78
- class MatrixResponse(BaseModel):
79
- answer: MatrixAnswerSubModel
80
- comment: Optional[str] = None
81
- generated_tokens: Optional[Any] = None
139
+ # Create the full response model with custom validation
140
+ class MatrixResponse(MatrixResponseBase):
141
+ """
142
+ Model for matrix question responses with validation for specific items and options.
143
+ """
144
+ answer: MatrixAnswerSubModel # Use the dynamically created submodel
145
+
146
+ @model_validator(mode='after')
147
+ def validate_matrix_constraints(self):
148
+ """
149
+ Validates that:
150
+ 1. All required items have responses
151
+ 2. All responses are valid options
152
+ 3. No unexpected items are included (unless permissive)
153
+ """
154
+ matrix_answer = self.answer.model_dump()
155
+
156
+ # Check that all required items have responses
157
+ missing_items = [item for item in question_items if item not in matrix_answer]
158
+ if missing_items and not permissive:
159
+ missing_str = ", ".join(missing_items)
160
+ validation_error = ValidationError.from_exception_data(
161
+ title='MatrixResponse',
162
+ line_errors=[{
163
+ 'type': 'value_error',
164
+ 'loc': ('answer',),
165
+ 'msg': f'Missing responses for items: {missing_str}',
166
+ 'input': matrix_answer,
167
+ 'ctx': {'missing_items': missing_items}
168
+ }]
169
+ )
170
+ raise QuestionAnswerValidationError(
171
+ message=f"Missing responses for items: {missing_str}",
172
+ data=self.model_dump(),
173
+ model=self.__class__,
174
+ pydantic_error=validation_error
175
+ )
176
+
177
+ # Check that all responses are valid options
178
+ if not permissive:
179
+ invalid_items = {}
180
+ for item, value in matrix_answer.items():
181
+ if value not in option_tuple:
182
+ invalid_items[item] = value
183
+
184
+ if invalid_items:
185
+ items_str = ", ".join(f"{k}: {v}" for k, v in invalid_items.items())
186
+ validation_error = ValidationError.from_exception_data(
187
+ title='MatrixResponse',
188
+ line_errors=[{
189
+ 'type': 'value_error',
190
+ 'loc': ('answer',),
191
+ 'msg': f'Invalid options selected: {items_str}',
192
+ 'input': matrix_answer,
193
+ 'ctx': {'invalid_items': invalid_items, 'allowed_options': option_tuple}
194
+ }]
195
+ )
196
+ raise QuestionAnswerValidationError(
197
+ message=f"Invalid options selected: {items_str}. Allowed options: {option_tuple}",
198
+ data=self.model_dump(),
199
+ model=self.__class__,
200
+ pydantic_error=validation_error
201
+ )
202
+
203
+ return self
82
204
 
83
205
  class Config:
84
- # If permissive=False, forbid extra items in `answer`.
85
- # If permissive=True, allow them.
206
+ # If permissive=True, allow extra fields in the answer dict
86
207
  extra = "allow" if permissive else "forbid"
208
+
209
+ @staticmethod
210
+ def json_schema_extra(schema: dict, model: BaseModel) -> None:
211
+ # Add the options to the schema for better documentation
212
+ if "properties" in schema and "answer" in schema["properties"]:
213
+ schema["properties"]["answer"]["description"] = "Matrix responses for each item"
214
+ if "properties" in schema["properties"]["answer"]:
215
+ for _, prop in schema["properties"]["answer"]["properties"].items():
216
+ prop["enum"] = list(question_options)
87
217
 
88
218
  return MatrixResponse
89
219
 
90
220
 
91
221
  class MatrixResponseValidator(ResponseValidatorABC):
92
- """Optional placeholder validator, if still needed for example/fixing logic."""
222
+ """
223
+ Validator for matrix question responses that attempts to fix invalid responses.
224
+
225
+ This validator tries multiple approaches to recover valid matrix responses from
226
+ malformed inputs, including JSON parsing, remapping numeric keys, and extracting
227
+ structured data from text.
228
+ """
93
229
  required_params = ["question_items", "question_options", "permissive"]
94
230
 
95
231
  valid_examples = [
@@ -98,8 +234,17 @@ class MatrixResponseValidator(ResponseValidatorABC):
98
234
  {
99
235
  "question_items": ["Item1", "Item2"],
100
236
  "question_options": [1, 2, 3],
237
+ "permissive": False
101
238
  },
102
- )
239
+ ),
240
+ (
241
+ {"answer": {"Item1": "Yes", "Item2": "No"}},
242
+ {
243
+ "question_items": ["Item1", "Item2"],
244
+ "question_options": ["Yes", "No", "Maybe"],
245
+ "permissive": False
246
+ },
247
+ ),
103
248
  ]
104
249
 
105
250
  invalid_examples = [
@@ -108,14 +253,16 @@ class MatrixResponseValidator(ResponseValidatorABC):
108
253
  {
109
254
  "question_items": ["Item1", "Item2"],
110
255
  "question_options": [1, 2, 3],
256
+ "permissive": False
111
257
  },
112
- "Missing responses for some items",
258
+ "Missing responses for items",
113
259
  ),
114
260
  (
115
261
  {"answer": {"Item1": 4, "Item2": 5}},
116
262
  {
117
263
  "question_items": ["Item1", "Item2"],
118
264
  "question_options": [1, 2, 3],
265
+ "permissive": False
119
266
  },
120
267
  "Invalid options selected",
121
268
  ),
@@ -123,39 +270,373 @@ class MatrixResponseValidator(ResponseValidatorABC):
123
270
 
124
271
  def fix(self, response, verbose=False):
125
272
  """
126
- Example fix() method to try and repair a partially invalid response.
127
- (This logic is carried over from your original code.)
273
+ Attempts to fix an invalid matrix response by trying multiple parsing strategies.
274
+
275
+ Args:
276
+ response: The invalid response to fix
277
+ verbose: Whether to print verbose debugging information
278
+
279
+ Returns:
280
+ A fixed response dict if fixable, otherwise the original response
128
281
  """
129
282
  if verbose:
130
283
  print(f"Fixing matrix response: {response}")
131
-
132
- # If we have generated tokens, try to parse them
133
- if "generated_tokens" in response:
284
+
285
+ # If response doesn't have an answer field, nothing to do
286
+ if "answer" not in response:
287
+ if verbose:
288
+ print("Response has no answer field, cannot fix")
289
+ return response
290
+
291
+ # Strategy 1: If we have generated_tokens, try to parse them as JSON
292
+ if "generated_tokens" in response and response["generated_tokens"]:
134
293
  try:
135
- import json
136
-
137
- fixed = json.loads(response["generated_tokens"])
138
- if isinstance(fixed, dict):
139
- # Map numeric keys to question items
140
- mapped_answer = {}
141
- for idx, item in enumerate(self.question_items):
142
- if str(idx) in fixed:
143
- mapped_answer[item] = fixed[str(idx)]
144
- if mapped_answer:
145
- return {"answer": mapped_answer}
146
- except (ValueError, KeyError, TypeError):
147
- pass # Just continue
148
-
149
- # If answer uses numeric keys, map them to question items
150
- if "answer" in response and isinstance(response["answer"], dict):
151
- if all(str(key).isdigit() for key in response["answer"].keys()):
294
+ # Try to parse generated_tokens as JSON
295
+ tokens_text = str(response["generated_tokens"])
296
+ json_match = re.search(r'\{.*\}', tokens_text, re.DOTALL)
297
+
298
+ if json_match:
299
+ json_str = json_match.group(0)
300
+ fixed = json.loads(json_str)
301
+
302
+ if isinstance(fixed, dict):
303
+ # Map numeric keys to question items if needed
304
+ if all(str(k).isdigit() for k in fixed.keys()):
305
+ if verbose:
306
+ print(f"JSON extraction found numeric keys: {fixed}")
307
+ print(f"Question items: {self.question_items}")
308
+ print(f"Question options: {self.question_options}")
309
+
310
+ # Special handling for case when numeric keys directly represent option indices
311
+ # This is the case we're trying to fix: {"0": 1, "1": 3, "2": 0} maps to options at those indices
312
+ direct_mapped_answer = {}
313
+ if verbose:
314
+ print(f"Attempting to map numeric key/value format in JSON: {fixed}")
315
+
316
+ for idx, item in enumerate(self.question_items):
317
+ if str(idx) in fixed:
318
+ # Get the option index directly from the value
319
+ option_idx = fixed[str(idx)]
320
+
321
+ # Convert to int if needed
322
+ if isinstance(option_idx, str) and option_idx.isdigit():
323
+ option_idx = int(option_idx)
324
+
325
+ if verbose:
326
+ print(f"Item {item} at index {idx} maps to value {option_idx}")
327
+
328
+ if isinstance(option_idx, (int, float)) and 0 <= option_idx < len(self.question_options):
329
+ direct_mapped_answer[item] = self.question_options[option_idx]
330
+ if verbose:
331
+ print(f"Mapped option_idx {option_idx} to {self.question_options[option_idx]}")
332
+
333
+ if direct_mapped_answer and len(direct_mapped_answer) == len(self.question_items):
334
+ proposed_data = {
335
+ "answer": direct_mapped_answer,
336
+ "comment": response.get("comment"),
337
+ "generated_tokens": response.get("generated_tokens")
338
+ }
339
+ if verbose:
340
+ print(f"Created direct option mapping from JSON: {proposed_data}")
341
+ try:
342
+ self.response_model(**proposed_data)
343
+ if verbose:
344
+ print(f"Successfully fixed with direct option mapping from JSON: {proposed_data}")
345
+ return proposed_data
346
+ except Exception as e:
347
+ if verbose:
348
+ print(f"Direct option mapping from JSON failed validation: {e}")
349
+
350
+ # Try the standard approach as well
351
+ mapped_answer = {}
352
+ for idx, item in enumerate(self.question_items):
353
+ if str(idx) in fixed:
354
+ # Get the value (column index) from the response
355
+ value_idx = fixed[str(idx)]
356
+
357
+ # Convert to int if it's a digit string
358
+ if isinstance(value_idx, str) and value_idx.isdigit():
359
+ value_idx = int(value_idx)
360
+
361
+ # Convert column index to actual option value
362
+ if isinstance(value_idx, (int, float)) and 0 <= value_idx < len(self.question_options):
363
+ option_value = self.question_options[value_idx]
364
+ mapped_answer[item] = option_value
365
+ else:
366
+ # If the value is already a valid option, use it directly
367
+ if value_idx in self.question_options:
368
+ mapped_answer[item] = value_idx
369
+ else:
370
+ # Last resort - try to use it as a direct value even if not in options
371
+ mapped_answer[item] = value_idx
372
+
373
+ if mapped_answer and (len(mapped_answer) == len(self.question_items) or self.permissive):
374
+ proposed_data = {
375
+ "answer": mapped_answer,
376
+ "comment": response.get("comment"),
377
+ "generated_tokens": response.get("generated_tokens")
378
+ }
379
+ try:
380
+ # Validate the fixed response
381
+ self.response_model(**proposed_data)
382
+ if verbose:
383
+ print(f"Successfully fixed by parsing JSON: {proposed_data}")
384
+ return proposed_data
385
+ except Exception as e:
386
+ if verbose:
387
+ print(f"Fixed response failed validation: {e}")
388
+
389
+ # Try again with string values for all options
390
+ text_mapped_answer = {}
391
+ for item_name, option_value in mapped_answer.items():
392
+ text_mapped_answer[item_name] = str(option_value)
393
+
394
+ proposed_data = {
395
+ "answer": text_mapped_answer,
396
+ "comment": response.get("comment"),
397
+ "generated_tokens": response.get("generated_tokens")
398
+ }
399
+ try:
400
+ self.response_model(**proposed_data)
401
+ if verbose:
402
+ print(f"Successfully fixed with string conversion from JSON: {proposed_data}")
403
+ return proposed_data
404
+ except Exception as e:
405
+ if verbose:
406
+ print(f"String conversion from JSON failed validation: {e}")
407
+ else:
408
+ # The JSON already has string keys, use directly
409
+ proposed_data = {
410
+ "answer": fixed,
411
+ "comment": response.get("comment"),
412
+ "generated_tokens": response.get("generated_tokens")
413
+ }
414
+ try:
415
+ self.response_model(**proposed_data)
416
+ if verbose:
417
+ print(f"Successfully fixed by direct JSON: {proposed_data}")
418
+ return proposed_data
419
+ except Exception as e:
420
+ if verbose:
421
+ print(f"Fixed response failed validation: {e}")
422
+
423
+ # If validation failed, check if we need to map string keys to item names
424
+ # This handles cases where the model responded with something like {"Row 0": 1, "Row 1": 2}
425
+ # instead of using the exact item names
426
+ item_map = {}
427
+ for item in self.question_items:
428
+ # Create various forms of the item name that might appear in responses
429
+ item_variants = [
430
+ item.lower(),
431
+ item.upper(),
432
+ item.strip(),
433
+ f"Row {self.question_items.index(item)}",
434
+ f"Item {self.question_items.index(item)}",
435
+ f"{self.question_items.index(item)}"
436
+ ]
437
+ for key in fixed.keys():
438
+ if isinstance(key, str):
439
+ key_lower = key.lower().strip()
440
+ if key_lower in item_variants or item.lower() in key_lower:
441
+ item_map[key] = item
442
+
443
+ if item_map:
444
+ mapped_answer = {}
445
+ for key, value in fixed.items():
446
+ if key in item_map:
447
+ # Handle both numeric indices and direct values
448
+ if isinstance(value, (int, float)) and 0 <= value < len(self.question_options):
449
+ mapped_answer[item_map[key]] = self.question_options[value]
450
+ else:
451
+ mapped_answer[item_map[key]] = value
452
+
453
+ if mapped_answer:
454
+ proposed_data = {
455
+ "answer": mapped_answer,
456
+ "comment": response.get("comment"),
457
+ "generated_tokens": response.get("generated_tokens")
458
+ }
459
+ try:
460
+ self.response_model(**proposed_data)
461
+ if verbose:
462
+ print(f"Successfully fixed by mapping item names: {proposed_data}")
463
+ return proposed_data
464
+ except Exception as e:
465
+ if verbose:
466
+ print(f"Item-mapped response failed validation: {e}")
467
+ except (ValueError, KeyError, TypeError, json.JSONDecodeError) as e:
468
+ if verbose:
469
+ print(f"JSON parsing failed: {e}")
470
+ # Continue to other strategies
471
+
472
+ # Strategy 2: If answer uses numeric keys, map them to question items
473
+ if isinstance(response.get("answer"), dict):
474
+ answer_dict = response["answer"]
475
+
476
+ if all(str(k).isdigit() for k in answer_dict.keys()):
477
+ if verbose:
478
+ print(f"Processing answer with numeric keys: {answer_dict}")
479
+ print(f"Question items: {self.question_items}")
480
+ print(f"Question options: {self.question_options}")
481
+
152
482
  mapped_answer = {}
153
483
  for idx, item in enumerate(self.question_items):
154
- if str(idx) in response["answer"]:
155
- mapped_answer[item] = response["answer"][str(idx)]
156
- if mapped_answer:
157
- response["answer"] = mapped_answer
158
-
484
+ if str(idx) in answer_dict:
485
+ # Get the value (column index) from the response
486
+ value_idx = answer_dict[str(idx)]
487
+
488
+ # Convert to int if it's a digit string
489
+ if isinstance(value_idx, str) and value_idx.isdigit():
490
+ value_idx = int(value_idx)
491
+
492
+ if verbose:
493
+ print(f"Processing item {item} at index {idx}, value_idx={value_idx}")
494
+
495
+ # Convert column index to actual option value
496
+ if isinstance(value_idx, (int, float)) and 0 <= value_idx < len(self.question_options):
497
+ option_value = self.question_options[value_idx]
498
+ mapped_answer[item] = option_value
499
+ if verbose:
500
+ print(f"Mapped column index {value_idx} to option '{option_value}'")
501
+ else:
502
+ # If the value is already a valid option, use it directly
503
+ if value_idx in self.question_options:
504
+ mapped_answer[item] = value_idx
505
+ if verbose:
506
+ print(f"Used direct option value '{value_idx}'")
507
+ else:
508
+ # Last resort - try to use it as a direct value even if not in options
509
+ # (this helps with permissive mode)
510
+ mapped_answer[item] = value_idx
511
+ if verbose:
512
+ print(f"Used non-option value '{value_idx}' as direct value")
513
+
514
+ if mapped_answer and len(mapped_answer) == len(self.question_items):
515
+ if verbose:
516
+ print(f"Created complete mapped answer: {mapped_answer}")
517
+
518
+ proposed_data = {
519
+ "answer": mapped_answer,
520
+ "comment": response.get("comment"),
521
+ "generated_tokens": response.get("generated_tokens")
522
+ }
523
+ try:
524
+ self.response_model(**proposed_data)
525
+ if verbose:
526
+ print(f"Successfully fixed by mapping numeric keys: {proposed_data}")
527
+ return proposed_data
528
+ except Exception as e:
529
+ if verbose:
530
+ print(f"Fixed response failed validation: {e}")
531
+
532
+ # Try again with string values for the options
533
+ text_mapped_answer = {}
534
+ for item_name, option_value in mapped_answer.items():
535
+ text_mapped_answer[item_name] = str(option_value)
536
+
537
+ proposed_data = {
538
+ "answer": text_mapped_answer,
539
+ "comment": response.get("comment"),
540
+ "generated_tokens": response.get("generated_tokens")
541
+ }
542
+ try:
543
+ self.response_model(**proposed_data)
544
+ if verbose:
545
+ print(f"Successfully fixed with string conversion: {proposed_data}")
546
+ return proposed_data
547
+ except Exception as e:
548
+ if verbose:
549
+ print(f"String conversion failed validation: {e}")
550
+
551
+ # Special handling for case when numeric keys directly represent option indices
552
+ # This is the case we're trying to fix: {"0": 1, "1": 3, "2": 0} maps to options at those indices
553
+ direct_mapped_answer = {}
554
+ if verbose:
555
+ print(f"Attempting to map numeric key/value format in answer: {answer_dict}")
556
+
557
+ for idx, item in enumerate(self.question_items):
558
+ if str(idx) in answer_dict:
559
+ # Get the option index directly from the value
560
+ option_idx = answer_dict[str(idx)]
561
+
562
+ # Convert to int if needed
563
+ if isinstance(option_idx, str) and option_idx.isdigit():
564
+ option_idx = int(option_idx)
565
+
566
+ if verbose:
567
+ print(f"Item {item} at index {idx} maps to value {option_idx}")
568
+
569
+ if isinstance(option_idx, (int, float)) and 0 <= option_idx < len(self.question_options):
570
+ direct_mapped_answer[item] = self.question_options[option_idx]
571
+ if verbose:
572
+ print(f"Mapped option_idx {option_idx} to {self.question_options[option_idx]}")
573
+
574
+ if direct_mapped_answer and len(direct_mapped_answer) == len(self.question_items):
575
+ proposed_data = {
576
+ "answer": direct_mapped_answer,
577
+ "comment": response.get("comment"),
578
+ "generated_tokens": response.get("generated_tokens")
579
+ }
580
+ if verbose:
581
+ print(f"Created direct option mapping: {proposed_data}")
582
+ try:
583
+ self.response_model(**proposed_data)
584
+ if verbose:
585
+ print(f"Successfully fixed with direct option mapping: {proposed_data}")
586
+ return proposed_data
587
+ except Exception as e:
588
+ if verbose:
589
+ print(f"Direct option mapping failed validation: {e}")
590
+
591
+ # Strategy 3: If answer is a string, try to extract a structured response
592
+ if isinstance(response.get("answer"), str):
593
+ answer_text = response["answer"]
594
+
595
+ # Try to extract item-option pairs using regex
596
+ pairs = re.findall(r'([^:,]+):\s*([^,]+)', answer_text)
597
+ if pairs:
598
+ extracted = {}
599
+ for item, option in pairs:
600
+ item = item.strip()
601
+ option = option.strip()
602
+
603
+ # Match the item name with the closest question item
604
+ best_match = None
605
+ for q_item in self.question_items:
606
+ if q_item.lower() in item.lower():
607
+ best_match = q_item
608
+ break
609
+
610
+ if best_match:
611
+ # Try to match the option with question options
612
+ matched_option = None
613
+ for q_option in self.question_options:
614
+ q_option_str = str(q_option)
615
+ if q_option_str == option or q_option_str in option:
616
+ matched_option = q_option
617
+ break
618
+
619
+ if matched_option is not None:
620
+ extracted[best_match] = matched_option
621
+
622
+ if extracted and (len(extracted) == len(self.question_items) or self.permissive):
623
+ proposed_data = {
624
+ "answer": extracted,
625
+ "comment": response.get("comment"),
626
+ "generated_tokens": response.get("generated_tokens")
627
+ }
628
+ try:
629
+ self.response_model(**proposed_data)
630
+ if verbose:
631
+ print(f"Successfully fixed by extracting pairs: {proposed_data}")
632
+ return proposed_data
633
+ except Exception as e:
634
+ if verbose:
635
+ print(f"Fixed response failed validation: {e}")
636
+
637
+ # If we got here, we couldn't fix the response
638
+ if verbose:
639
+ print("Could not fix matrix response, returning original")
159
640
  return response
160
641
 
161
642
 
@@ -163,10 +644,23 @@ class QuestionMatrix(QuestionBase):
163
644
  """
164
645
  A question that presents a matrix/grid where multiple items are rated
165
646
  or selected from the same set of options.
166
-
167
- This version dynamically builds a Pydantic model at runtime
168
- (via `create_matrix_response`) and automatically raises ValidationError
169
- if the user provides an invalid or incomplete answer.
647
+
648
+ This question type allows respondents to provide an answer for each row
649
+ in a grid, selecting from the same set of options for each row. It's often
650
+ used for Likert scales, ratings grids, or any scenario where multiple items
651
+ need to be rated using the same scale.
652
+
653
+ Examples:
654
+ >>> # Create a happiness rating matrix
655
+ >>> question = QuestionMatrix(
656
+ ... question_name="happiness_matrix",
657
+ ... question_text="Rate your happiness with each aspect:",
658
+ ... question_items=["Work", "Family", "Social life"],
659
+ ... question_options=[1, 2, 3, 4, 5],
660
+ ... option_labels={1: "Very unhappy", 3: "Neutral", 5: "Very happy"}
661
+ ... )
662
+ >>> # The response is a dict matching each item to a rating
663
+ >>> response = {"answer": {"Work": 4, "Family": 5, "Social life": 3}}
170
664
  """
171
665
 
172
666
  question_type = "matrix"
@@ -196,12 +690,12 @@ class QuestionMatrix(QuestionBase):
196
690
  Args:
197
691
  question_name: The name of the question
198
692
  question_text: The text of the question
199
- question_items: List of items to be rated or answered
200
- question_options: Possible answer options (e.g., [1,2,3] or ["Yes","No"])
693
+ question_items: List of items to be rated or answered (rows)
694
+ question_options: Possible answer options for each item (columns)
201
695
  option_labels: Optional mapping of options to labels (e.g. {1: "Sad", 5: "Happy"})
202
696
  include_comment: Whether to include a comment field
203
- answering_instructions: Custom instructions
204
- question_presentation: Custom presentation
697
+ answering_instructions: Custom instructions template
698
+ question_presentation: Custom presentation template
205
699
  permissive: Whether to allow any values & extra items instead of strictly checking
206
700
  """
207
701
  self.question_name = question_name
@@ -224,7 +718,10 @@ class QuestionMatrix(QuestionBase):
224
718
 
225
719
  def create_response_model(self) -> Type[BaseModel]:
226
720
  """
227
- Returns the pydantic model that will parse/validate a user answer.
721
+ Returns the pydantic model for validating responses to this question.
722
+
723
+ The model is dynamically created based on the question's configuration,
724
+ including allowed items, options, and permissiveness.
228
725
  """
229
726
  return create_matrix_response(
230
727
  self.question_items,
@@ -232,9 +729,29 @@ class QuestionMatrix(QuestionBase):
232
729
  self.permissive
233
730
  )
234
731
 
732
+ def _simulate_answer(self) -> dict:
733
+ """
734
+ Simulate a random valid answer for testing purposes.
735
+
736
+ Returns:
737
+ A valid simulated response with random selections
738
+ """
739
+ return {
740
+ "answer": {
741
+ item: random.choice(self.question_options)
742
+ for item in self.question_items
743
+ },
744
+ "comment": "Sample matrix response"
745
+ }
746
+
235
747
  @property
236
748
  def question_html_content(self) -> str:
237
- """Generate HTML representation of the matrix question."""
749
+ """
750
+ Generate an HTML representation of the matrix question.
751
+
752
+ Returns:
753
+ HTML content string for rendering the question
754
+ """
238
755
  template = Template(
239
756
  """
240
757
  <table class="matrix-question">
@@ -276,7 +793,12 @@ class QuestionMatrix(QuestionBase):
276
793
  @classmethod
277
794
  @inject_exception
278
795
  def example(cls) -> QuestionMatrix:
279
- """Return an example matrix question."""
796
+ """
797
+ Return an example matrix question.
798
+
799
+ Returns:
800
+ An example QuestionMatrix instance for happiness ratings by family size
801
+ """
280
802
  return cls(
281
803
  question_name="child_happiness",
282
804
  question_text="How happy would you be with different numbers of children?",
@@ -288,13 +810,4 @@ class QuestionMatrix(QuestionBase):
288
810
  ],
289
811
  question_options=[1, 2, 3, 4, 5],
290
812
  option_labels={1: "Very sad", 3: "Neutral", 5: "Extremely happy"},
291
- )
292
-
293
- def _simulate_answer(self) -> dict:
294
- """Simulate a random valid answer."""
295
- return {
296
- "answer": {
297
- item: random.choice(self.question_options)
298
- for item in self.question_items
299
- }
300
- }
813
+ )