edsl 0.1.53__py3-none-any.whl → 0.1.55__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. edsl/__init__.py +8 -1
  2. edsl/__init__original.py +134 -0
  3. edsl/__version__.py +1 -1
  4. edsl/agents/agent.py +29 -0
  5. edsl/agents/agent_list.py +36 -1
  6. edsl/base/base_class.py +281 -151
  7. edsl/buckets/__init__.py +8 -3
  8. edsl/buckets/bucket_collection.py +9 -3
  9. edsl/buckets/model_buckets.py +4 -2
  10. edsl/buckets/token_bucket.py +2 -2
  11. edsl/buckets/token_bucket_client.py +5 -3
  12. edsl/caching/cache.py +131 -62
  13. edsl/caching/cache_entry.py +70 -58
  14. edsl/caching/sql_dict.py +17 -0
  15. edsl/cli.py +99 -0
  16. edsl/config/config_class.py +16 -0
  17. edsl/conversation/__init__.py +31 -0
  18. edsl/coop/coop.py +276 -242
  19. edsl/coop/coop_jobs_objects.py +59 -0
  20. edsl/coop/coop_objects.py +29 -0
  21. edsl/coop/coop_regular_objects.py +26 -0
  22. edsl/coop/utils.py +24 -19
  23. edsl/dataset/dataset.py +338 -101
  24. edsl/db_list/sqlite_list.py +349 -0
  25. edsl/inference_services/__init__.py +40 -5
  26. edsl/inference_services/exceptions.py +11 -0
  27. edsl/inference_services/services/anthropic_service.py +5 -2
  28. edsl/inference_services/services/aws_bedrock.py +6 -2
  29. edsl/inference_services/services/azure_ai.py +6 -2
  30. edsl/inference_services/services/google_service.py +3 -2
  31. edsl/inference_services/services/mistral_ai_service.py +6 -2
  32. edsl/inference_services/services/open_ai_service.py +6 -2
  33. edsl/inference_services/services/perplexity_service.py +6 -2
  34. edsl/inference_services/services/test_service.py +105 -7
  35. edsl/interviews/answering_function.py +167 -59
  36. edsl/interviews/interview.py +124 -72
  37. edsl/interviews/interview_task_manager.py +10 -0
  38. edsl/invigilators/invigilators.py +10 -1
  39. edsl/jobs/async_interview_runner.py +146 -104
  40. edsl/jobs/data_structures.py +6 -4
  41. edsl/jobs/decorators.py +61 -0
  42. edsl/jobs/fetch_invigilator.py +61 -18
  43. edsl/jobs/html_table_job_logger.py +14 -2
  44. edsl/jobs/jobs.py +180 -104
  45. edsl/jobs/jobs_component_constructor.py +2 -2
  46. edsl/jobs/jobs_interview_constructor.py +2 -0
  47. edsl/jobs/jobs_pricing_estimation.py +127 -46
  48. edsl/jobs/jobs_remote_inference_logger.py +4 -0
  49. edsl/jobs/jobs_runner_status.py +30 -25
  50. edsl/jobs/progress_bar_manager.py +79 -0
  51. edsl/jobs/remote_inference.py +35 -1
  52. edsl/key_management/key_lookup_builder.py +6 -1
  53. edsl/language_models/language_model.py +102 -12
  54. edsl/language_models/model.py +10 -3
  55. edsl/language_models/price_manager.py +45 -75
  56. edsl/language_models/registry.py +5 -0
  57. edsl/language_models/utilities.py +2 -1
  58. edsl/notebooks/notebook.py +77 -10
  59. edsl/questions/VALIDATION_README.md +134 -0
  60. edsl/questions/__init__.py +24 -1
  61. edsl/questions/exceptions.py +21 -0
  62. edsl/questions/question_check_box.py +171 -149
  63. edsl/questions/question_dict.py +243 -51
  64. edsl/questions/question_multiple_choice_with_other.py +624 -0
  65. edsl/questions/question_registry.py +2 -1
  66. edsl/questions/templates/multiple_choice_with_other/__init__.py +0 -0
  67. edsl/questions/templates/multiple_choice_with_other/answering_instructions.jinja +15 -0
  68. edsl/questions/templates/multiple_choice_with_other/question_presentation.jinja +17 -0
  69. edsl/questions/validation_analysis.py +185 -0
  70. edsl/questions/validation_cli.py +131 -0
  71. edsl/questions/validation_html_report.py +404 -0
  72. edsl/questions/validation_logger.py +136 -0
  73. edsl/results/result.py +63 -16
  74. edsl/results/results.py +702 -171
  75. edsl/scenarios/construct_download_link.py +16 -3
  76. edsl/scenarios/directory_scanner.py +226 -226
  77. edsl/scenarios/file_methods.py +5 -0
  78. edsl/scenarios/file_store.py +117 -6
  79. edsl/scenarios/handlers/__init__.py +5 -1
  80. edsl/scenarios/handlers/mp4_file_store.py +104 -0
  81. edsl/scenarios/handlers/webm_file_store.py +104 -0
  82. edsl/scenarios/scenario.py +120 -101
  83. edsl/scenarios/scenario_list.py +800 -727
  84. edsl/scenarios/scenario_list_gc_test.py +146 -0
  85. edsl/scenarios/scenario_list_memory_test.py +214 -0
  86. edsl/scenarios/scenario_list_source_refactor.md +35 -0
  87. edsl/scenarios/scenario_selector.py +5 -4
  88. edsl/scenarios/scenario_source.py +1990 -0
  89. edsl/scenarios/tests/test_scenario_list_sources.py +52 -0
  90. edsl/surveys/survey.py +22 -0
  91. edsl/tasks/__init__.py +4 -2
  92. edsl/tasks/task_history.py +198 -36
  93. edsl/tests/scenarios/test_ScenarioSource.py +51 -0
  94. edsl/tests/scenarios/test_scenario_list_sources.py +51 -0
  95. edsl/utilities/__init__.py +2 -1
  96. edsl/utilities/decorators.py +121 -0
  97. edsl/utilities/memory_debugger.py +1010 -0
  98. {edsl-0.1.53.dist-info → edsl-0.1.55.dist-info}/METADATA +52 -76
  99. {edsl-0.1.53.dist-info → edsl-0.1.55.dist-info}/RECORD +102 -78
  100. edsl/jobs/jobs_runner_asyncio.py +0 -281
  101. edsl/language_models/unused/fake_openai_service.py +0 -60
  102. {edsl-0.1.53.dist-info → edsl-0.1.55.dist-info}/LICENSE +0 -0
  103. {edsl-0.1.53.dist-info → edsl-0.1.55.dist-info}/WHEEL +0 -0
  104. {edsl-0.1.53.dist-info → edsl-0.1.55.dist-info}/entry_points.txt +0 -0
@@ -24,29 +24,29 @@ if TYPE_CHECKING:
24
24
  class CheckboxResponse(BaseModel):
25
25
  """
26
26
  Base Pydantic model for validating checkbox responses.
27
-
27
+
28
28
  This model defines the structure and validation rules for responses to
29
29
  checkbox questions, ensuring that selected options are properly formatted
30
30
  as a list of choices.
31
-
31
+
32
32
  Attributes:
33
33
  answer: List of selected choices
34
34
  comment: Optional comment provided with the answer
35
35
  generated_tokens: Optional raw LLM output for token tracking
36
-
36
+
37
37
  Examples:
38
38
  >>> # Valid response with list of options
39
39
  >>> response = CheckboxResponse(answer=[0, 1])
40
40
  >>> response.answer
41
41
  [0, 1]
42
-
42
+
43
43
  >>> # Valid response with comment
44
44
  >>> response = CheckboxResponse(answer=[1], comment="This is my choice")
45
45
  >>> response.answer
46
46
  [1]
47
47
  >>> response.comment
48
48
  'This is my choice'
49
-
49
+
50
50
  >>> # Invalid non-list answer
51
51
  >>> try:
52
52
  ... CheckboxResponse(answer=1)
@@ -54,6 +54,7 @@ class CheckboxResponse(BaseModel):
54
54
  ... print("Validation error occurred")
55
55
  Validation error occurred
56
56
  """
57
+
57
58
  answer: List[Any]
58
59
  comment: Optional[str] = None
59
60
  generated_tokens: Optional[Any] = None
@@ -67,59 +68,59 @@ def create_checkbox_response_model(
67
68
  ):
68
69
  """
69
70
  Dynamically create a CheckboxResponse model with a predefined list of choices.
70
-
71
+
71
72
  This function creates a customized Pydantic model for checkbox questions that
72
73
  validates both the format of the response and any constraints on selection count.
73
-
74
+
74
75
  Args:
75
76
  choices: A list of allowed values for the answer field
76
77
  min_selections: Optional minimum number of selections required
77
78
  max_selections: Optional maximum number of selections allowed
78
79
  permissive: If True, constraints are not enforced
79
-
80
+
80
81
  Returns:
81
82
  A new Pydantic model class with appropriate validation
82
-
83
+
83
84
  Examples:
84
85
  >>> # Create model with constraints
85
86
  >>> choices = [0, 1, 2, 3]
86
87
  >>> ConstrainedModel = create_checkbox_response_model(
87
- ... choices=choices,
88
- ... min_selections=1,
88
+ ... choices=choices,
89
+ ... min_selections=1,
89
90
  ... max_selections=2
90
91
  ... )
91
-
92
+
92
93
  >>> # Valid response within constraints
93
94
  >>> response = ConstrainedModel(answer=[0, 1])
94
95
  >>> response.answer
95
96
  [0, 1]
96
-
97
+
97
98
  >>> # Too few selections fails validation
98
99
  >>> try:
99
100
  ... ConstrainedModel(answer=[])
100
101
  ... except Exception as e:
101
102
  ... "at least 1" in str(e)
102
103
  True
103
-
104
+
104
105
  >>> # Too many selections fails validation
105
106
  >>> try:
106
107
  ... ConstrainedModel(answer=[0, 1, 2])
107
108
  ... except Exception as e:
108
109
  ... "at most 2" in str(e)
109
110
  True
110
-
111
+
111
112
  >>> # Invalid choice fails validation
112
113
  >>> try:
113
114
  ... ConstrainedModel(answer=[4])
114
115
  ... except Exception as e:
115
116
  ... any(x in str(e) for x in ["Invalid choice", "not a valid enumeration member", "validation error"])
116
117
  True
117
-
118
+
118
119
  >>> # Permissive model ignores constraints
119
120
  >>> PermissiveModel = create_checkbox_response_model(
120
- ... choices=choices,
121
- ... min_selections=1,
122
- ... max_selections=2,
121
+ ... choices=choices,
122
+ ... min_selections=1,
123
+ ... max_selections=2,
123
124
  ... permissive=True
124
125
  ... )
125
126
  >>> response = PermissiveModel(answer=[0, 1, 2])
@@ -133,152 +134,161 @@ def create_checkbox_response_model(
133
134
  # For permissive mode, we still validate the choice values but ignore count constraints
134
135
  class PermissiveCheckboxResponse(CheckboxResponse):
135
136
  """Checkbox response model with choices validation but no count constraints."""
136
-
137
+
137
138
  answer: Annotated[
138
139
  List[Literal[choice_tuple]],
139
140
  Field(description="List of selected choices"),
140
141
  ]
141
-
142
- @model_validator(mode='after')
142
+
143
+ @model_validator(mode="after")
143
144
  def validate_choices(self):
144
145
  """Validate that each selected choice is valid."""
145
146
  for choice in self.answer:
146
147
  if choice not in choices:
147
148
  validation_error = ValidationError.from_exception_data(
148
- title='CheckboxResponse',
149
- line_errors=[{
150
- 'type': 'value_error',
151
- 'loc': ('answer',),
152
- 'msg': f'Invalid choice: {choice}. Must be one of: {choices}',
153
- 'input': choice,
154
- 'ctx': {'error': 'Invalid choice'}
155
- }]
149
+ title="CheckboxResponse",
150
+ line_errors=[
151
+ {
152
+ "type": "value_error",
153
+ "loc": ("answer",),
154
+ "msg": f"Invalid choice: {choice}. Must be one of: {choices}",
155
+ "input": choice,
156
+ "ctx": {"error": "Invalid choice"},
157
+ }
158
+ ],
156
159
  )
157
160
  raise QuestionAnswerValidationError(
158
161
  message=f"Invalid choice: {choice}. Must be one of: {choices}",
159
162
  data=self.model_dump(),
160
163
  model=self.__class__,
161
- pydantic_error=validation_error
164
+ pydantic_error=validation_error,
162
165
  )
163
166
  return self
164
-
167
+
165
168
  return PermissiveCheckboxResponse
166
169
  else:
167
170
  # For non-permissive mode, enforce both choice values and count constraints
168
171
  class ConstrainedCheckboxResponse(CheckboxResponse):
169
172
  """Checkbox response model with both choice and count constraints."""
170
-
173
+
171
174
  answer: Annotated[
172
175
  List[Literal[choice_tuple]],
173
176
  Field(description="List of selected choices"),
174
177
  ]
175
-
176
- @model_validator(mode='after')
178
+
179
+ @model_validator(mode="after")
177
180
  def validate_selection_count(self):
178
181
  """Validate that the number of selections meets constraints."""
179
182
  if min_selections is not None and len(self.answer) < min_selections:
180
183
  validation_error = ValidationError.from_exception_data(
181
- title='CheckboxResponse',
182
- line_errors=[{
183
- 'type': 'value_error',
184
- 'loc': ('answer',),
185
- 'msg': f'Must select at least {min_selections} option(s)',
186
- 'input': self.answer,
187
- 'ctx': {'error': 'Too few selections'}
188
- }]
184
+ title="CheckboxResponse",
185
+ line_errors=[
186
+ {
187
+ "type": "value_error",
188
+ "loc": ("answer",),
189
+ "msg": f"Must select at least {min_selections} option(s)",
190
+ "input": self.answer,
191
+ "ctx": {"error": "Too few selections"},
192
+ }
193
+ ],
189
194
  )
190
195
  raise QuestionAnswerValidationError(
191
196
  message=f"Must select at least {min_selections} option(s), got {len(self.answer)}",
192
197
  data=self.model_dump(),
193
198
  model=self.__class__,
194
- pydantic_error=validation_error
199
+ pydantic_error=validation_error,
195
200
  )
196
-
201
+
197
202
  if max_selections is not None and len(self.answer) > max_selections:
198
203
  validation_error = ValidationError.from_exception_data(
199
- title='CheckboxResponse',
200
- line_errors=[{
201
- 'type': 'value_error',
202
- 'loc': ('answer',),
203
- 'msg': f'Must select at most {max_selections} option(s)',
204
- 'input': self.answer,
205
- 'ctx': {'error': 'Too many selections'}
206
- }]
204
+ title="CheckboxResponse",
205
+ line_errors=[
206
+ {
207
+ "type": "value_error",
208
+ "loc": ("answer",),
209
+ "msg": f"Must select at most {max_selections} option(s)",
210
+ "input": self.answer,
211
+ "ctx": {"error": "Too many selections"},
212
+ }
213
+ ],
207
214
  )
208
215
  raise QuestionAnswerValidationError(
209
216
  message=f"Must select at most {max_selections} option(s), got {len(self.answer)}",
210
217
  data=self.model_dump(),
211
218
  model=self.__class__,
212
- pydantic_error=validation_error
219
+ pydantic_error=validation_error,
213
220
  )
214
-
221
+
215
222
  # Also validate that each choice is valid
216
223
  for choice in self.answer:
217
224
  if choice not in choices:
218
225
  validation_error = ValidationError.from_exception_data(
219
- title='CheckboxResponse',
220
- line_errors=[{
221
- 'type': 'value_error',
222
- 'loc': ('answer',),
223
- 'msg': f'Invalid choice: {choice}. Must be one of: {choices}',
224
- 'input': choice,
225
- 'ctx': {'error': 'Invalid choice'}
226
- }]
226
+ title="CheckboxResponse",
227
+ line_errors=[
228
+ {
229
+ "type": "value_error",
230
+ "loc": ("answer",),
231
+ "msg": f"Invalid choice: {choice}. Must be one of: {choices}",
232
+ "input": choice,
233
+ "ctx": {"error": "Invalid choice"},
234
+ }
235
+ ],
227
236
  )
228
237
  raise QuestionAnswerValidationError(
229
238
  message=f"Invalid choice: {choice}. Must be one of: {choices}",
230
239
  data=self.model_dump(),
231
240
  model=self.__class__,
232
- pydantic_error=validation_error
241
+ pydantic_error=validation_error,
233
242
  )
234
-
243
+
235
244
  return self
236
-
245
+
237
246
  return ConstrainedCheckboxResponse
238
247
 
239
248
 
240
249
  class CheckBoxResponseValidator(ResponseValidatorABC):
241
250
  """
242
251
  Validator for checkbox question responses.
243
-
252
+
244
253
  This class implements the validation and fixing logic for checkbox responses.
245
254
  It ensures that responses contain valid selections from the available options
246
255
  and that the number of selections meets any constraints.
247
-
256
+
248
257
  Attributes:
249
258
  required_params: List of required parameters for validation.
250
259
  valid_examples: Examples of valid responses for testing.
251
260
  invalid_examples: Examples of invalid responses for testing.
252
-
261
+
253
262
  Examples:
254
263
  >>> from edsl import QuestionCheckBox
255
- >>> q = QuestionCheckBox.example()
264
+ >>> q = QuestionCheckBox.example(use_code=True)
256
265
  >>> validator = q.response_validator
257
-
266
+
258
267
  >>> # Fix string to list
259
268
  >>> response = {"answer": 1}
260
269
  >>> fixed = validator.fix(response)
261
270
  >>> isinstance(fixed["answer"], list)
262
271
  True
263
-
272
+
264
273
  >>> # Extract selections from text
265
274
  >>> response = {"generated_tokens": "I choose options 0 and 2"}
266
275
  >>> fixed = validator.fix(response)
267
276
  >>> sorted(fixed["answer"])
268
277
  [0, 2]
269
-
278
+
270
279
  >>> # Fix comma-separated list
271
280
  >>> response = {"generated_tokens": "0, 1, 3"}
272
281
  >>> fixed = validator.fix(response)
273
282
  >>> sorted(fixed["answer"])
274
283
  [0, 1, 3]
275
-
284
+
276
285
  >>> # Preserve comments when fixing
277
286
  >>> response = {"answer": 1, "comment": "My explanation"}
278
287
  >>> fixed = validator.fix(response)
279
288
  >>> "comment" in fixed and fixed["comment"] == "My explanation"
280
289
  True
281
290
  """
291
+
282
292
  required_params = [
283
293
  "question_options",
284
294
  "min_selections",
@@ -316,20 +326,20 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
316
326
  def fix(self, response, verbose=False):
317
327
  """
318
328
  Fix common issues in checkbox responses.
319
-
329
+
320
330
  This method attempts to extract valid selections from responses with
321
331
  format issues. It can handle:
322
332
  1. Single values that should be lists
323
333
  2. Comma-separated strings in answer field or generated_tokens
324
334
  3. Finding option indices mentioned in text
325
-
335
+
326
336
  Args:
327
337
  response: The response dictionary to fix
328
338
  verbose: If True, print information about the fixing process
329
-
339
+
330
340
  Returns:
331
341
  A fixed version of the response dictionary with a valid list of selections
332
-
342
+
333
343
  Notes:
334
344
  - First tries to convert to a list if the answer is not already a list
335
345
  - Then tries to parse comma-separated values from answer or generated_tokens
@@ -338,16 +348,20 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
338
348
  """
339
349
  if verbose:
340
350
  print("Invalid response of QuestionCheckBox was: ", response)
341
-
351
+
342
352
  # Check if answer exists and is a comma-separated string (common LLM output format)
343
- if "answer" in response and isinstance(response["answer"], str) and "," in response["answer"]:
353
+ if (
354
+ "answer" in response
355
+ and isinstance(response["answer"], str)
356
+ and "," in response["answer"]
357
+ ):
344
358
  if verbose:
345
359
  print(f"Parsing comma-separated answer string: {response['answer']}")
346
-
360
+
347
361
  # Split by commas and strip whitespace
348
362
  proposed_list = response["answer"].split(",")
349
363
  proposed_list = [item.strip() for item in proposed_list]
350
-
364
+
351
365
  # Try to convert to integers if use_code is True
352
366
  if self.use_code:
353
367
  try:
@@ -355,31 +369,33 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
355
369
  except ValueError:
356
370
  # If we can't convert to integers, try to match values to indices
357
371
  if verbose:
358
- print("Could not convert comma-separated values to integers, trying to match options")
359
-
372
+ print(
373
+ "Could not convert comma-separated values to integers, trying to match options"
374
+ )
375
+
360
376
  # Try to match option text values to their indices
361
377
  index_map = {}
362
378
  for i, option in enumerate(self.question_options):
363
379
  index_map[option.lower().strip()] = i
364
-
380
+
365
381
  converted_list = []
366
382
  for item in proposed_list:
367
383
  item_lower = item.lower().strip()
368
384
  if item_lower in index_map:
369
385
  converted_list.append(index_map[item_lower])
370
-
386
+
371
387
  if converted_list:
372
388
  proposed_list = converted_list
373
-
389
+
374
390
  if verbose:
375
391
  print("Proposed solution from comma separation is: ", proposed_list)
376
-
392
+
377
393
  proposed_data = {
378
394
  "answer": proposed_list,
379
395
  "comment": response.get("comment"),
380
396
  "generated_tokens": response.get("generated_tokens"),
381
397
  }
382
-
398
+
383
399
  # Try validating with the proposed solution
384
400
  try:
385
401
  validated = self._base_validate(proposed_data)
@@ -387,14 +403,14 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
387
403
  except Exception as e:
388
404
  if verbose:
389
405
  print(f"Comma-separated solution invalid: {e}")
390
-
406
+
391
407
  # If answer exists but is not a list, convert it to a list
392
408
  elif "answer" in response and not isinstance(response["answer"], list):
393
409
  if verbose:
394
410
  print(f"Converting non-list answer {response['answer']} to a list")
395
411
  answer_value = response["answer"]
396
412
  response = {**response, "answer": [answer_value]}
397
-
413
+
398
414
  # Try validating the fixed response
399
415
  try:
400
416
  validated = self._base_validate(response)
@@ -402,7 +418,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
402
418
  except Exception:
403
419
  if verbose:
404
420
  print("Converting to list didn't fix the issue")
405
-
421
+
406
422
  # Try parsing from generated_tokens if present
407
423
  response_text = response.get("generated_tokens")
408
424
  if response_text and isinstance(response_text, str):
@@ -410,38 +426,40 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
410
426
  if "," in response_text:
411
427
  proposed_list = response_text.split(",")
412
428
  proposed_list = [item.strip() for item in proposed_list]
413
-
429
+
414
430
  if self.use_code:
415
431
  try:
416
432
  proposed_list = [int(i) for i in proposed_list]
417
433
  except ValueError:
418
434
  # If we can't convert to integers, try to match values to indices
419
435
  if verbose:
420
- print("Could not convert comma-separated values to integers, trying to match options")
421
-
436
+ print(
437
+ "Could not convert comma-separated values to integers, trying to match options"
438
+ )
439
+
422
440
  # Try to match option text values to their indices
423
441
  index_map = {}
424
442
  for i, option in enumerate(self.question_options):
425
443
  index_map[option.lower().strip()] = i
426
-
444
+
427
445
  converted_list = []
428
446
  for item in proposed_list:
429
447
  item_lower = item.lower().strip()
430
448
  if item_lower in index_map:
431
449
  converted_list.append(index_map[item_lower])
432
-
450
+
433
451
  if converted_list:
434
452
  proposed_list = converted_list
435
-
453
+
436
454
  if verbose:
437
455
  print("Proposed solution from comma separation is: ", proposed_list)
438
-
456
+
439
457
  proposed_data = {
440
458
  "answer": proposed_list,
441
459
  "comment": response.get("comment"),
442
460
  "generated_tokens": response.get("generated_tokens"),
443
461
  }
444
-
462
+
445
463
  # Try validating with the proposed solution
446
464
  try:
447
465
  validated = self._base_validate(proposed_data)
@@ -449,7 +467,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
449
467
  except Exception as e:
450
468
  if verbose:
451
469
  print(f"Comma-separated solution invalid: {e}")
452
-
470
+
453
471
  # Try finding option indices mentioned in the text
454
472
  matches = []
455
473
  for index, option in enumerate(self.question_options):
@@ -459,17 +477,17 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
459
477
  else:
460
478
  if option in response_text:
461
479
  matches.append(option)
462
-
480
+
463
481
  if matches:
464
482
  if verbose:
465
483
  print(f"Found options mentioned in text: {matches}")
466
-
484
+
467
485
  proposed_data = {
468
486
  "answer": matches,
469
487
  "comment": response.get("comment"),
470
488
  "generated_tokens": response.get("generated_tokens"),
471
489
  }
472
-
490
+
473
491
  # Try validating with the proposed solution
474
492
  try:
475
493
  validated = self._base_validate(proposed_data)
@@ -477,7 +495,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
477
495
  except Exception as e:
478
496
  if verbose:
479
497
  print(f"Text matching solution invalid: {e}")
480
-
498
+
481
499
  # If nothing worked, return the original response
482
500
  return response
483
501
 
@@ -485,11 +503,11 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
485
503
  class QuestionCheckBox(QuestionBase):
486
504
  """
487
505
  A question that prompts the agent to select multiple options from a list.
488
-
506
+
489
507
  QuestionCheckBox allows agents to select one or more items from a predefined
490
508
  list of options. It's useful for "select all that apply" scenarios, multi-select
491
509
  preferences, or any question where multiple valid selections are possible.
492
-
510
+
493
511
  Attributes:
494
512
  question_type (str): Identifier for this question type, set to "checkbox".
495
513
  purpose (str): Brief description of when to use this question type.
@@ -498,16 +516,16 @@ class QuestionCheckBox(QuestionBase):
498
516
  max_selections: Optional maximum number of selections allowed.
499
517
  _response_model: Initially None, set by create_response_model().
500
518
  response_validator_class: Class used to validate and fix responses.
501
-
519
+
502
520
  Examples:
503
521
  >>> # Basic creation works
504
522
  >>> q = QuestionCheckBox.example()
505
523
  >>> q.question_type
506
524
  'checkbox'
507
-
525
+
508
526
  >>> # Create preferences question with selection constraints
509
527
  >>> q = QuestionCheckBox(
510
- ... question_name="favorite_fruits",
528
+ ... question_name="favorite_fruits",
511
529
  ... question_text="Which fruits do you like?",
512
530
  ... question_options=["Apple", "Banana", "Cherry", "Durian", "Elderberry"],
513
531
  ... min_selections=1,
@@ -538,14 +556,14 @@ class QuestionCheckBox(QuestionBase):
538
556
  min_selections: Optional[int] = None,
539
557
  max_selections: Optional[int] = None,
540
558
  include_comment: bool = True,
541
- use_code: bool = True,
559
+ use_code: bool = False,
542
560
  question_presentation: Optional[str] = None,
543
561
  answering_instructions: Optional[str] = None,
544
562
  permissive: bool = False,
545
563
  ):
546
564
  """
547
565
  Initialize a new checkbox question.
548
-
566
+
549
567
  Args:
550
568
  question_name: Identifier for the question, used in results and templates.
551
569
  Must be a valid Python variable name.
@@ -558,7 +576,7 @@ class QuestionCheckBox(QuestionBase):
558
576
  question_presentation: Optional custom presentation template.
559
577
  answering_instructions: Optional additional instructions.
560
578
  permissive: If True, ignore selection count constraints during validation.
561
-
579
+
562
580
  Examples:
563
581
  >>> q = QuestionCheckBox(
564
582
  ... question_name="symptoms",
@@ -568,12 +586,12 @@ class QuestionCheckBox(QuestionBase):
568
586
  ... )
569
587
  >>> q.question_name
570
588
  'symptoms'
571
-
589
+
572
590
  >>> # Question with both min and max
573
591
  >>> q = QuestionCheckBox(
574
592
  ... question_name="pizza_toppings",
575
593
  ... question_text="Choose 2-4 toppings for your pizza:",
576
- ... question_options=["Cheese", "Pepperoni", "Mushroom", "Onion",
594
+ ... question_options=["Cheese", "Pepperoni", "Mushroom", "Onion",
577
595
  ... "Sausage", "Bacon", "Pineapple"],
578
596
  ... min_selections=2,
579
597
  ... max_selections=4
@@ -595,15 +613,15 @@ class QuestionCheckBox(QuestionBase):
595
613
  def create_response_model(self):
596
614
  """
597
615
  Create a response model with the appropriate constraints.
598
-
616
+
599
617
  This method creates a Pydantic model customized with the options and
600
618
  selection count constraints specified for this question instance.
601
-
619
+
602
620
  Returns:
603
621
  A Pydantic model class tailored to this question's constraints.
604
-
622
+
605
623
  Examples:
606
- >>> q = QuestionCheckBox.example()
624
+ >>> q = QuestionCheckBox.example(use_code=True)
607
625
  >>> model = q.create_response_model()
608
626
  >>> model(answer=[0, 2]) # Select first and third options
609
627
  ConstrainedCheckboxResponse(answer=[0, 2], comment=None, generated_tokens=None)
@@ -630,23 +648,23 @@ class QuestionCheckBox(QuestionBase):
630
648
  ):
631
649
  """
632
650
  Translate the answer codes to the actual answer text.
633
-
651
+
634
652
  For checkbox questions with use_code=True, the agent responds with
635
653
  option indices (e.g., [0, 1]) which need to be translated to their
636
654
  corresponding option text values (e.g., ["Option A", "Option B"]).
637
-
655
+
638
656
  Args:
639
657
  answer_codes: List of selected option indices or values
640
658
  scenario: Optional scenario with variables for template rendering
641
-
659
+
642
660
  Returns:
643
661
  List of selected option texts
644
-
662
+
645
663
  Examples:
646
664
  >>> q = QuestionCheckBox(
647
665
  ... question_name="example",
648
666
  ... question_text="Select options:",
649
- ... question_options=["A", "B", "C"]
667
+ ... question_options=["A", "B", "C"],use_code=True
650
668
  ... )
651
669
  >>> q._translate_answer_code_to_answer([0, 2])
652
670
  ['A', 'C']
@@ -666,16 +684,16 @@ class QuestionCheckBox(QuestionBase):
666
684
  def _simulate_answer(self, human_readable=True):
667
685
  """
668
686
  Simulate a valid answer for debugging purposes.
669
-
687
+
670
688
  This method generates a random valid answer for the checkbox question,
671
689
  useful for testing and demonstrations.
672
-
690
+
673
691
  Args:
674
692
  human_readable: If True, return option text values; if False, return indices
675
-
693
+
676
694
  Returns:
677
695
  A dictionary with a valid random answer
678
-
696
+
679
697
  Examples:
680
698
  >>> q = QuestionCheckBox.example()
681
699
  >>> answer = q._simulate_answer(human_readable=False)
@@ -691,9 +709,9 @@ class QuestionCheckBox(QuestionBase):
691
709
  # Ensure we don't try to select more options than available
692
710
  max_sel = min(max_sel, len(self.question_options))
693
711
  min_sel = min(min_sel, max_sel)
694
-
712
+
695
713
  num_selections = random.randint(min_sel, max_sel)
696
-
714
+
697
715
  if human_readable:
698
716
  # Select a random number of options from self.question_options
699
717
  selected_options = random.sample(self.question_options, num_selections)
@@ -716,11 +734,11 @@ class QuestionCheckBox(QuestionBase):
716
734
  def question_html_content(self) -> str:
717
735
  """
718
736
  Generate HTML content for rendering the question in web interfaces.
719
-
737
+
720
738
  This property generates HTML markup for the question when it needs to be
721
739
  displayed in web interfaces or HTML contexts. For a checkbox question,
722
740
  this is a set of checkbox input elements, one for each option.
723
-
741
+
724
742
  Returns:
725
743
  str: HTML markup for rendering the question.
726
744
  """
@@ -729,7 +747,7 @@ class QuestionCheckBox(QuestionBase):
729
747
  instructions += f"Select at least {self.min_selections} option(s). "
730
748
  if self.max_selections is not None:
731
749
  instructions += f"Select at most {self.max_selections} option(s)."
732
-
750
+
733
751
  question_html_content = Template(
734
752
  """
735
753
  <p>{{ instructions }}</p>
@@ -752,20 +770,20 @@ class QuestionCheckBox(QuestionBase):
752
770
  ################
753
771
  @classmethod
754
772
  @inject_exception
755
- def example(cls, include_comment=False, use_code=True) -> QuestionCheckBox:
773
+ def example(cls, include_comment=False, use_code=False) -> QuestionCheckBox:
756
774
  """
757
775
  Create an example instance of a checkbox question.
758
-
776
+
759
777
  This class method creates a predefined example of a checkbox question
760
778
  for demonstration, testing, and documentation purposes.
761
-
779
+
762
780
  Args:
763
781
  include_comment: Whether to include a comment field with the answer.
764
782
  use_code: Whether to use indices (True) or values (False) for answer codes.
765
-
783
+
766
784
  Returns:
767
785
  QuestionCheckBox: An example checkbox question.
768
-
786
+
769
787
  Examples:
770
788
  >>> q = QuestionCheckBox.example()
771
789
  >>> q.question_name
@@ -797,11 +815,11 @@ class QuestionCheckBox(QuestionBase):
797
815
  def main():
798
816
  """
799
817
  Demonstrate the functionality of the QuestionCheckBox class.
800
-
818
+
801
819
  This function creates an example checkbox question and demonstrates its
802
820
  key features including validation, serialization, and answer simulation.
803
821
  It's primarily intended for testing and development purposes.
804
-
822
+
805
823
  Note:
806
824
  This function will be executed when the module is run directly,
807
825
  but not when imported.
@@ -813,47 +831,51 @@ def main():
813
831
  print(f"Question options: {q.question_options}")
814
832
  print(f"Min selections: {q.min_selections}")
815
833
  print(f"Max selections: {q.max_selections}")
816
-
834
+
817
835
  # Validate an answer
818
836
  print("\nValidating an answer...")
819
837
  valid_answer = {"answer": [1, 2], "comment": "I like these foods"}
820
838
  validated = q._validate_answer(valid_answer)
821
839
  print(f"Validated answer: {validated}")
822
-
840
+
823
841
  # Translate answer codes
824
842
  print("\nTranslating answer codes...")
825
843
  translated = q._translate_answer_code_to_answer([1, 2])
826
844
  print(f"Translated answer: {translated}")
827
-
845
+
828
846
  # Simulate answers
829
847
  print("\nSimulating answers...")
830
848
  simulated_human = q._simulate_answer(human_readable=True)
831
849
  print(f"Simulated human-readable answer: {simulated_human}")
832
-
850
+
833
851
  simulated_codes = q._simulate_answer(human_readable=False)
834
852
  print(f"Simulated code answer: {simulated_codes}")
835
-
853
+
836
854
  # Validate simulated answer
837
855
  validated_simulated = q._validate_answer(simulated_codes)
838
856
  print(f"Validated simulated answer: {validated_simulated}")
839
-
857
+
840
858
  # Serialization demonstration
841
859
  print("\nTesting serialization...")
842
860
  serialized = q.to_dict()
843
861
  print(f"Serialized question (keys): {list(serialized.keys())}")
844
862
  deserialized = QuestionBase.from_dict(serialized)
845
- print(f"Deserialization successful: {deserialized.question_text == q.question_text}")
846
-
863
+ print(
864
+ f"Deserialization successful: {deserialized.question_text == q.question_text}"
865
+ )
866
+
847
867
  # Run doctests
848
868
  print("\nRunning doctests...")
849
869
  import doctest
870
+
850
871
  doctest.testmod(optionflags=doctest.ELLIPSIS)
851
872
  print("Doctests completed")
852
873
 
853
874
 
854
875
  if __name__ == "__main__":
855
876
  import doctest
877
+
856
878
  doctest.testmod(optionflags=doctest.ELLIPSIS)
857
-
879
+
858
880
  # Uncomment to run demonstration
859
881
  # main()