edsl 0.1.53__py3-none-any.whl → 0.1.54__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__version__.py +1 -1
- edsl/inference_services/services/test_service.py +11 -2
- edsl/invigilators/invigilators.py +1 -1
- edsl/jobs/jobs_pricing_estimation.py +127 -46
- edsl/language_models/language_model.py +16 -6
- edsl/language_models/utilities.py +2 -1
- edsl/questions/question_check_box.py +171 -149
- edsl/questions/question_dict.py +47 -40
- {edsl-0.1.53.dist-info → edsl-0.1.54.dist-info}/METADATA +2 -1
- {edsl-0.1.53.dist-info → edsl-0.1.54.dist-info}/RECORD +13 -13
- {edsl-0.1.53.dist-info → edsl-0.1.54.dist-info}/LICENSE +0 -0
- {edsl-0.1.53.dist-info → edsl-0.1.54.dist-info}/WHEEL +0 -0
- {edsl-0.1.53.dist-info → edsl-0.1.54.dist-info}/entry_points.txt +0 -0
@@ -24,29 +24,29 @@ if TYPE_CHECKING:
|
|
24
24
|
class CheckboxResponse(BaseModel):
|
25
25
|
"""
|
26
26
|
Base Pydantic model for validating checkbox responses.
|
27
|
-
|
27
|
+
|
28
28
|
This model defines the structure and validation rules for responses to
|
29
29
|
checkbox questions, ensuring that selected options are properly formatted
|
30
30
|
as a list of choices.
|
31
|
-
|
31
|
+
|
32
32
|
Attributes:
|
33
33
|
answer: List of selected choices
|
34
34
|
comment: Optional comment provided with the answer
|
35
35
|
generated_tokens: Optional raw LLM output for token tracking
|
36
|
-
|
36
|
+
|
37
37
|
Examples:
|
38
38
|
>>> # Valid response with list of options
|
39
39
|
>>> response = CheckboxResponse(answer=[0, 1])
|
40
40
|
>>> response.answer
|
41
41
|
[0, 1]
|
42
|
-
|
42
|
+
|
43
43
|
>>> # Valid response with comment
|
44
44
|
>>> response = CheckboxResponse(answer=[1], comment="This is my choice")
|
45
45
|
>>> response.answer
|
46
46
|
[1]
|
47
47
|
>>> response.comment
|
48
48
|
'This is my choice'
|
49
|
-
|
49
|
+
|
50
50
|
>>> # Invalid non-list answer
|
51
51
|
>>> try:
|
52
52
|
... CheckboxResponse(answer=1)
|
@@ -54,6 +54,7 @@ class CheckboxResponse(BaseModel):
|
|
54
54
|
... print("Validation error occurred")
|
55
55
|
Validation error occurred
|
56
56
|
"""
|
57
|
+
|
57
58
|
answer: List[Any]
|
58
59
|
comment: Optional[str] = None
|
59
60
|
generated_tokens: Optional[Any] = None
|
@@ -67,59 +68,59 @@ def create_checkbox_response_model(
|
|
67
68
|
):
|
68
69
|
"""
|
69
70
|
Dynamically create a CheckboxResponse model with a predefined list of choices.
|
70
|
-
|
71
|
+
|
71
72
|
This function creates a customized Pydantic model for checkbox questions that
|
72
73
|
validates both the format of the response and any constraints on selection count.
|
73
|
-
|
74
|
+
|
74
75
|
Args:
|
75
76
|
choices: A list of allowed values for the answer field
|
76
77
|
min_selections: Optional minimum number of selections required
|
77
78
|
max_selections: Optional maximum number of selections allowed
|
78
79
|
permissive: If True, constraints are not enforced
|
79
|
-
|
80
|
+
|
80
81
|
Returns:
|
81
82
|
A new Pydantic model class with appropriate validation
|
82
|
-
|
83
|
+
|
83
84
|
Examples:
|
84
85
|
>>> # Create model with constraints
|
85
86
|
>>> choices = [0, 1, 2, 3]
|
86
87
|
>>> ConstrainedModel = create_checkbox_response_model(
|
87
|
-
... choices=choices,
|
88
|
-
... min_selections=1,
|
88
|
+
... choices=choices,
|
89
|
+
... min_selections=1,
|
89
90
|
... max_selections=2
|
90
91
|
... )
|
91
|
-
|
92
|
+
|
92
93
|
>>> # Valid response within constraints
|
93
94
|
>>> response = ConstrainedModel(answer=[0, 1])
|
94
95
|
>>> response.answer
|
95
96
|
[0, 1]
|
96
|
-
|
97
|
+
|
97
98
|
>>> # Too few selections fails validation
|
98
99
|
>>> try:
|
99
100
|
... ConstrainedModel(answer=[])
|
100
101
|
... except Exception as e:
|
101
102
|
... "at least 1" in str(e)
|
102
103
|
True
|
103
|
-
|
104
|
+
|
104
105
|
>>> # Too many selections fails validation
|
105
106
|
>>> try:
|
106
107
|
... ConstrainedModel(answer=[0, 1, 2])
|
107
108
|
... except Exception as e:
|
108
109
|
... "at most 2" in str(e)
|
109
110
|
True
|
110
|
-
|
111
|
+
|
111
112
|
>>> # Invalid choice fails validation
|
112
113
|
>>> try:
|
113
114
|
... ConstrainedModel(answer=[4])
|
114
115
|
... except Exception as e:
|
115
116
|
... any(x in str(e) for x in ["Invalid choice", "not a valid enumeration member", "validation error"])
|
116
117
|
True
|
117
|
-
|
118
|
+
|
118
119
|
>>> # Permissive model ignores constraints
|
119
120
|
>>> PermissiveModel = create_checkbox_response_model(
|
120
|
-
... choices=choices,
|
121
|
-
... min_selections=1,
|
122
|
-
... max_selections=2,
|
121
|
+
... choices=choices,
|
122
|
+
... min_selections=1,
|
123
|
+
... max_selections=2,
|
123
124
|
... permissive=True
|
124
125
|
... )
|
125
126
|
>>> response = PermissiveModel(answer=[0, 1, 2])
|
@@ -133,152 +134,161 @@ def create_checkbox_response_model(
|
|
133
134
|
# For permissive mode, we still validate the choice values but ignore count constraints
|
134
135
|
class PermissiveCheckboxResponse(CheckboxResponse):
|
135
136
|
"""Checkbox response model with choices validation but no count constraints."""
|
136
|
-
|
137
|
+
|
137
138
|
answer: Annotated[
|
138
139
|
List[Literal[choice_tuple]],
|
139
140
|
Field(description="List of selected choices"),
|
140
141
|
]
|
141
|
-
|
142
|
-
@model_validator(mode=
|
142
|
+
|
143
|
+
@model_validator(mode="after")
|
143
144
|
def validate_choices(self):
|
144
145
|
"""Validate that each selected choice is valid."""
|
145
146
|
for choice in self.answer:
|
146
147
|
if choice not in choices:
|
147
148
|
validation_error = ValidationError.from_exception_data(
|
148
|
-
title=
|
149
|
-
line_errors=[
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
149
|
+
title="CheckboxResponse",
|
150
|
+
line_errors=[
|
151
|
+
{
|
152
|
+
"type": "value_error",
|
153
|
+
"loc": ("answer",),
|
154
|
+
"msg": f"Invalid choice: {choice}. Must be one of: {choices}",
|
155
|
+
"input": choice,
|
156
|
+
"ctx": {"error": "Invalid choice"},
|
157
|
+
}
|
158
|
+
],
|
156
159
|
)
|
157
160
|
raise QuestionAnswerValidationError(
|
158
161
|
message=f"Invalid choice: {choice}. Must be one of: {choices}",
|
159
162
|
data=self.model_dump(),
|
160
163
|
model=self.__class__,
|
161
|
-
pydantic_error=validation_error
|
164
|
+
pydantic_error=validation_error,
|
162
165
|
)
|
163
166
|
return self
|
164
|
-
|
167
|
+
|
165
168
|
return PermissiveCheckboxResponse
|
166
169
|
else:
|
167
170
|
# For non-permissive mode, enforce both choice values and count constraints
|
168
171
|
class ConstrainedCheckboxResponse(CheckboxResponse):
|
169
172
|
"""Checkbox response model with both choice and count constraints."""
|
170
|
-
|
173
|
+
|
171
174
|
answer: Annotated[
|
172
175
|
List[Literal[choice_tuple]],
|
173
176
|
Field(description="List of selected choices"),
|
174
177
|
]
|
175
|
-
|
176
|
-
@model_validator(mode=
|
178
|
+
|
179
|
+
@model_validator(mode="after")
|
177
180
|
def validate_selection_count(self):
|
178
181
|
"""Validate that the number of selections meets constraints."""
|
179
182
|
if min_selections is not None and len(self.answer) < min_selections:
|
180
183
|
validation_error = ValidationError.from_exception_data(
|
181
|
-
title=
|
182
|
-
line_errors=[
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
184
|
+
title="CheckboxResponse",
|
185
|
+
line_errors=[
|
186
|
+
{
|
187
|
+
"type": "value_error",
|
188
|
+
"loc": ("answer",),
|
189
|
+
"msg": f"Must select at least {min_selections} option(s)",
|
190
|
+
"input": self.answer,
|
191
|
+
"ctx": {"error": "Too few selections"},
|
192
|
+
}
|
193
|
+
],
|
189
194
|
)
|
190
195
|
raise QuestionAnswerValidationError(
|
191
196
|
message=f"Must select at least {min_selections} option(s), got {len(self.answer)}",
|
192
197
|
data=self.model_dump(),
|
193
198
|
model=self.__class__,
|
194
|
-
pydantic_error=validation_error
|
199
|
+
pydantic_error=validation_error,
|
195
200
|
)
|
196
|
-
|
201
|
+
|
197
202
|
if max_selections is not None and len(self.answer) > max_selections:
|
198
203
|
validation_error = ValidationError.from_exception_data(
|
199
|
-
title=
|
200
|
-
line_errors=[
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
204
|
+
title="CheckboxResponse",
|
205
|
+
line_errors=[
|
206
|
+
{
|
207
|
+
"type": "value_error",
|
208
|
+
"loc": ("answer",),
|
209
|
+
"msg": f"Must select at most {max_selections} option(s)",
|
210
|
+
"input": self.answer,
|
211
|
+
"ctx": {"error": "Too many selections"},
|
212
|
+
}
|
213
|
+
],
|
207
214
|
)
|
208
215
|
raise QuestionAnswerValidationError(
|
209
216
|
message=f"Must select at most {max_selections} option(s), got {len(self.answer)}",
|
210
217
|
data=self.model_dump(),
|
211
218
|
model=self.__class__,
|
212
|
-
pydantic_error=validation_error
|
219
|
+
pydantic_error=validation_error,
|
213
220
|
)
|
214
|
-
|
221
|
+
|
215
222
|
# Also validate that each choice is valid
|
216
223
|
for choice in self.answer:
|
217
224
|
if choice not in choices:
|
218
225
|
validation_error = ValidationError.from_exception_data(
|
219
|
-
title=
|
220
|
-
line_errors=[
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
226
|
+
title="CheckboxResponse",
|
227
|
+
line_errors=[
|
228
|
+
{
|
229
|
+
"type": "value_error",
|
230
|
+
"loc": ("answer",),
|
231
|
+
"msg": f"Invalid choice: {choice}. Must be one of: {choices}",
|
232
|
+
"input": choice,
|
233
|
+
"ctx": {"error": "Invalid choice"},
|
234
|
+
}
|
235
|
+
],
|
227
236
|
)
|
228
237
|
raise QuestionAnswerValidationError(
|
229
238
|
message=f"Invalid choice: {choice}. Must be one of: {choices}",
|
230
239
|
data=self.model_dump(),
|
231
240
|
model=self.__class__,
|
232
|
-
pydantic_error=validation_error
|
241
|
+
pydantic_error=validation_error,
|
233
242
|
)
|
234
|
-
|
243
|
+
|
235
244
|
return self
|
236
|
-
|
245
|
+
|
237
246
|
return ConstrainedCheckboxResponse
|
238
247
|
|
239
248
|
|
240
249
|
class CheckBoxResponseValidator(ResponseValidatorABC):
|
241
250
|
"""
|
242
251
|
Validator for checkbox question responses.
|
243
|
-
|
252
|
+
|
244
253
|
This class implements the validation and fixing logic for checkbox responses.
|
245
254
|
It ensures that responses contain valid selections from the available options
|
246
255
|
and that the number of selections meets any constraints.
|
247
|
-
|
256
|
+
|
248
257
|
Attributes:
|
249
258
|
required_params: List of required parameters for validation.
|
250
259
|
valid_examples: Examples of valid responses for testing.
|
251
260
|
invalid_examples: Examples of invalid responses for testing.
|
252
|
-
|
261
|
+
|
253
262
|
Examples:
|
254
263
|
>>> from edsl import QuestionCheckBox
|
255
|
-
>>> q = QuestionCheckBox.example()
|
264
|
+
>>> q = QuestionCheckBox.example(use_code=True)
|
256
265
|
>>> validator = q.response_validator
|
257
|
-
|
266
|
+
|
258
267
|
>>> # Fix string to list
|
259
268
|
>>> response = {"answer": 1}
|
260
269
|
>>> fixed = validator.fix(response)
|
261
270
|
>>> isinstance(fixed["answer"], list)
|
262
271
|
True
|
263
|
-
|
272
|
+
|
264
273
|
>>> # Extract selections from text
|
265
274
|
>>> response = {"generated_tokens": "I choose options 0 and 2"}
|
266
275
|
>>> fixed = validator.fix(response)
|
267
276
|
>>> sorted(fixed["answer"])
|
268
277
|
[0, 2]
|
269
|
-
|
278
|
+
|
270
279
|
>>> # Fix comma-separated list
|
271
280
|
>>> response = {"generated_tokens": "0, 1, 3"}
|
272
281
|
>>> fixed = validator.fix(response)
|
273
282
|
>>> sorted(fixed["answer"])
|
274
283
|
[0, 1, 3]
|
275
|
-
|
284
|
+
|
276
285
|
>>> # Preserve comments when fixing
|
277
286
|
>>> response = {"answer": 1, "comment": "My explanation"}
|
278
287
|
>>> fixed = validator.fix(response)
|
279
288
|
>>> "comment" in fixed and fixed["comment"] == "My explanation"
|
280
289
|
True
|
281
290
|
"""
|
291
|
+
|
282
292
|
required_params = [
|
283
293
|
"question_options",
|
284
294
|
"min_selections",
|
@@ -316,20 +326,20 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
316
326
|
def fix(self, response, verbose=False):
|
317
327
|
"""
|
318
328
|
Fix common issues in checkbox responses.
|
319
|
-
|
329
|
+
|
320
330
|
This method attempts to extract valid selections from responses with
|
321
331
|
format issues. It can handle:
|
322
332
|
1. Single values that should be lists
|
323
333
|
2. Comma-separated strings in answer field or generated_tokens
|
324
334
|
3. Finding option indices mentioned in text
|
325
|
-
|
335
|
+
|
326
336
|
Args:
|
327
337
|
response: The response dictionary to fix
|
328
338
|
verbose: If True, print information about the fixing process
|
329
|
-
|
339
|
+
|
330
340
|
Returns:
|
331
341
|
A fixed version of the response dictionary with a valid list of selections
|
332
|
-
|
342
|
+
|
333
343
|
Notes:
|
334
344
|
- First tries to convert to a list if the answer is not already a list
|
335
345
|
- Then tries to parse comma-separated values from answer or generated_tokens
|
@@ -338,16 +348,20 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
338
348
|
"""
|
339
349
|
if verbose:
|
340
350
|
print("Invalid response of QuestionCheckBox was: ", response)
|
341
|
-
|
351
|
+
|
342
352
|
# Check if answer exists and is a comma-separated string (common LLM output format)
|
343
|
-
if
|
353
|
+
if (
|
354
|
+
"answer" in response
|
355
|
+
and isinstance(response["answer"], str)
|
356
|
+
and "," in response["answer"]
|
357
|
+
):
|
344
358
|
if verbose:
|
345
359
|
print(f"Parsing comma-separated answer string: {response['answer']}")
|
346
|
-
|
360
|
+
|
347
361
|
# Split by commas and strip whitespace
|
348
362
|
proposed_list = response["answer"].split(",")
|
349
363
|
proposed_list = [item.strip() for item in proposed_list]
|
350
|
-
|
364
|
+
|
351
365
|
# Try to convert to integers if use_code is True
|
352
366
|
if self.use_code:
|
353
367
|
try:
|
@@ -355,31 +369,33 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
355
369
|
except ValueError:
|
356
370
|
# If we can't convert to integers, try to match values to indices
|
357
371
|
if verbose:
|
358
|
-
print(
|
359
|
-
|
372
|
+
print(
|
373
|
+
"Could not convert comma-separated values to integers, trying to match options"
|
374
|
+
)
|
375
|
+
|
360
376
|
# Try to match option text values to their indices
|
361
377
|
index_map = {}
|
362
378
|
for i, option in enumerate(self.question_options):
|
363
379
|
index_map[option.lower().strip()] = i
|
364
|
-
|
380
|
+
|
365
381
|
converted_list = []
|
366
382
|
for item in proposed_list:
|
367
383
|
item_lower = item.lower().strip()
|
368
384
|
if item_lower in index_map:
|
369
385
|
converted_list.append(index_map[item_lower])
|
370
|
-
|
386
|
+
|
371
387
|
if converted_list:
|
372
388
|
proposed_list = converted_list
|
373
|
-
|
389
|
+
|
374
390
|
if verbose:
|
375
391
|
print("Proposed solution from comma separation is: ", proposed_list)
|
376
|
-
|
392
|
+
|
377
393
|
proposed_data = {
|
378
394
|
"answer": proposed_list,
|
379
395
|
"comment": response.get("comment"),
|
380
396
|
"generated_tokens": response.get("generated_tokens"),
|
381
397
|
}
|
382
|
-
|
398
|
+
|
383
399
|
# Try validating with the proposed solution
|
384
400
|
try:
|
385
401
|
validated = self._base_validate(proposed_data)
|
@@ -387,14 +403,14 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
387
403
|
except Exception as e:
|
388
404
|
if verbose:
|
389
405
|
print(f"Comma-separated solution invalid: {e}")
|
390
|
-
|
406
|
+
|
391
407
|
# If answer exists but is not a list, convert it to a list
|
392
408
|
elif "answer" in response and not isinstance(response["answer"], list):
|
393
409
|
if verbose:
|
394
410
|
print(f"Converting non-list answer {response['answer']} to a list")
|
395
411
|
answer_value = response["answer"]
|
396
412
|
response = {**response, "answer": [answer_value]}
|
397
|
-
|
413
|
+
|
398
414
|
# Try validating the fixed response
|
399
415
|
try:
|
400
416
|
validated = self._base_validate(response)
|
@@ -402,7 +418,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
402
418
|
except Exception:
|
403
419
|
if verbose:
|
404
420
|
print("Converting to list didn't fix the issue")
|
405
|
-
|
421
|
+
|
406
422
|
# Try parsing from generated_tokens if present
|
407
423
|
response_text = response.get("generated_tokens")
|
408
424
|
if response_text and isinstance(response_text, str):
|
@@ -410,38 +426,40 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
410
426
|
if "," in response_text:
|
411
427
|
proposed_list = response_text.split(",")
|
412
428
|
proposed_list = [item.strip() for item in proposed_list]
|
413
|
-
|
429
|
+
|
414
430
|
if self.use_code:
|
415
431
|
try:
|
416
432
|
proposed_list = [int(i) for i in proposed_list]
|
417
433
|
except ValueError:
|
418
434
|
# If we can't convert to integers, try to match values to indices
|
419
435
|
if verbose:
|
420
|
-
print(
|
421
|
-
|
436
|
+
print(
|
437
|
+
"Could not convert comma-separated values to integers, trying to match options"
|
438
|
+
)
|
439
|
+
|
422
440
|
# Try to match option text values to their indices
|
423
441
|
index_map = {}
|
424
442
|
for i, option in enumerate(self.question_options):
|
425
443
|
index_map[option.lower().strip()] = i
|
426
|
-
|
444
|
+
|
427
445
|
converted_list = []
|
428
446
|
for item in proposed_list:
|
429
447
|
item_lower = item.lower().strip()
|
430
448
|
if item_lower in index_map:
|
431
449
|
converted_list.append(index_map[item_lower])
|
432
|
-
|
450
|
+
|
433
451
|
if converted_list:
|
434
452
|
proposed_list = converted_list
|
435
|
-
|
453
|
+
|
436
454
|
if verbose:
|
437
455
|
print("Proposed solution from comma separation is: ", proposed_list)
|
438
|
-
|
456
|
+
|
439
457
|
proposed_data = {
|
440
458
|
"answer": proposed_list,
|
441
459
|
"comment": response.get("comment"),
|
442
460
|
"generated_tokens": response.get("generated_tokens"),
|
443
461
|
}
|
444
|
-
|
462
|
+
|
445
463
|
# Try validating with the proposed solution
|
446
464
|
try:
|
447
465
|
validated = self._base_validate(proposed_data)
|
@@ -449,7 +467,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
449
467
|
except Exception as e:
|
450
468
|
if verbose:
|
451
469
|
print(f"Comma-separated solution invalid: {e}")
|
452
|
-
|
470
|
+
|
453
471
|
# Try finding option indices mentioned in the text
|
454
472
|
matches = []
|
455
473
|
for index, option in enumerate(self.question_options):
|
@@ -459,17 +477,17 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
459
477
|
else:
|
460
478
|
if option in response_text:
|
461
479
|
matches.append(option)
|
462
|
-
|
480
|
+
|
463
481
|
if matches:
|
464
482
|
if verbose:
|
465
483
|
print(f"Found options mentioned in text: {matches}")
|
466
|
-
|
484
|
+
|
467
485
|
proposed_data = {
|
468
486
|
"answer": matches,
|
469
487
|
"comment": response.get("comment"),
|
470
488
|
"generated_tokens": response.get("generated_tokens"),
|
471
489
|
}
|
472
|
-
|
490
|
+
|
473
491
|
# Try validating with the proposed solution
|
474
492
|
try:
|
475
493
|
validated = self._base_validate(proposed_data)
|
@@ -477,7 +495,7 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
477
495
|
except Exception as e:
|
478
496
|
if verbose:
|
479
497
|
print(f"Text matching solution invalid: {e}")
|
480
|
-
|
498
|
+
|
481
499
|
# If nothing worked, return the original response
|
482
500
|
return response
|
483
501
|
|
@@ -485,11 +503,11 @@ class CheckBoxResponseValidator(ResponseValidatorABC):
|
|
485
503
|
class QuestionCheckBox(QuestionBase):
|
486
504
|
"""
|
487
505
|
A question that prompts the agent to select multiple options from a list.
|
488
|
-
|
506
|
+
|
489
507
|
QuestionCheckBox allows agents to select one or more items from a predefined
|
490
508
|
list of options. It's useful for "select all that apply" scenarios, multi-select
|
491
509
|
preferences, or any question where multiple valid selections are possible.
|
492
|
-
|
510
|
+
|
493
511
|
Attributes:
|
494
512
|
question_type (str): Identifier for this question type, set to "checkbox".
|
495
513
|
purpose (str): Brief description of when to use this question type.
|
@@ -498,16 +516,16 @@ class QuestionCheckBox(QuestionBase):
|
|
498
516
|
max_selections: Optional maximum number of selections allowed.
|
499
517
|
_response_model: Initially None, set by create_response_model().
|
500
518
|
response_validator_class: Class used to validate and fix responses.
|
501
|
-
|
519
|
+
|
502
520
|
Examples:
|
503
521
|
>>> # Basic creation works
|
504
522
|
>>> q = QuestionCheckBox.example()
|
505
523
|
>>> q.question_type
|
506
524
|
'checkbox'
|
507
|
-
|
525
|
+
|
508
526
|
>>> # Create preferences question with selection constraints
|
509
527
|
>>> q = QuestionCheckBox(
|
510
|
-
... question_name="favorite_fruits",
|
528
|
+
... question_name="favorite_fruits",
|
511
529
|
... question_text="Which fruits do you like?",
|
512
530
|
... question_options=["Apple", "Banana", "Cherry", "Durian", "Elderberry"],
|
513
531
|
... min_selections=1,
|
@@ -538,14 +556,14 @@ class QuestionCheckBox(QuestionBase):
|
|
538
556
|
min_selections: Optional[int] = None,
|
539
557
|
max_selections: Optional[int] = None,
|
540
558
|
include_comment: bool = True,
|
541
|
-
use_code: bool =
|
559
|
+
use_code: bool = False,
|
542
560
|
question_presentation: Optional[str] = None,
|
543
561
|
answering_instructions: Optional[str] = None,
|
544
562
|
permissive: bool = False,
|
545
563
|
):
|
546
564
|
"""
|
547
565
|
Initialize a new checkbox question.
|
548
|
-
|
566
|
+
|
549
567
|
Args:
|
550
568
|
question_name: Identifier for the question, used in results and templates.
|
551
569
|
Must be a valid Python variable name.
|
@@ -558,7 +576,7 @@ class QuestionCheckBox(QuestionBase):
|
|
558
576
|
question_presentation: Optional custom presentation template.
|
559
577
|
answering_instructions: Optional additional instructions.
|
560
578
|
permissive: If True, ignore selection count constraints during validation.
|
561
|
-
|
579
|
+
|
562
580
|
Examples:
|
563
581
|
>>> q = QuestionCheckBox(
|
564
582
|
... question_name="symptoms",
|
@@ -568,12 +586,12 @@ class QuestionCheckBox(QuestionBase):
|
|
568
586
|
... )
|
569
587
|
>>> q.question_name
|
570
588
|
'symptoms'
|
571
|
-
|
589
|
+
|
572
590
|
>>> # Question with both min and max
|
573
591
|
>>> q = QuestionCheckBox(
|
574
592
|
... question_name="pizza_toppings",
|
575
593
|
... question_text="Choose 2-4 toppings for your pizza:",
|
576
|
-
... question_options=["Cheese", "Pepperoni", "Mushroom", "Onion",
|
594
|
+
... question_options=["Cheese", "Pepperoni", "Mushroom", "Onion",
|
577
595
|
... "Sausage", "Bacon", "Pineapple"],
|
578
596
|
... min_selections=2,
|
579
597
|
... max_selections=4
|
@@ -595,15 +613,15 @@ class QuestionCheckBox(QuestionBase):
|
|
595
613
|
def create_response_model(self):
|
596
614
|
"""
|
597
615
|
Create a response model with the appropriate constraints.
|
598
|
-
|
616
|
+
|
599
617
|
This method creates a Pydantic model customized with the options and
|
600
618
|
selection count constraints specified for this question instance.
|
601
|
-
|
619
|
+
|
602
620
|
Returns:
|
603
621
|
A Pydantic model class tailored to this question's constraints.
|
604
|
-
|
622
|
+
|
605
623
|
Examples:
|
606
|
-
>>> q = QuestionCheckBox.example()
|
624
|
+
>>> q = QuestionCheckBox.example(use_code=True)
|
607
625
|
>>> model = q.create_response_model()
|
608
626
|
>>> model(answer=[0, 2]) # Select first and third options
|
609
627
|
ConstrainedCheckboxResponse(answer=[0, 2], comment=None, generated_tokens=None)
|
@@ -630,23 +648,23 @@ class QuestionCheckBox(QuestionBase):
|
|
630
648
|
):
|
631
649
|
"""
|
632
650
|
Translate the answer codes to the actual answer text.
|
633
|
-
|
651
|
+
|
634
652
|
For checkbox questions with use_code=True, the agent responds with
|
635
653
|
option indices (e.g., [0, 1]) which need to be translated to their
|
636
654
|
corresponding option text values (e.g., ["Option A", "Option B"]).
|
637
|
-
|
655
|
+
|
638
656
|
Args:
|
639
657
|
answer_codes: List of selected option indices or values
|
640
658
|
scenario: Optional scenario with variables for template rendering
|
641
|
-
|
659
|
+
|
642
660
|
Returns:
|
643
661
|
List of selected option texts
|
644
|
-
|
662
|
+
|
645
663
|
Examples:
|
646
664
|
>>> q = QuestionCheckBox(
|
647
665
|
... question_name="example",
|
648
666
|
... question_text="Select options:",
|
649
|
-
... question_options=["A", "B", "C"]
|
667
|
+
... question_options=["A", "B", "C"],use_code=True
|
650
668
|
... )
|
651
669
|
>>> q._translate_answer_code_to_answer([0, 2])
|
652
670
|
['A', 'C']
|
@@ -666,16 +684,16 @@ class QuestionCheckBox(QuestionBase):
|
|
666
684
|
def _simulate_answer(self, human_readable=True):
|
667
685
|
"""
|
668
686
|
Simulate a valid answer for debugging purposes.
|
669
|
-
|
687
|
+
|
670
688
|
This method generates a random valid answer for the checkbox question,
|
671
689
|
useful for testing and demonstrations.
|
672
|
-
|
690
|
+
|
673
691
|
Args:
|
674
692
|
human_readable: If True, return option text values; if False, return indices
|
675
|
-
|
693
|
+
|
676
694
|
Returns:
|
677
695
|
A dictionary with a valid random answer
|
678
|
-
|
696
|
+
|
679
697
|
Examples:
|
680
698
|
>>> q = QuestionCheckBox.example()
|
681
699
|
>>> answer = q._simulate_answer(human_readable=False)
|
@@ -691,9 +709,9 @@ class QuestionCheckBox(QuestionBase):
|
|
691
709
|
# Ensure we don't try to select more options than available
|
692
710
|
max_sel = min(max_sel, len(self.question_options))
|
693
711
|
min_sel = min(min_sel, max_sel)
|
694
|
-
|
712
|
+
|
695
713
|
num_selections = random.randint(min_sel, max_sel)
|
696
|
-
|
714
|
+
|
697
715
|
if human_readable:
|
698
716
|
# Select a random number of options from self.question_options
|
699
717
|
selected_options = random.sample(self.question_options, num_selections)
|
@@ -716,11 +734,11 @@ class QuestionCheckBox(QuestionBase):
|
|
716
734
|
def question_html_content(self) -> str:
|
717
735
|
"""
|
718
736
|
Generate HTML content for rendering the question in web interfaces.
|
719
|
-
|
737
|
+
|
720
738
|
This property generates HTML markup for the question when it needs to be
|
721
739
|
displayed in web interfaces or HTML contexts. For a checkbox question,
|
722
740
|
this is a set of checkbox input elements, one for each option.
|
723
|
-
|
741
|
+
|
724
742
|
Returns:
|
725
743
|
str: HTML markup for rendering the question.
|
726
744
|
"""
|
@@ -729,7 +747,7 @@ class QuestionCheckBox(QuestionBase):
|
|
729
747
|
instructions += f"Select at least {self.min_selections} option(s). "
|
730
748
|
if self.max_selections is not None:
|
731
749
|
instructions += f"Select at most {self.max_selections} option(s)."
|
732
|
-
|
750
|
+
|
733
751
|
question_html_content = Template(
|
734
752
|
"""
|
735
753
|
<p>{{ instructions }}</p>
|
@@ -752,20 +770,20 @@ class QuestionCheckBox(QuestionBase):
|
|
752
770
|
################
|
753
771
|
@classmethod
|
754
772
|
@inject_exception
|
755
|
-
def example(cls, include_comment=False, use_code=
|
773
|
+
def example(cls, include_comment=False, use_code=False) -> QuestionCheckBox:
|
756
774
|
"""
|
757
775
|
Create an example instance of a checkbox question.
|
758
|
-
|
776
|
+
|
759
777
|
This class method creates a predefined example of a checkbox question
|
760
778
|
for demonstration, testing, and documentation purposes.
|
761
|
-
|
779
|
+
|
762
780
|
Args:
|
763
781
|
include_comment: Whether to include a comment field with the answer.
|
764
782
|
use_code: Whether to use indices (True) or values (False) for answer codes.
|
765
|
-
|
783
|
+
|
766
784
|
Returns:
|
767
785
|
QuestionCheckBox: An example checkbox question.
|
768
|
-
|
786
|
+
|
769
787
|
Examples:
|
770
788
|
>>> q = QuestionCheckBox.example()
|
771
789
|
>>> q.question_name
|
@@ -797,11 +815,11 @@ class QuestionCheckBox(QuestionBase):
|
|
797
815
|
def main():
|
798
816
|
"""
|
799
817
|
Demonstrate the functionality of the QuestionCheckBox class.
|
800
|
-
|
818
|
+
|
801
819
|
This function creates an example checkbox question and demonstrates its
|
802
820
|
key features including validation, serialization, and answer simulation.
|
803
821
|
It's primarily intended for testing and development purposes.
|
804
|
-
|
822
|
+
|
805
823
|
Note:
|
806
824
|
This function will be executed when the module is run directly,
|
807
825
|
but not when imported.
|
@@ -813,47 +831,51 @@ def main():
|
|
813
831
|
print(f"Question options: {q.question_options}")
|
814
832
|
print(f"Min selections: {q.min_selections}")
|
815
833
|
print(f"Max selections: {q.max_selections}")
|
816
|
-
|
834
|
+
|
817
835
|
# Validate an answer
|
818
836
|
print("\nValidating an answer...")
|
819
837
|
valid_answer = {"answer": [1, 2], "comment": "I like these foods"}
|
820
838
|
validated = q._validate_answer(valid_answer)
|
821
839
|
print(f"Validated answer: {validated}")
|
822
|
-
|
840
|
+
|
823
841
|
# Translate answer codes
|
824
842
|
print("\nTranslating answer codes...")
|
825
843
|
translated = q._translate_answer_code_to_answer([1, 2])
|
826
844
|
print(f"Translated answer: {translated}")
|
827
|
-
|
845
|
+
|
828
846
|
# Simulate answers
|
829
847
|
print("\nSimulating answers...")
|
830
848
|
simulated_human = q._simulate_answer(human_readable=True)
|
831
849
|
print(f"Simulated human-readable answer: {simulated_human}")
|
832
|
-
|
850
|
+
|
833
851
|
simulated_codes = q._simulate_answer(human_readable=False)
|
834
852
|
print(f"Simulated code answer: {simulated_codes}")
|
835
|
-
|
853
|
+
|
836
854
|
# Validate simulated answer
|
837
855
|
validated_simulated = q._validate_answer(simulated_codes)
|
838
856
|
print(f"Validated simulated answer: {validated_simulated}")
|
839
|
-
|
857
|
+
|
840
858
|
# Serialization demonstration
|
841
859
|
print("\nTesting serialization...")
|
842
860
|
serialized = q.to_dict()
|
843
861
|
print(f"Serialized question (keys): {list(serialized.keys())}")
|
844
862
|
deserialized = QuestionBase.from_dict(serialized)
|
845
|
-
print(
|
846
|
-
|
863
|
+
print(
|
864
|
+
f"Deserialization successful: {deserialized.question_text == q.question_text}"
|
865
|
+
)
|
866
|
+
|
847
867
|
# Run doctests
|
848
868
|
print("\nRunning doctests...")
|
849
869
|
import doctest
|
870
|
+
|
850
871
|
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
851
872
|
print("Doctests completed")
|
852
873
|
|
853
874
|
|
854
875
|
if __name__ == "__main__":
|
855
876
|
import doctest
|
877
|
+
|
856
878
|
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
857
|
-
|
879
|
+
|
858
880
|
# Uncomment to run demonstration
|
859
881
|
# main()
|