edsl 0.1.30.dev4__py3-none-any.whl → 0.1.31__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__version__.py +1 -1
- edsl/agents/Invigilator.py +7 -2
- edsl/agents/PromptConstructionMixin.py +18 -1
- edsl/config.py +4 -0
- edsl/conjure/Conjure.py +6 -0
- edsl/coop/coop.py +4 -0
- edsl/coop/utils.py +9 -1
- edsl/data/CacheHandler.py +3 -4
- edsl/enums.py +2 -0
- edsl/inference_services/DeepInfraService.py +6 -91
- edsl/inference_services/GroqService.py +18 -0
- edsl/inference_services/InferenceServicesCollection.py +13 -5
- edsl/inference_services/OpenAIService.py +64 -21
- edsl/inference_services/registry.py +2 -1
- edsl/jobs/Jobs.py +80 -33
- edsl/jobs/buckets/TokenBucket.py +24 -5
- edsl/jobs/interviews/Interview.py +122 -75
- edsl/jobs/interviews/InterviewExceptionEntry.py +101 -0
- edsl/jobs/interviews/InterviewTaskBuildingMixin.py +58 -52
- edsl/jobs/interviews/interview_exception_tracking.py +68 -10
- edsl/jobs/runners/JobsRunnerAsyncio.py +112 -81
- edsl/jobs/runners/JobsRunnerStatusData.py +0 -237
- edsl/jobs/runners/JobsRunnerStatusMixin.py +291 -35
- edsl/jobs/tasks/QuestionTaskCreator.py +1 -5
- edsl/jobs/tasks/TaskCreators.py +8 -2
- edsl/jobs/tasks/TaskHistory.py +145 -1
- edsl/language_models/LanguageModel.py +135 -75
- edsl/language_models/ModelList.py +8 -2
- edsl/language_models/registry.py +16 -0
- edsl/questions/QuestionFunctional.py +34 -2
- edsl/questions/QuestionMultipleChoice.py +58 -8
- edsl/questions/QuestionNumerical.py +0 -1
- edsl/questions/descriptors.py +42 -2
- edsl/results/DatasetExportMixin.py +258 -75
- edsl/results/Result.py +53 -5
- edsl/results/Results.py +66 -27
- edsl/results/ResultsToolsMixin.py +1 -1
- edsl/scenarios/Scenario.py +14 -0
- edsl/scenarios/ScenarioList.py +59 -21
- edsl/scenarios/ScenarioListExportMixin.py +16 -5
- edsl/scenarios/ScenarioListPdfMixin.py +3 -0
- edsl/study/Study.py +2 -2
- edsl/surveys/Survey.py +35 -1
- {edsl-0.1.30.dev4.dist-info → edsl-0.1.31.dist-info}/METADATA +4 -2
- {edsl-0.1.30.dev4.dist-info → edsl-0.1.31.dist-info}/RECORD +47 -45
- {edsl-0.1.30.dev4.dist-info → edsl-0.1.31.dist-info}/WHEEL +1 -1
- {edsl-0.1.30.dev4.dist-info → edsl-0.1.31.dist-info}/LICENSE +0 -0
@@ -2,7 +2,7 @@ from __future__ import annotations
|
|
2
2
|
import time
|
3
3
|
from typing import Union
|
4
4
|
import random
|
5
|
-
|
5
|
+
from typing import Optional
|
6
6
|
from jinja2 import Template
|
7
7
|
|
8
8
|
from edsl.questions.QuestionBase import QuestionBase
|
@@ -10,7 +10,11 @@ from edsl.questions.descriptors import QuestionOptionsDescriptor
|
|
10
10
|
|
11
11
|
|
12
12
|
class QuestionMultipleChoice(QuestionBase):
|
13
|
-
"""This question prompts the agent to select one option from a list of options.
|
13
|
+
"""This question prompts the agent to select one option from a list of options.
|
14
|
+
|
15
|
+
https://docs.expectedparrot.com/en/latest/questions.html#questionmultiplechoice-class
|
16
|
+
|
17
|
+
"""
|
14
18
|
|
15
19
|
question_type = "multiple_choice"
|
16
20
|
purpose = "When options are known and limited"
|
@@ -35,27 +39,72 @@ class QuestionMultipleChoice(QuestionBase):
|
|
35
39
|
self.question_text = question_text
|
36
40
|
self.question_options = question_options
|
37
41
|
|
42
|
+
# @property
|
43
|
+
# def question_options(self) -> Union[list[str], list[list], list[float], list[int]]:
|
44
|
+
# """Return the question options."""
|
45
|
+
# return self._question_options
|
46
|
+
|
38
47
|
################
|
39
48
|
# Answer methods
|
40
49
|
################
|
41
50
|
def _validate_answer(
|
42
51
|
self, answer: dict[str, Union[str, int]]
|
43
52
|
) -> dict[str, Union[str, int]]:
|
44
|
-
"""Validate the answer.
|
53
|
+
"""Validate the answer.
|
54
|
+
|
55
|
+
>>> q = QuestionMultipleChoice.example()
|
56
|
+
>>> q._validate_answer({"answer": 0, "comment": "I like custard"})
|
57
|
+
{'answer': 0, 'comment': 'I like custard'}
|
58
|
+
|
59
|
+
>>> q = QuestionMultipleChoice(question_name="how_feeling", question_text="How are you?", question_options=["Good", "Great", "OK", "Bad"])
|
60
|
+
>>> q._validate_answer({"answer": -1, "comment": "I like custard"})
|
61
|
+
Traceback (most recent call last):
|
62
|
+
...
|
63
|
+
edsl.exceptions.questions.QuestionAnswerValidationError: Answer code must be a non-negative integer (got -1).
|
64
|
+
"""
|
45
65
|
self._validate_answer_template_basic(answer)
|
46
66
|
self._validate_answer_multiple_choice(answer)
|
47
67
|
return answer
|
48
68
|
|
49
69
|
def _translate_answer_code_to_answer(
|
50
|
-
self, answer_code, scenario: "Scenario" = None
|
70
|
+
self, answer_code: int, scenario: Optional["Scenario"] = None
|
51
71
|
):
|
52
|
-
"""Translate the answer code to the actual answer.
|
72
|
+
"""Translate the answer code to the actual answer.
|
73
|
+
|
74
|
+
It is used to translate the answer code to the actual answer.
|
75
|
+
The question options might be templates, so they need to be rendered with the scenario.
|
76
|
+
|
77
|
+
>>> q = QuestionMultipleChoice.example()
|
78
|
+
>>> q._translate_answer_code_to_answer(0, {})
|
79
|
+
'Good'
|
80
|
+
|
81
|
+
>>> q = QuestionMultipleChoice(question_name="how_feeling", question_text="How are you?", question_options=["{{emotion[0]}}", "emotion[1]"])
|
82
|
+
>>> q._translate_answer_code_to_answer(0, {"emotion": ["Happy", "Sad"]})
|
83
|
+
'Happy'
|
84
|
+
|
85
|
+
"""
|
53
86
|
from edsl.scenarios.Scenario import Scenario
|
54
87
|
|
55
88
|
scenario = scenario or Scenario()
|
56
|
-
|
57
|
-
|
58
|
-
|
89
|
+
|
90
|
+
if isinstance(self.question_options, str):
|
91
|
+
# If dynamic options are provided like {{ options }}, render them with the scenario
|
92
|
+
from jinja2 import Environment, meta
|
93
|
+
|
94
|
+
env = Environment()
|
95
|
+
parsed_content = env.parse(self.question_options)
|
96
|
+
question_option_key = list(meta.find_undeclared_variables(parsed_content))[
|
97
|
+
0
|
98
|
+
]
|
99
|
+
# breakpoint()
|
100
|
+
translated_options = scenario.get(question_option_key)
|
101
|
+
else:
|
102
|
+
translated_options = [
|
103
|
+
Template(str(option)).render(scenario)
|
104
|
+
for option in self.question_options
|
105
|
+
]
|
106
|
+
# print("Translated options:", translated_options)
|
107
|
+
# breakpoint()
|
59
108
|
return translated_options[int(answer_code)]
|
60
109
|
|
61
110
|
def _simulate_answer(
|
@@ -75,6 +124,7 @@ class QuestionMultipleChoice(QuestionBase):
|
|
75
124
|
|
76
125
|
@property
|
77
126
|
def question_html_content(self) -> str:
|
127
|
+
"""Return the HTML version of the question."""
|
78
128
|
if hasattr(self, "option_labels"):
|
79
129
|
option_labels = self.option_labels
|
80
130
|
else:
|
@@ -26,7 +26,6 @@ class QuestionNumerical(QuestionBase):
|
|
26
26
|
|
27
27
|
:param question_name: The name of the question.
|
28
28
|
:param question_text: The text of the question.
|
29
|
-
:param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionNumerical.default_instructions`.
|
30
29
|
:param min_value: The minimum value of the answer.
|
31
30
|
:param max_value: The maximum value of the answer.
|
32
31
|
"""
|
edsl/questions/descriptors.py
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
from abc import ABC, abstractmethod
|
4
4
|
import re
|
5
|
-
from typing import Any, Callable
|
5
|
+
from typing import Any, Callable, List, Optional
|
6
6
|
from edsl.exceptions import (
|
7
7
|
QuestionCreationValidationError,
|
8
8
|
QuestionAnswerValidationError,
|
@@ -242,6 +242,16 @@ class QuestionNameDescriptor(BaseDescriptor):
|
|
242
242
|
class QuestionOptionsDescriptor(BaseDescriptor):
|
243
243
|
"""Validate that `question_options` is a list, does not exceed the min/max lengths, and has unique items."""
|
244
244
|
|
245
|
+
@classmethod
|
246
|
+
def example(cls):
|
247
|
+
class TestQuestion:
|
248
|
+
question_options = QuestionOptionsDescriptor()
|
249
|
+
|
250
|
+
def __init__(self, question_options: List[str]):
|
251
|
+
self.question_options = question_options
|
252
|
+
|
253
|
+
return TestQuestion
|
254
|
+
|
245
255
|
def __init__(
|
246
256
|
self,
|
247
257
|
num_choices: int = None,
|
@@ -254,7 +264,31 @@ class QuestionOptionsDescriptor(BaseDescriptor):
|
|
254
264
|
self.q_budget = q_budget
|
255
265
|
|
256
266
|
def validate(self, value: Any, instance) -> None:
|
257
|
-
"""Validate the question options.
|
267
|
+
"""Validate the question options.
|
268
|
+
|
269
|
+
>>> q_class = QuestionOptionsDescriptor.example()
|
270
|
+
>>> _ = q_class(["a", "b", "c"])
|
271
|
+
>>> _ = q_class(["a", "b", "c", "d", "d"])
|
272
|
+
Traceback (most recent call last):
|
273
|
+
...
|
274
|
+
edsl.exceptions.questions.QuestionCreationValidationError: Question options must be unique (got ['a', 'b', 'c', 'd', 'd']).
|
275
|
+
|
276
|
+
We allow dynamic question options, which are strings of the form '{{ question_options }}'.
|
277
|
+
|
278
|
+
>>> _ = q_class("{{dynamic_options}}")
|
279
|
+
>>> _ = q_class("dynamic_options")
|
280
|
+
Traceback (most recent call last):
|
281
|
+
...
|
282
|
+
edsl.exceptions.questions.QuestionCreationValidationError: Dynamic question options must be of the form: '{{ question_options }}'.
|
283
|
+
"""
|
284
|
+
if isinstance(value, str):
|
285
|
+
# Check if the string is a dynamic question option
|
286
|
+
if "{{" in value and "}}" in value:
|
287
|
+
return None
|
288
|
+
else:
|
289
|
+
raise QuestionCreationValidationError(
|
290
|
+
"Dynamic question options must be of the form: '{{ question_options }}'."
|
291
|
+
)
|
258
292
|
if not isinstance(value, list):
|
259
293
|
raise QuestionCreationValidationError(
|
260
294
|
f"Question options must be a list (got {value})."
|
@@ -339,3 +373,9 @@ class QuestionTextDescriptor(BaseDescriptor):
|
|
339
373
|
f"WARNING: Question text contains a single-braced substring: If you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html",
|
340
374
|
UserWarning,
|
341
375
|
)
|
376
|
+
|
377
|
+
|
378
|
+
if __name__ == "__main__":
|
379
|
+
import doctest
|
380
|
+
|
381
|
+
doctest.testmod(optionflags=doctest.ELLIPSIS)
|