edsl 0.1.33.dev2__py3-none-any.whl → 0.1.34__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +24 -14
- edsl/__init__.py +1 -0
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +6 -6
- edsl/agents/Invigilator.py +28 -6
- edsl/agents/InvigilatorBase.py +8 -27
- edsl/agents/{PromptConstructionMixin.py → PromptConstructor.py} +150 -182
- edsl/agents/prompt_helpers.py +129 -0
- edsl/config.py +26 -34
- edsl/coop/coop.py +14 -4
- edsl/data_transfer_models.py +26 -73
- edsl/enums.py +2 -0
- edsl/inference_services/AnthropicService.py +5 -2
- edsl/inference_services/AwsBedrock.py +5 -2
- edsl/inference_services/AzureAI.py +5 -2
- edsl/inference_services/GoogleService.py +108 -33
- edsl/inference_services/InferenceServiceABC.py +44 -13
- edsl/inference_services/MistralAIService.py +5 -2
- edsl/inference_services/OpenAIService.py +10 -6
- edsl/inference_services/TestService.py +34 -16
- edsl/inference_services/TogetherAIService.py +170 -0
- edsl/inference_services/registry.py +2 -0
- edsl/jobs/Jobs.py +109 -18
- edsl/jobs/buckets/BucketCollection.py +24 -15
- edsl/jobs/buckets/TokenBucket.py +64 -10
- edsl/jobs/interviews/Interview.py +130 -49
- edsl/jobs/interviews/{interview_exception_tracking.py → InterviewExceptionCollection.py} +16 -0
- edsl/jobs/interviews/InterviewExceptionEntry.py +2 -0
- edsl/jobs/runners/JobsRunnerAsyncio.py +119 -173
- edsl/jobs/runners/JobsRunnerStatus.py +332 -0
- edsl/jobs/tasks/QuestionTaskCreator.py +1 -13
- edsl/jobs/tasks/TaskHistory.py +17 -0
- edsl/language_models/LanguageModel.py +36 -38
- edsl/language_models/registry.py +13 -9
- edsl/language_models/utilities.py +5 -2
- edsl/questions/QuestionBase.py +74 -16
- edsl/questions/QuestionBaseGenMixin.py +28 -0
- edsl/questions/QuestionBudget.py +93 -41
- edsl/questions/QuestionCheckBox.py +1 -1
- edsl/questions/QuestionFreeText.py +6 -0
- edsl/questions/QuestionMultipleChoice.py +13 -24
- edsl/questions/QuestionNumerical.py +5 -4
- edsl/questions/Quick.py +41 -0
- edsl/questions/ResponseValidatorABC.py +11 -6
- edsl/questions/derived/QuestionLinearScale.py +4 -1
- edsl/questions/derived/QuestionTopK.py +4 -1
- edsl/questions/derived/QuestionYesNo.py +8 -2
- edsl/questions/descriptors.py +12 -11
- edsl/questions/templates/budget/__init__.py +0 -0
- edsl/questions/templates/budget/answering_instructions.jinja +7 -0
- edsl/questions/templates/budget/question_presentation.jinja +7 -0
- edsl/questions/templates/extract/__init__.py +0 -0
- edsl/questions/templates/numerical/answering_instructions.jinja +0 -1
- edsl/questions/templates/rank/__init__.py +0 -0
- edsl/questions/templates/yes_no/answering_instructions.jinja +2 -2
- edsl/results/DatasetExportMixin.py +5 -1
- edsl/results/Result.py +1 -1
- edsl/results/Results.py +4 -1
- edsl/scenarios/FileStore.py +178 -34
- edsl/scenarios/Scenario.py +76 -37
- edsl/scenarios/ScenarioList.py +19 -2
- edsl/scenarios/ScenarioListPdfMixin.py +150 -4
- edsl/study/Study.py +32 -0
- edsl/surveys/DAG.py +62 -0
- edsl/surveys/MemoryPlan.py +26 -0
- edsl/surveys/Rule.py +34 -1
- edsl/surveys/RuleCollection.py +55 -5
- edsl/surveys/Survey.py +189 -10
- edsl/surveys/base.py +4 -0
- edsl/templates/error_reporting/interview_details.html +6 -1
- edsl/utilities/utilities.py +9 -1
- {edsl-0.1.33.dev2.dist-info → edsl-0.1.34.dist-info}/METADATA +3 -1
- {edsl-0.1.33.dev2.dist-info → edsl-0.1.34.dist-info}/RECORD +75 -69
- edsl/jobs/interviews/retry_management.py +0 -39
- edsl/jobs/runners/JobsRunnerStatusMixin.py +0 -333
- edsl/scenarios/ScenarioImageMixin.py +0 -100
- {edsl-0.1.33.dev2.dist-info → edsl-0.1.34.dist-info}/LICENSE +0 -0
- {edsl-0.1.33.dev2.dist-info → edsl-0.1.34.dist-info}/WHEEL +0 -0
edsl/Base.py
CHANGED
@@ -47,21 +47,27 @@ class PersistenceMixin:
|
|
47
47
|
self,
|
48
48
|
description: Optional[str] = None,
|
49
49
|
visibility: Optional[str] = "unlisted",
|
50
|
+
expected_parrot_url: Optional[str] = None,
|
50
51
|
):
|
51
52
|
"""Post the object to coop."""
|
52
53
|
from edsl.coop import Coop
|
53
54
|
|
54
|
-
c = Coop()
|
55
|
+
c = Coop(url=expected_parrot_url)
|
55
56
|
return c.create(self, description, visibility)
|
56
57
|
|
57
58
|
@classmethod
|
58
|
-
def pull(
|
59
|
+
def pull(
|
60
|
+
cls,
|
61
|
+
uuid: Optional[Union[str, UUID]] = None,
|
62
|
+
url: Optional[str] = None,
|
63
|
+
expected_parrot_url: Optional[str] = None,
|
64
|
+
):
|
59
65
|
"""Pull the object from coop."""
|
60
66
|
from edsl.coop import Coop
|
61
67
|
from edsl.coop.utils import ObjectRegistry
|
62
68
|
|
63
69
|
object_type = ObjectRegistry.get_object_type_by_edsl_class(cls)
|
64
|
-
coop = Coop()
|
70
|
+
coop = Coop(url=expected_parrot_url)
|
65
71
|
return coop.get(uuid, url, object_type)
|
66
72
|
|
67
73
|
@classmethod
|
@@ -109,23 +115,27 @@ class PersistenceMixin:
|
|
109
115
|
if filename.endswith("json.gz"):
|
110
116
|
import warnings
|
111
117
|
|
112
|
-
warnings.warn(
|
113
|
-
|
114
|
-
)
|
118
|
+
# warnings.warn(
|
119
|
+
# "Do not apply the file extensions. The filename should not end with 'json.gz'."
|
120
|
+
# )
|
115
121
|
filename = filename[:-7]
|
116
122
|
if filename.endswith("json"):
|
117
123
|
filename = filename[:-4]
|
118
|
-
warnings.warn(
|
119
|
-
|
120
|
-
)
|
124
|
+
# warnings.warn(
|
125
|
+
# "Do not apply the file extensions. The filename should not end with 'json'."
|
126
|
+
# )
|
121
127
|
|
122
128
|
if compress:
|
123
|
-
|
129
|
+
full_file_name = filename + ".json.gz"
|
130
|
+
with gzip.open(full_file_name, "wb") as f:
|
124
131
|
f.write(json.dumps(self.to_dict()).encode("utf-8"))
|
125
132
|
else:
|
133
|
+
full_file_name = filename + ".json"
|
126
134
|
with open(filename + ".json", "w") as f:
|
127
135
|
f.write(json.dumps(self.to_dict()))
|
128
136
|
|
137
|
+
print("Saved to", full_file_name)
|
138
|
+
|
129
139
|
@staticmethod
|
130
140
|
def open_compressed_file(filename):
|
131
141
|
with gzip.open(filename, "rb") as f:
|
@@ -154,11 +164,11 @@ class PersistenceMixin:
|
|
154
164
|
d = cls.open_regular_file(filename)
|
155
165
|
else:
|
156
166
|
try:
|
157
|
-
d = cls.open_compressed_file(filename)
|
167
|
+
d = cls.open_compressed_file(filename + ".json.gz")
|
158
168
|
except:
|
159
|
-
d = cls.open_regular_file(filename)
|
160
|
-
finally:
|
161
|
-
|
169
|
+
d = cls.open_regular_file(filename + ".json")
|
170
|
+
# finally:
|
171
|
+
# raise ValueError("File must be a json or json.gz file")
|
162
172
|
|
163
173
|
return cls.from_dict(d)
|
164
174
|
|
edsl/__init__.py
CHANGED
@@ -23,6 +23,7 @@ from edsl.questions import QuestionNumerical
|
|
23
23
|
from edsl.questions import QuestionYesNo
|
24
24
|
from edsl.questions import QuestionBudget
|
25
25
|
from edsl.questions import QuestionRank
|
26
|
+
from edsl.questions import QuestionTopK
|
26
27
|
|
27
28
|
from edsl.scenarios import Scenario
|
28
29
|
from edsl.scenarios import ScenarioList
|
edsl/__version__.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "0.1.
|
1
|
+
__version__ = "0.1.34"
|
edsl/agents/Agent.py
CHANGED
@@ -586,9 +586,9 @@ class Agent(Base):
|
|
586
586
|
if dynamic_traits_func:
|
587
587
|
func = inspect.getsource(dynamic_traits_func)
|
588
588
|
raw_data["dynamic_traits_function_source_code"] = func
|
589
|
-
raw_data[
|
590
|
-
|
591
|
-
|
589
|
+
raw_data[
|
590
|
+
"dynamic_traits_function_name"
|
591
|
+
] = self.dynamic_traits_function_name
|
592
592
|
if hasattr(self, "answer_question_directly"):
|
593
593
|
raw_data.pop(
|
594
594
|
"answer_question_directly", None
|
@@ -604,9 +604,9 @@ class Agent(Base):
|
|
604
604
|
raw_data["answer_question_directly_source_code"] = inspect.getsource(
|
605
605
|
answer_question_directly_func
|
606
606
|
)
|
607
|
-
raw_data[
|
608
|
-
|
609
|
-
|
607
|
+
raw_data[
|
608
|
+
"answer_question_directly_function_name"
|
609
|
+
] = self.answer_question_directly_function_name
|
610
610
|
|
611
611
|
return raw_data
|
612
612
|
|
edsl/agents/Invigilator.py
CHANGED
@@ -2,14 +2,13 @@
|
|
2
2
|
|
3
3
|
from typing import Dict, Any, Optional
|
4
4
|
|
5
|
-
from edsl.exceptions import AgentRespondedWithBadJSONError
|
6
5
|
from edsl.prompts.Prompt import Prompt
|
7
6
|
from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
|
8
7
|
from edsl.prompts.registry import get_classes as prompt_lookup
|
9
8
|
from edsl.exceptions.questions import QuestionAnswerValidationError
|
10
|
-
from edsl.agents.PromptConstructionMixin import PromptConstructorMixin
|
11
9
|
from edsl.agents.InvigilatorBase import InvigilatorBase
|
12
10
|
from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
|
11
|
+
from edsl.agents.PromptConstructor import PromptConstructor
|
13
12
|
|
14
13
|
|
15
14
|
class NotApplicable(str):
|
@@ -19,9 +18,13 @@ class NotApplicable(str):
|
|
19
18
|
return instance
|
20
19
|
|
21
20
|
|
22
|
-
class InvigilatorAI(
|
21
|
+
class InvigilatorAI(InvigilatorBase):
|
23
22
|
"""An invigilator that uses an AI model to answer questions."""
|
24
23
|
|
24
|
+
def get_prompts(self) -> Dict[str, Prompt]:
|
25
|
+
"""Return the prompts used."""
|
26
|
+
return self.prompt_constructor.get_prompts()
|
27
|
+
|
25
28
|
async def async_answer_question(self) -> AgentResponseDict:
|
26
29
|
"""Answer a question using the AI model.
|
27
30
|
|
@@ -36,6 +39,8 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
|
|
36
39
|
}
|
37
40
|
if "encoded_image" in prompts:
|
38
41
|
params["encoded_image"] = prompts["encoded_image"]
|
42
|
+
if "files_list" in prompts:
|
43
|
+
params["files_list"] = prompts["files_list"]
|
39
44
|
|
40
45
|
params.update({"iteration": self.iteration, "cache": self.cache})
|
41
46
|
|
@@ -77,15 +82,32 @@ class InvigilatorAI(PromptConstructorMixin, InvigilatorBase):
|
|
77
82
|
exception_occurred = None
|
78
83
|
validated = False
|
79
84
|
try:
|
80
|
-
|
85
|
+
# if the question has jinja parameters, it might be easier to make a new question
|
86
|
+
# with those all filled in & then validate that
|
87
|
+
# breakpoint()
|
88
|
+
if self.question.parameters:
|
89
|
+
prior_answers_dict = self.prompt_constructor.prior_answers_dict()
|
90
|
+
question_with_validators = self.question.render(
|
91
|
+
self.scenario | prior_answers_dict
|
92
|
+
)
|
93
|
+
question_with_validators.use_code = self.question.use_code
|
94
|
+
# if question_with_validators.parameters:
|
95
|
+
# raise ValueError(
|
96
|
+
# f"The question still has parameters after rendering: {question_with_validators}"
|
97
|
+
# )
|
98
|
+
else:
|
99
|
+
question_with_validators = self.question
|
100
|
+
|
101
|
+
# breakpoint()
|
102
|
+
validated_edsl_dict = question_with_validators._validate_answer(edsl_dict)
|
81
103
|
answer = self.determine_answer(validated_edsl_dict["answer"])
|
82
104
|
comment = validated_edsl_dict.get("comment", "")
|
83
105
|
validated = True
|
84
106
|
except QuestionAnswerValidationError as e:
|
85
107
|
answer = None
|
86
108
|
comment = "The response was not valid."
|
87
|
-
if self.raise_validation_errors:
|
88
|
-
|
109
|
+
# if self.raise_validation_errors:
|
110
|
+
exception_occurred = e
|
89
111
|
except Exception as non_validation_error:
|
90
112
|
answer = None
|
91
113
|
comment = "Some other error occurred."
|
edsl/agents/InvigilatorBase.py
CHANGED
@@ -14,6 +14,7 @@ from edsl.surveys.MemoryPlan import MemoryPlan
|
|
14
14
|
from edsl.language_models.LanguageModel import LanguageModel
|
15
15
|
|
16
16
|
from edsl.data_transfer_models import EDSLResultObjectInput
|
17
|
+
from edsl.agents.PromptConstructor import PromptConstructor
|
17
18
|
|
18
19
|
|
19
20
|
class InvigilatorBase(ABC):
|
@@ -27,16 +28,7 @@ class InvigilatorBase(ABC):
|
|
27
28
|
|
28
29
|
This returns an empty prompt because there is no memory the agent needs to have at q0.
|
29
30
|
|
30
|
-
>>> InvigilatorBase.example().create_memory_prompt("q0")
|
31
|
-
Prompt(text=\"""\""")
|
32
31
|
|
33
|
-
>>> i = InvigilatorBase.example()
|
34
|
-
>>> i.current_answers = {"q0": "Prior answer"}
|
35
|
-
>>> i.memory_plan.add_single_memory("q1", "q0")
|
36
|
-
>>> i.create_memory_prompt("q1")
|
37
|
-
Prompt(text=\"""
|
38
|
-
Before the question you are now answering, you already answered the following question(s):
|
39
|
-
...
|
40
32
|
"""
|
41
33
|
|
42
34
|
def __init__(
|
@@ -72,6 +64,11 @@ class InvigilatorBase(ABC):
|
|
72
64
|
None # placeholder for the raw response from the model
|
73
65
|
)
|
74
66
|
|
67
|
+
@property
|
68
|
+
def prompt_constructor(self) -> PromptConstructor:
|
69
|
+
"""Return the prompt constructor."""
|
70
|
+
return PromptConstructor(self)
|
71
|
+
|
75
72
|
def to_dict(self):
|
76
73
|
attributes = [
|
77
74
|
"agent",
|
@@ -207,22 +204,6 @@ class InvigilatorBase(ABC):
|
|
207
204
|
|
208
205
|
return main()
|
209
206
|
|
210
|
-
def create_memory_prompt(self, question_name: str) -> Prompt:
|
211
|
-
"""Create a memory for the agent.
|
212
|
-
|
213
|
-
The returns a memory prompt for the agent.
|
214
|
-
|
215
|
-
>>> i = InvigilatorBase.example()
|
216
|
-
>>> i.current_answers = {"q0": "Prior answer"}
|
217
|
-
>>> i.memory_plan.add_single_memory("q1", "q0")
|
218
|
-
>>> p = i.create_memory_prompt("q1")
|
219
|
-
>>> p.text.strip().replace("\\n", " ").replace("\\t", " ")
|
220
|
-
'Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer'
|
221
|
-
"""
|
222
|
-
return self.memory_plan.get_memory_prompt_fragment(
|
223
|
-
question_name, self.current_answers
|
224
|
-
)
|
225
|
-
|
226
207
|
@classmethod
|
227
208
|
def example(
|
228
209
|
cls, throw_an_exception=False, question=None, scenario=None, survey=None
|
@@ -285,9 +266,9 @@ class InvigilatorBase(ABC):
|
|
285
266
|
|
286
267
|
memory_plan = MemoryPlan(survey=survey)
|
287
268
|
current_answers = None
|
288
|
-
from edsl.agents.
|
269
|
+
from edsl.agents.PromptConstructor import PromptConstructor
|
289
270
|
|
290
|
-
class InvigilatorExample(
|
271
|
+
class InvigilatorExample(InvigilatorBase):
|
291
272
|
"""An example invigilator."""
|
292
273
|
|
293
274
|
async def async_answer_question(self):
|