edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +169 -116
- edsl/__init__.py +14 -6
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +358 -146
- edsl/agents/AgentList.py +211 -73
- edsl/agents/Invigilator.py +88 -36
- edsl/agents/InvigilatorBase.py +59 -70
- edsl/agents/PromptConstructor.py +117 -219
- edsl/agents/QuestionInstructionPromptBuilder.py +128 -0
- edsl/agents/QuestionOptionProcessor.py +172 -0
- edsl/agents/QuestionTemplateReplacementsBuilder.py +137 -0
- edsl/agents/__init__.py +0 -1
- edsl/agents/prompt_helpers.py +3 -3
- edsl/config.py +22 -2
- edsl/conversation/car_buying.py +2 -1
- edsl/coop/CoopFunctionsMixin.py +15 -0
- edsl/coop/ExpectedParrotKeyHandler.py +125 -0
- edsl/coop/PriceFetcher.py +1 -1
- edsl/coop/coop.py +104 -42
- edsl/coop/utils.py +14 -14
- edsl/data/Cache.py +21 -14
- edsl/data/CacheEntry.py +12 -15
- edsl/data/CacheHandler.py +33 -12
- edsl/data/__init__.py +4 -3
- edsl/data_transfer_models.py +2 -1
- edsl/enums.py +20 -0
- edsl/exceptions/__init__.py +50 -50
- edsl/exceptions/agents.py +12 -0
- edsl/exceptions/inference_services.py +5 -0
- edsl/exceptions/questions.py +24 -6
- edsl/exceptions/scenarios.py +7 -0
- edsl/inference_services/AnthropicService.py +0 -3
- edsl/inference_services/AvailableModelCacheHandler.py +184 -0
- edsl/inference_services/AvailableModelFetcher.py +209 -0
- edsl/inference_services/AwsBedrock.py +0 -2
- edsl/inference_services/AzureAI.py +0 -2
- edsl/inference_services/GoogleService.py +2 -11
- edsl/inference_services/InferenceServiceABC.py +18 -85
- edsl/inference_services/InferenceServicesCollection.py +105 -80
- edsl/inference_services/MistralAIService.py +0 -3
- edsl/inference_services/OpenAIService.py +1 -4
- edsl/inference_services/PerplexityService.py +0 -3
- edsl/inference_services/ServiceAvailability.py +135 -0
- edsl/inference_services/TestService.py +11 -8
- edsl/inference_services/data_structures.py +62 -0
- edsl/jobs/AnswerQuestionFunctionConstructor.py +188 -0
- edsl/jobs/Answers.py +1 -14
- edsl/jobs/FetchInvigilator.py +40 -0
- edsl/jobs/InterviewTaskManager.py +98 -0
- edsl/jobs/InterviewsConstructor.py +48 -0
- edsl/jobs/Jobs.py +102 -243
- edsl/jobs/JobsChecks.py +35 -10
- edsl/jobs/JobsComponentConstructor.py +189 -0
- edsl/jobs/JobsPrompts.py +5 -3
- edsl/jobs/JobsRemoteInferenceHandler.py +128 -80
- edsl/jobs/JobsRemoteInferenceLogger.py +239 -0
- edsl/jobs/RequestTokenEstimator.py +30 -0
- edsl/jobs/buckets/BucketCollection.py +44 -3
- edsl/jobs/buckets/TokenBucket.py +53 -21
- edsl/jobs/buckets/TokenBucketAPI.py +211 -0
- edsl/jobs/buckets/TokenBucketClient.py +191 -0
- edsl/jobs/decorators.py +35 -0
- edsl/jobs/interviews/Interview.py +77 -380
- edsl/jobs/jobs_status_enums.py +9 -0
- edsl/jobs/loggers/HTMLTableJobLogger.py +304 -0
- edsl/jobs/runners/JobsRunnerAsyncio.py +4 -49
- edsl/jobs/tasks/QuestionTaskCreator.py +21 -19
- edsl/jobs/tasks/TaskHistory.py +14 -15
- edsl/jobs/tasks/task_status_enum.py +0 -2
- edsl/language_models/ComputeCost.py +63 -0
- edsl/language_models/LanguageModel.py +137 -234
- edsl/language_models/ModelList.py +11 -13
- edsl/language_models/PriceManager.py +127 -0
- edsl/language_models/RawResponseHandler.py +106 -0
- edsl/language_models/ServiceDataSources.py +0 -0
- edsl/language_models/__init__.py +0 -1
- edsl/language_models/key_management/KeyLookup.py +63 -0
- edsl/language_models/key_management/KeyLookupBuilder.py +273 -0
- edsl/language_models/key_management/KeyLookupCollection.py +38 -0
- edsl/language_models/key_management/__init__.py +0 -0
- edsl/language_models/key_management/models.py +131 -0
- edsl/language_models/registry.py +49 -59
- edsl/language_models/repair.py +2 -2
- edsl/language_models/utilities.py +5 -4
- edsl/notebooks/Notebook.py +19 -14
- edsl/notebooks/NotebookToLaTeX.py +142 -0
- edsl/prompts/Prompt.py +29 -39
- edsl/questions/AnswerValidatorMixin.py +47 -2
- edsl/questions/ExceptionExplainer.py +77 -0
- edsl/questions/HTMLQuestion.py +103 -0
- edsl/questions/LoopProcessor.py +149 -0
- edsl/questions/QuestionBase.py +37 -192
- edsl/questions/QuestionBaseGenMixin.py +52 -48
- edsl/questions/QuestionBasePromptsMixin.py +7 -3
- edsl/questions/QuestionCheckBox.py +1 -1
- edsl/questions/QuestionExtract.py +1 -1
- edsl/questions/QuestionFreeText.py +1 -2
- edsl/questions/QuestionList.py +3 -5
- edsl/questions/QuestionMatrix.py +265 -0
- edsl/questions/QuestionMultipleChoice.py +66 -22
- edsl/questions/QuestionNumerical.py +1 -3
- edsl/questions/QuestionRank.py +6 -16
- edsl/questions/ResponseValidatorABC.py +37 -11
- edsl/questions/ResponseValidatorFactory.py +28 -0
- edsl/questions/SimpleAskMixin.py +4 -3
- edsl/questions/__init__.py +1 -0
- edsl/questions/derived/QuestionLinearScale.py +6 -3
- edsl/questions/derived/QuestionTopK.py +1 -1
- edsl/questions/descriptors.py +17 -3
- edsl/questions/question_registry.py +1 -1
- edsl/questions/templates/matrix/__init__.py +1 -0
- edsl/questions/templates/matrix/answering_instructions.jinja +5 -0
- edsl/questions/templates/matrix/question_presentation.jinja +20 -0
- edsl/results/CSSParameterizer.py +1 -1
- edsl/results/Dataset.py +170 -7
- edsl/results/DatasetExportMixin.py +224 -302
- edsl/results/DatasetTree.py +28 -8
- edsl/results/MarkdownToDocx.py +122 -0
- edsl/results/MarkdownToPDF.py +111 -0
- edsl/results/Result.py +192 -206
- edsl/results/Results.py +120 -113
- edsl/results/ResultsExportMixin.py +2 -0
- edsl/results/Selector.py +23 -13
- edsl/results/TableDisplay.py +98 -171
- edsl/results/TextEditor.py +50 -0
- edsl/results/__init__.py +1 -1
- edsl/results/smart_objects.py +96 -0
- edsl/results/table_data_class.py +12 -0
- edsl/results/table_renderers.py +118 -0
- edsl/scenarios/ConstructDownloadLink.py +109 -0
- edsl/scenarios/DirectoryScanner.py +96 -0
- edsl/scenarios/DocumentChunker.py +102 -0
- edsl/scenarios/DocxScenario.py +16 -0
- edsl/scenarios/FileStore.py +118 -239
- edsl/scenarios/PdfExtractor.py +40 -0
- edsl/scenarios/Scenario.py +90 -193
- edsl/scenarios/ScenarioHtmlMixin.py +4 -3
- edsl/scenarios/ScenarioJoin.py +10 -6
- edsl/scenarios/ScenarioList.py +383 -240
- edsl/scenarios/ScenarioListExportMixin.py +0 -7
- edsl/scenarios/ScenarioListPdfMixin.py +15 -37
- edsl/scenarios/ScenarioSelector.py +156 -0
- edsl/scenarios/__init__.py +1 -2
- edsl/scenarios/file_methods.py +85 -0
- edsl/scenarios/handlers/__init__.py +13 -0
- edsl/scenarios/handlers/csv.py +38 -0
- edsl/scenarios/handlers/docx.py +76 -0
- edsl/scenarios/handlers/html.py +37 -0
- edsl/scenarios/handlers/json.py +111 -0
- edsl/scenarios/handlers/latex.py +5 -0
- edsl/scenarios/handlers/md.py +51 -0
- edsl/scenarios/handlers/pdf.py +68 -0
- edsl/scenarios/handlers/png.py +39 -0
- edsl/scenarios/handlers/pptx.py +105 -0
- edsl/scenarios/handlers/py.py +294 -0
- edsl/scenarios/handlers/sql.py +313 -0
- edsl/scenarios/handlers/sqlite.py +149 -0
- edsl/scenarios/handlers/txt.py +33 -0
- edsl/study/ObjectEntry.py +1 -1
- edsl/study/SnapShot.py +1 -1
- edsl/study/Study.py +5 -12
- edsl/surveys/ConstructDAG.py +92 -0
- edsl/surveys/EditSurvey.py +221 -0
- edsl/surveys/InstructionHandler.py +100 -0
- edsl/surveys/MemoryManagement.py +72 -0
- edsl/surveys/Rule.py +5 -4
- edsl/surveys/RuleCollection.py +25 -27
- edsl/surveys/RuleManager.py +172 -0
- edsl/surveys/Simulator.py +75 -0
- edsl/surveys/Survey.py +199 -771
- edsl/surveys/SurveyCSS.py +20 -8
- edsl/surveys/{SurveyFlowVisualizationMixin.py → SurveyFlowVisualization.py} +11 -9
- edsl/surveys/SurveyToApp.py +141 -0
- edsl/surveys/__init__.py +4 -2
- edsl/surveys/descriptors.py +6 -2
- edsl/surveys/instructions/ChangeInstruction.py +1 -2
- edsl/surveys/instructions/Instruction.py +4 -13
- edsl/surveys/instructions/InstructionCollection.py +11 -6
- edsl/templates/error_reporting/interview_details.html +1 -1
- edsl/templates/error_reporting/report.html +1 -1
- edsl/tools/plotting.py +1 -1
- edsl/utilities/PrettyList.py +56 -0
- edsl/utilities/is_notebook.py +18 -0
- edsl/utilities/is_valid_variable_name.py +11 -0
- edsl/utilities/remove_edsl_version.py +24 -0
- edsl/utilities/utilities.py +35 -23
- {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/METADATA +12 -10
- edsl-0.1.39.dev2.dist-info/RECORD +352 -0
- edsl/language_models/KeyLookup.py +0 -30
- edsl/language_models/unused/ReplicateBase.py +0 -83
- edsl/results/ResultsDBMixin.py +0 -238
- edsl-0.1.39.dev1.dist-info/RECORD +0 -277
- {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/LICENSE +0 -0
- {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/WHEEL +0 -0
edsl/agents/InvigilatorBase.py
CHANGED
@@ -1,21 +1,22 @@
|
|
1
1
|
from abc import ABC, abstractmethod
|
2
2
|
import asyncio
|
3
|
-
from typing import Coroutine, Dict, Any, Optional
|
3
|
+
from typing import Coroutine, Dict, Any, Optional, TYPE_CHECKING
|
4
4
|
|
5
|
-
from edsl.prompts.Prompt import Prompt
|
6
5
|
from edsl.utilities.decorators import jupyter_nb_handler
|
7
6
|
from edsl.data_transfer_models import AgentResponseDict
|
8
7
|
|
9
|
-
|
10
|
-
|
11
|
-
from edsl.
|
12
|
-
from edsl.
|
13
|
-
from edsl.
|
14
|
-
from edsl.
|
8
|
+
if TYPE_CHECKING:
|
9
|
+
from edsl.prompts.Prompt import Prompt
|
10
|
+
from edsl.data.Cache import Cache
|
11
|
+
from edsl.questions.QuestionBase import QuestionBase
|
12
|
+
from edsl.scenarios.Scenario import Scenario
|
13
|
+
from edsl.surveys.MemoryPlan import MemoryPlan
|
14
|
+
from edsl.language_models.LanguageModel import LanguageModel
|
15
|
+
from edsl.surveys.Survey import Survey
|
16
|
+
from edsl.agents.Agent import Agent
|
15
17
|
|
16
18
|
from edsl.data_transfer_models import EDSLResultObjectInput
|
17
19
|
from edsl.agents.PromptConstructor import PromptConstructor
|
18
|
-
|
19
20
|
from edsl.agents.prompt_helpers import PromptPlan
|
20
21
|
|
21
22
|
|
@@ -29,23 +30,20 @@ class InvigilatorBase(ABC):
|
|
29
30
|
'Failed to get response'
|
30
31
|
|
31
32
|
This returns an empty prompt because there is no memory the agent needs to have at q0.
|
32
|
-
|
33
|
-
|
34
33
|
"""
|
35
34
|
|
36
35
|
def __init__(
|
37
36
|
self,
|
38
37
|
agent: "Agent",
|
39
|
-
question: QuestionBase,
|
40
|
-
scenario: Scenario,
|
41
|
-
model: LanguageModel,
|
42
|
-
memory_plan: MemoryPlan,
|
38
|
+
question: "QuestionBase",
|
39
|
+
scenario: "Scenario",
|
40
|
+
model: "LanguageModel",
|
41
|
+
memory_plan: "MemoryPlan",
|
43
42
|
current_answers: dict,
|
44
43
|
survey: Optional["Survey"],
|
45
|
-
cache: Optional[Cache] = None,
|
44
|
+
cache: Optional["Cache"] = None,
|
46
45
|
iteration: Optional[int] = 1,
|
47
46
|
additional_prompt_data: Optional[dict] = None,
|
48
|
-
sidecar_model: Optional[LanguageModel] = None,
|
49
47
|
raise_validation_errors: Optional[bool] = True,
|
50
48
|
prompt_plan: Optional["PromptPlan"] = None,
|
51
49
|
):
|
@@ -59,24 +57,23 @@ class InvigilatorBase(ABC):
|
|
59
57
|
self.iteration = iteration
|
60
58
|
self.additional_prompt_data = additional_prompt_data
|
61
59
|
self.cache = cache
|
62
|
-
self.sidecar_model = sidecar_model
|
63
60
|
self.survey = survey
|
64
61
|
self.raise_validation_errors = raise_validation_errors
|
62
|
+
|
65
63
|
if prompt_plan is None:
|
66
64
|
self.prompt_plan = PromptPlan()
|
67
65
|
else:
|
68
66
|
self.prompt_plan = prompt_plan
|
69
67
|
|
70
|
-
|
71
|
-
|
72
|
-
)
|
68
|
+
# placeholder to store the raw model response
|
69
|
+
self.raw_model_response = None
|
73
70
|
|
74
71
|
@property
|
75
72
|
def prompt_constructor(self) -> PromptConstructor:
|
76
73
|
"""Return the prompt constructor."""
|
77
74
|
return PromptConstructor(self, prompt_plan=self.prompt_plan)
|
78
75
|
|
79
|
-
def to_dict(self):
|
76
|
+
def to_dict(self, include_cache=False) -> Dict[str, Any]:
|
80
77
|
attributes = [
|
81
78
|
"agent",
|
82
79
|
"question",
|
@@ -86,10 +83,10 @@ class InvigilatorBase(ABC):
|
|
86
83
|
"current_answers",
|
87
84
|
"iteration",
|
88
85
|
"additional_prompt_data",
|
89
|
-
"cache",
|
90
|
-
"sidecar_model",
|
91
86
|
"survey",
|
92
87
|
]
|
88
|
+
if include_cache:
|
89
|
+
attributes.append("cache")
|
93
90
|
|
94
91
|
def serialize_attribute(attr):
|
95
92
|
value = getattr(self, attr)
|
@@ -104,43 +101,37 @@ class InvigilatorBase(ABC):
|
|
104
101
|
return {attr: serialize_attribute(attr) for attr in attributes}
|
105
102
|
|
106
103
|
@classmethod
|
107
|
-
def from_dict(cls, data):
|
104
|
+
def from_dict(cls, data) -> "InvigilatorBase":
|
108
105
|
from edsl.agents.Agent import Agent
|
109
106
|
from edsl.questions import QuestionBase
|
110
107
|
from edsl.scenarios.Scenario import Scenario
|
111
108
|
from edsl.surveys.MemoryPlan import MemoryPlan
|
112
109
|
from edsl.language_models.LanguageModel import LanguageModel
|
113
110
|
from edsl.surveys.Survey import Survey
|
111
|
+
from edsl.data.Cache import Cache
|
112
|
+
|
113
|
+
attributes_to_classes = {
|
114
|
+
"agent": Agent,
|
115
|
+
"question": QuestionBase,
|
116
|
+
"scenario": Scenario,
|
117
|
+
"model": LanguageModel,
|
118
|
+
"memory_plan": MemoryPlan,
|
119
|
+
"survey": Survey,
|
120
|
+
"cache": Cache,
|
121
|
+
}
|
122
|
+
d = {}
|
123
|
+
for attr, cls_ in attributes_to_classes.items():
|
124
|
+
if attr in data and data[attr] is not None:
|
125
|
+
if attr not in data:
|
126
|
+
d[attr] = {}
|
127
|
+
else:
|
128
|
+
d[attr] = cls_.from_dict(data[attr])
|
114
129
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
model = LanguageModel.from_dict(data["model"])
|
119
|
-
memory_plan = MemoryPlan.from_dict(data["memory_plan"])
|
120
|
-
survey = Survey.from_dict(data["survey"])
|
121
|
-
current_answers = data["current_answers"]
|
122
|
-
iteration = data["iteration"]
|
123
|
-
additional_prompt_data = data["additional_prompt_data"]
|
124
|
-
cache = Cache.from_dict(data["cache"])
|
125
|
-
|
126
|
-
if data["sidecar_model"] is None:
|
127
|
-
sidecar_model = None
|
128
|
-
else:
|
129
|
-
sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
|
130
|
+
d["current_answers"] = data["current_answers"]
|
131
|
+
d["iteration"] = data["iteration"]
|
132
|
+
d["additional_prompt_data"] = data["additional_prompt_data"]
|
130
133
|
|
131
|
-
|
132
|
-
agent=agent,
|
133
|
-
question=question,
|
134
|
-
scenario=scenario,
|
135
|
-
model=model,
|
136
|
-
memory_plan=memory_plan,
|
137
|
-
current_answers=current_answers,
|
138
|
-
survey=survey,
|
139
|
-
iteration=iteration,
|
140
|
-
additional_prompt_data=additional_prompt_data,
|
141
|
-
cache=cache,
|
142
|
-
sidecar_model=sidecar_model,
|
143
|
-
)
|
134
|
+
d = cls(**d)
|
144
135
|
|
145
136
|
def __repr__(self) -> str:
|
146
137
|
"""Return a string representation of the Invigilator.
|
@@ -149,9 +140,9 @@ class InvigilatorBase(ABC):
|
|
149
140
|
'InvigilatorExample(...)'
|
150
141
|
|
151
142
|
"""
|
152
|
-
return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)}
|
143
|
+
return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
|
153
144
|
|
154
|
-
def get_failed_task_result(self, failure_reason) -> EDSLResultObjectInput:
|
145
|
+
def get_failed_task_result(self, failure_reason: str) -> EDSLResultObjectInput:
|
155
146
|
"""Return an AgentResponseDict used in case the question-asking fails.
|
156
147
|
|
157
148
|
Possible reasons include:
|
@@ -172,8 +163,9 @@ class InvigilatorBase(ABC):
|
|
172
163
|
}
|
173
164
|
return EDSLResultObjectInput(**data)
|
174
165
|
|
175
|
-
def get_prompts(self) -> Dict[str, Prompt]:
|
166
|
+
def get_prompts(self) -> Dict[str, "Prompt"]:
|
176
167
|
"""Return the prompt used."""
|
168
|
+
from edsl.prompts.Prompt import Prompt
|
177
169
|
|
178
170
|
return {
|
179
171
|
"user_prompt": Prompt("NA"),
|
@@ -205,24 +197,25 @@ class InvigilatorBase(ABC):
|
|
205
197
|
>>> InvigilatorBase.example()
|
206
198
|
InvigilatorExample(...)
|
207
199
|
|
200
|
+
>>> InvigilatorBase.example().answer_question()
|
201
|
+
{'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
|
202
|
+
|
203
|
+
>>> InvigilatorBase.example(throw_an_exception=True).answer_question()
|
204
|
+
Traceback (most recent call last):
|
205
|
+
...
|
206
|
+
Exception: This is a test error
|
208
207
|
"""
|
209
208
|
from edsl.agents.Agent import Agent
|
210
|
-
from edsl.questions import QuestionMultipleChoice
|
211
209
|
from edsl.scenarios.Scenario import Scenario
|
212
|
-
from edsl.language_models import LanguageModel
|
213
210
|
from edsl.surveys.MemoryPlan import MemoryPlan
|
214
|
-
|
215
|
-
from edsl.
|
216
|
-
|
217
|
-
from edsl import Model
|
211
|
+
from edsl.language_models.registry import Model
|
212
|
+
from edsl.surveys.Survey import Survey
|
218
213
|
|
219
214
|
model = Model("test", canned_response="SPAM!")
|
220
215
|
|
221
216
|
if throw_an_exception:
|
222
|
-
model.
|
217
|
+
model.throw_exception = True
|
223
218
|
agent = Agent.example()
|
224
|
-
# question = QuestionMultipleChoice.example()
|
225
|
-
from edsl.surveys import Survey
|
226
219
|
|
227
220
|
if not survey:
|
228
221
|
survey = Survey.example()
|
@@ -232,14 +225,10 @@ class InvigilatorBase(ABC):
|
|
232
225
|
|
233
226
|
question = question or survey.questions[0]
|
234
227
|
scenario = scenario or Scenario.example()
|
235
|
-
# memory_plan = None #memory_plan = MemoryPlan()
|
236
|
-
from edsl import Survey
|
237
|
-
|
238
228
|
memory_plan = MemoryPlan(survey=survey)
|
239
229
|
current_answers = None
|
240
|
-
from edsl.agents.PromptConstructor import PromptConstructor
|
241
230
|
|
242
|
-
class InvigilatorExample(
|
231
|
+
class InvigilatorExample(cls):
|
243
232
|
"""An example invigilator."""
|
244
233
|
|
245
234
|
async def async_answer_question(self):
|
edsl/agents/PromptConstructor.py
CHANGED
@@ -1,10 +1,18 @@
|
|
1
1
|
from __future__ import annotations
|
2
|
-
from typing import Dict, Any, Optional, Set
|
3
|
-
|
4
|
-
from jinja2 import Environment, meta
|
2
|
+
from typing import Dict, Any, Optional, Set, Union, TYPE_CHECKING
|
3
|
+
from functools import cached_property
|
5
4
|
|
6
5
|
from edsl.prompts.Prompt import Prompt
|
7
|
-
|
6
|
+
|
7
|
+
from .prompt_helpers import PromptPlan
|
8
|
+
from .QuestionTemplateReplacementsBuilder import (
|
9
|
+
QuestionTemplateReplacementsBuilder,
|
10
|
+
)
|
11
|
+
from .QuestionOptionProcessor import QuestionOptionProcessor
|
12
|
+
|
13
|
+
if TYPE_CHECKING:
|
14
|
+
from edsl.agents.InvigilatorBase import InvigilatorBase
|
15
|
+
from edsl.questions.QuestionBase import QuestionBase
|
8
16
|
|
9
17
|
|
10
18
|
class PlaceholderAnswer:
|
@@ -18,39 +26,34 @@ class PlaceholderAnswer:
|
|
18
26
|
return ""
|
19
27
|
|
20
28
|
def __str__(self):
|
21
|
-
return "<<
|
29
|
+
return f"<<{self.__class__.__name__}>>"
|
22
30
|
|
23
31
|
def __repr__(self):
|
24
|
-
return "<<
|
32
|
+
return f"<<{self.__class__.__name__}>>"
|
25
33
|
|
26
34
|
|
27
|
-
|
28
|
-
|
29
|
-
Extracts all variable names from a Jinja2 template using Jinja2's built-in parsing.
|
35
|
+
class PlaceholderComment(PlaceholderAnswer):
|
36
|
+
pass
|
30
37
|
|
31
|
-
Args:
|
32
|
-
template_str (str): The Jinja2 template string
|
33
38
|
|
34
|
-
|
35
|
-
|
36
|
-
"""
|
37
|
-
env = Environment()
|
38
|
-
ast = env.parse(template_str)
|
39
|
-
return meta.find_undeclared_variables(ast)
|
39
|
+
class PlaceholderGeneratedTokens(PlaceholderAnswer):
|
40
|
+
pass
|
40
41
|
|
41
42
|
|
42
43
|
class PromptConstructor:
|
43
44
|
"""
|
45
|
+
This class constructs the prompts for the language model.
|
46
|
+
|
44
47
|
The pieces of a prompt are:
|
45
48
|
- The agent instructions - "You are answering questions as if you were a human. Do not break character."
|
46
49
|
- The persona prompt - "You are an agent with the following persona: {'age': 22, 'hair': 'brown', 'height': 5.5}"
|
47
50
|
- The question instructions - "You are being asked the following question: Do you like school? The options are 0: yes 1: no Return a valid JSON formatted like this, selecting only the number of the option: {"answer": <put answer code here>, "comment": "<put explanation here>"} Only 1 option may be selected."
|
48
51
|
- The memory prompt - "Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer"
|
49
|
-
|
50
|
-
This is mixed into the Invigilator class.
|
51
52
|
"""
|
52
53
|
|
53
|
-
def __init__(
|
54
|
+
def __init__(
|
55
|
+
self, invigilator: "InvigilatorBase", prompt_plan: Optional["PromptPlan"] = None
|
56
|
+
):
|
54
57
|
self.invigilator = invigilator
|
55
58
|
self.agent = invigilator.agent
|
56
59
|
self.question = invigilator.question
|
@@ -61,20 +64,11 @@ class PromptConstructor:
|
|
61
64
|
self.memory_plan = invigilator.memory_plan
|
62
65
|
self.prompt_plan = prompt_plan or PromptPlan()
|
63
66
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
These will be used to append to the prompt a list of files that are part of the scenario.
|
68
|
-
"""
|
69
|
-
from edsl.scenarios.FileStore import FileStore
|
70
|
-
|
71
|
-
file_entries = []
|
72
|
-
for key, value in self.scenario.items():
|
73
|
-
if isinstance(value, FileStore):
|
74
|
-
file_entries.append(key)
|
75
|
-
return file_entries
|
67
|
+
def get_question_options(self, question_data):
|
68
|
+
"""Get the question options."""
|
69
|
+
return QuestionOptionProcessor(self).get_question_options(question_data)
|
76
70
|
|
77
|
-
@
|
71
|
+
@cached_property
|
78
72
|
def agent_instructions_prompt(self) -> Prompt:
|
79
73
|
"""
|
80
74
|
>>> from edsl.agents.InvigilatorBase import InvigilatorBase
|
@@ -82,14 +76,14 @@ class PromptConstructor:
|
|
82
76
|
>>> i.prompt_constructor.agent_instructions_prompt
|
83
77
|
Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
|
84
78
|
"""
|
85
|
-
from edsl import Agent
|
79
|
+
from edsl.agents.Agent import Agent
|
86
80
|
|
87
81
|
if self.agent == Agent(): # if agent is empty, then return an empty prompt
|
88
82
|
return Prompt(text="")
|
89
83
|
|
90
84
|
return Prompt(text=self.agent.instruction)
|
91
85
|
|
92
|
-
@
|
86
|
+
@cached_property
|
93
87
|
def agent_persona_prompt(self) -> Prompt:
|
94
88
|
"""
|
95
89
|
>>> from edsl.agents.InvigilatorBase import InvigilatorBase
|
@@ -97,159 +91,93 @@ class PromptConstructor:
|
|
97
91
|
>>> i.prompt_constructor.agent_persona_prompt
|
98
92
|
Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
|
99
93
|
"""
|
100
|
-
from edsl import Agent
|
94
|
+
from edsl.agents.Agent import Agent
|
101
95
|
|
102
96
|
if self.agent == Agent(): # if agent is empty, then return an empty prompt
|
103
97
|
return Prompt(text="")
|
104
98
|
|
105
99
|
return self.agent.prompt()
|
106
100
|
|
107
|
-
def prior_answers_dict(self) -> dict:
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
if question in self.current_answers:
|
113
|
-
d[question].answer = self.current_answers[question]
|
114
|
-
else:
|
115
|
-
d[question].answer = PlaceholderAnswer()
|
101
|
+
def prior_answers_dict(self) -> dict[str, "QuestionBase"]:
|
102
|
+
"""This is a dictionary of prior answers, if they exist."""
|
103
|
+
return self._add_answers(
|
104
|
+
self.survey.question_names_to_questions(), self.current_answers
|
105
|
+
)
|
116
106
|
|
117
|
-
|
118
|
-
|
119
|
-
|
107
|
+
@staticmethod
|
108
|
+
def _extract_quetion_and_entry_type(key_entry) -> tuple[str, str]:
|
109
|
+
"""
|
110
|
+
Extracts the question name and type for the current answer dictionary key entry.
|
111
|
+
|
112
|
+
>>> PromptConstructor._extract_quetion_and_entry_type("q0")
|
113
|
+
('q0', 'answer')
|
114
|
+
>>> PromptConstructor._extract_quetion_and_entry_type("q0_comment")
|
115
|
+
('q0', 'comment')
|
116
|
+
>>> PromptConstructor._extract_quetion_and_entry_type("q0_alternate_generated_tokens")
|
117
|
+
('q0_alternate', 'generated_tokens')
|
118
|
+
"""
|
119
|
+
split_list = key_entry.rsplit("_", maxsplit=1)
|
120
|
+
if len(split_list) == 1:
|
121
|
+
question_name = split_list[0]
|
122
|
+
entry_type = "answer"
|
123
|
+
else:
|
124
|
+
if split_list[1] == "comment":
|
125
|
+
question_name = split_list[0]
|
126
|
+
entry_type = "comment"
|
127
|
+
elif split_list[1] == "tokens": # it's actually 'generated_tokens'
|
128
|
+
question_name = key_entry.replace("_generated_tokens", "")
|
129
|
+
entry_type = "generated_tokens"
|
130
|
+
else:
|
131
|
+
question_name = key_entry
|
132
|
+
entry_type = "answer"
|
133
|
+
return question_name, entry_type
|
120
134
|
|
121
|
-
|
135
|
+
@staticmethod
|
136
|
+
def _augmented_answers_dict(current_answers: dict):
|
137
|
+
"""
|
138
|
+
>>> PromptConstructor._augmented_answers_dict({"q0": "LOVE IT!", "q0_comment": "I love school!"})
|
139
|
+
{'q0': {'answer': 'LOVE IT!', 'comment': 'I love school!'}}
|
140
|
+
"""
|
141
|
+
d = {}
|
142
|
+
for key, value in current_answers.items():
|
143
|
+
(
|
144
|
+
question_name,
|
145
|
+
entry_type,
|
146
|
+
) = PromptConstructor._extract_quetion_and_entry_type(key)
|
147
|
+
if question_name not in d:
|
148
|
+
d[question_name] = {}
|
149
|
+
d[question_name][entry_type] = value
|
122
150
|
return d
|
123
151
|
|
124
|
-
@
|
125
|
-
def
|
126
|
-
raw_question_text = self.question.question_text
|
127
|
-
variables = get_jinja2_variables(raw_question_text)
|
128
|
-
question_file_keys = []
|
129
|
-
for var in variables:
|
130
|
-
if var in self.scenario_file_keys:
|
131
|
-
question_file_keys.append(var)
|
132
|
-
return question_file_keys
|
133
|
-
|
134
|
-
def build_replacement_dict(self, question_data: dict):
|
152
|
+
@staticmethod
|
153
|
+
def _add_answers(answer_dict: dict, current_answers) -> dict[str, "QuestionBase"]:
|
135
154
|
"""
|
136
|
-
|
155
|
+
>>> from edsl import QuestionFreeText
|
156
|
+
>>> d = {"q0": QuestionFreeText(question_text="Do you like school?", question_name = "q0")}
|
157
|
+
>>> current_answers = {"q0": "LOVE IT!"}
|
158
|
+
>>> PromptConstructor._add_answers(d, current_answers)['q0'].answer
|
159
|
+
'LOVE IT!'
|
137
160
|
"""
|
138
|
-
|
139
|
-
file_refs = {key: f"<see file {key}>" for key in self.scenario_file_keys}
|
140
|
-
|
141
|
-
# Scenario items excluding file keys
|
142
|
-
scenario_items = {
|
143
|
-
k: v for k, v in self.scenario.items() if k not in self.scenario_file_keys
|
144
|
-
}
|
145
|
-
|
146
|
-
# Question settings with defaults
|
147
|
-
question_settings = {
|
148
|
-
"use_code": getattr(self.question, "_use_code", True),
|
149
|
-
"include_comment": getattr(self.question, "_include_comment", False),
|
150
|
-
}
|
151
|
-
|
152
|
-
# Combine all dictionaries using dict.update() for clarity
|
153
|
-
replacement_dict = {}
|
154
|
-
for d in [
|
155
|
-
file_refs,
|
156
|
-
question_data,
|
157
|
-
scenario_items,
|
158
|
-
self.prior_answers_dict(),
|
159
|
-
{"agent": self.agent},
|
160
|
-
question_settings,
|
161
|
-
]:
|
162
|
-
replacement_dict.update(d)
|
163
|
-
|
164
|
-
return replacement_dict
|
165
|
-
|
166
|
-
def _get_question_options(self, question_data):
|
167
|
-
question_options_entry = question_data.get("question_options", None)
|
168
|
-
question_options = question_options_entry
|
169
|
-
|
170
|
-
placeholder = ["<< Option 1 - Placholder >>", "<< Option 2 - Placholder >>"]
|
171
|
-
|
172
|
-
# print("Question options entry: ", question_options_entry)
|
173
|
-
|
174
|
-
if isinstance(question_options_entry, str):
|
175
|
-
env = Environment()
|
176
|
-
parsed_content = env.parse(question_options_entry)
|
177
|
-
question_option_key = list(meta.find_undeclared_variables(parsed_content))[
|
178
|
-
0
|
179
|
-
]
|
180
|
-
if isinstance(self.scenario.get(question_option_key), list):
|
181
|
-
question_options = self.scenario.get(question_option_key)
|
182
|
-
|
183
|
-
# might be getting it from the prior answers
|
184
|
-
if self.prior_answers_dict().get(question_option_key) is not None:
|
185
|
-
prior_question = self.prior_answers_dict().get(question_option_key)
|
186
|
-
if hasattr(prior_question, "answer"):
|
187
|
-
if isinstance(prior_question.answer, list):
|
188
|
-
question_options = prior_question.answer
|
189
|
-
else:
|
190
|
-
question_options = placeholder
|
191
|
-
else:
|
192
|
-
question_options = placeholder
|
193
|
-
|
194
|
-
return question_options
|
195
|
-
|
196
|
-
def build_question_instructions_prompt(self):
|
197
|
-
"""Buils the question instructions prompt."""
|
198
|
-
|
199
|
-
question_prompt = Prompt(self.question.get_instructions(model=self.model.model))
|
200
|
-
|
201
|
-
# Get the data for the question - this is a dictionary of the question data
|
202
|
-
# e.g., {'question_text': 'Do you like school?', 'question_name': 'q0', 'question_options': ['yes', 'no']}
|
203
|
-
question_data = self.question.data.copy()
|
204
|
-
|
205
|
-
if (
|
206
|
-
"question_options" in question_data
|
207
|
-
): # is this a question with question options?
|
208
|
-
question_options = self._get_question_options(question_data)
|
209
|
-
question_data["question_options"] = question_options
|
161
|
+
augmented_answers = PromptConstructor._augmented_answers_dict(current_answers)
|
210
162
|
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
if undefined_template_variables:
|
228
|
-
msg = f"Question instructions still has variables: {undefined_template_variables}."
|
229
|
-
import warnings
|
230
|
-
|
231
|
-
warnings.warn(msg)
|
232
|
-
# raise QuestionScenarioRenderError(
|
233
|
-
# f"Question instructions still has variables: {undefined_template_variables}."
|
234
|
-
# )
|
235
|
-
|
236
|
-
# Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions
|
237
|
-
relevant_instructions = self.survey.relevant_instructions(
|
238
|
-
self.question.question_name
|
239
|
-
)
|
240
|
-
|
241
|
-
if relevant_instructions != []:
|
242
|
-
# preamble_text = Prompt(
|
243
|
-
# text="You were given the following instructions: "
|
244
|
-
# )
|
245
|
-
preamble_text = Prompt(text="")
|
246
|
-
for instruction in relevant_instructions:
|
247
|
-
preamble_text += instruction.text
|
248
|
-
rendered_instructions = preamble_text + rendered_instructions
|
249
|
-
|
250
|
-
return rendered_instructions
|
163
|
+
for question in answer_dict:
|
164
|
+
if question in augmented_answers:
|
165
|
+
for entry_type, value in augmented_answers[question].items():
|
166
|
+
setattr(answer_dict[question], entry_type, value)
|
167
|
+
else:
|
168
|
+
answer_dict[question].answer = PlaceholderAnswer()
|
169
|
+
answer_dict[question].comment = PlaceholderComment()
|
170
|
+
answer_dict[question].generated_tokens = PlaceholderGeneratedTokens()
|
171
|
+
return answer_dict
|
172
|
+
|
173
|
+
@cached_property
|
174
|
+
def question_file_keys(self) -> list:
|
175
|
+
"""Extracts the file keys from the question text.
|
176
|
+
It checks if the variables in the question text are in the scenario file keys.
|
177
|
+
"""
|
178
|
+
return QuestionTemplateReplacementsBuilder(self).question_file_keys()
|
251
179
|
|
252
|
-
@
|
180
|
+
@cached_property
|
253
181
|
def question_instructions_prompt(self) -> Prompt:
|
254
182
|
"""
|
255
183
|
>>> from edsl.agents.InvigilatorBase import InvigilatorBase
|
@@ -258,25 +186,24 @@ class PromptConstructor:
|
|
258
186
|
Prompt(text=\"""...
|
259
187
|
...
|
260
188
|
"""
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
189
|
+
return self.build_question_instructions_prompt()
|
190
|
+
|
191
|
+
def build_question_instructions_prompt(self) -> Prompt:
|
192
|
+
"""Buils the question instructions prompt."""
|
193
|
+
from edsl.agents.QuestionInstructionPromptBuilder import (
|
194
|
+
QuestionInstructionPromptBuilder,
|
195
|
+
)
|
265
196
|
|
266
|
-
return self.
|
197
|
+
return QuestionInstructionPromptBuilder(self).build()
|
267
198
|
|
268
|
-
@
|
199
|
+
@cached_property
|
269
200
|
def prior_question_memory_prompt(self) -> Prompt:
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
self.question.question_name
|
277
|
-
).render(self.scenario | self.prior_answers_dict())
|
278
|
-
self._prior_question_memory_prompt = memory_prompt
|
279
|
-
return self._prior_question_memory_prompt
|
201
|
+
memory_prompt = Prompt(text="")
|
202
|
+
if self.memory_plan is not None:
|
203
|
+
memory_prompt += self.create_memory_prompt(
|
204
|
+
self.question.question_name
|
205
|
+
).render(self.scenario | self.prior_answers_dict())
|
206
|
+
return memory_prompt
|
280
207
|
|
281
208
|
def create_memory_prompt(self, question_name: str) -> Prompt:
|
282
209
|
"""Create a memory for the agent.
|
@@ -295,24 +222,6 @@ class PromptConstructor:
|
|
295
222
|
question_name, self.current_answers
|
296
223
|
)
|
297
224
|
|
298
|
-
def construct_system_prompt(self) -> Prompt:
|
299
|
-
"""Construct the system prompt for the LLM call."""
|
300
|
-
import warnings
|
301
|
-
|
302
|
-
warnings.warn(
|
303
|
-
"This method is deprecated. Use get_prompts instead.", DeprecationWarning
|
304
|
-
)
|
305
|
-
return self.get_prompts()["system_prompt"]
|
306
|
-
|
307
|
-
def construct_user_prompt(self) -> Prompt:
|
308
|
-
"""Construct the user prompt for the LLM call."""
|
309
|
-
import warnings
|
310
|
-
|
311
|
-
warnings.warn(
|
312
|
-
"This method is deprecated. Use get_prompts instead.", DeprecationWarning
|
313
|
-
)
|
314
|
-
return self.get_prompts()["user_prompt"]
|
315
|
-
|
316
225
|
def get_prompts(self) -> Dict[str, Prompt]:
|
317
226
|
"""Get both prompts for the LLM call.
|
318
227
|
|
@@ -323,7 +232,6 @@ class PromptConstructor:
|
|
323
232
|
>>> i.get_prompts()
|
324
233
|
{'user_prompt': ..., 'system_prompt': ...}
|
325
234
|
"""
|
326
|
-
# breakpoint()
|
327
235
|
prompts = self.prompt_plan.get_prompts(
|
328
236
|
agent_instructions=self.agent_instructions_prompt,
|
329
237
|
agent_persona=self.agent_persona_prompt,
|
@@ -337,16 +245,6 @@ class PromptConstructor:
|
|
337
245
|
prompts["files_list"] = files_list
|
338
246
|
return prompts
|
339
247
|
|
340
|
-
def _get_scenario_with_image(self) -> Scenario:
|
341
|
-
"""This is a helper function to get a scenario with an image, for testing purposes."""
|
342
|
-
from edsl import Scenario
|
343
|
-
|
344
|
-
try:
|
345
|
-
scenario = Scenario.from_image("../../static/logo.png")
|
346
|
-
except FileNotFoundError:
|
347
|
-
scenario = Scenario.from_image("static/logo.png")
|
348
|
-
return scenario
|
349
|
-
|
350
248
|
|
351
249
|
if __name__ == "__main__":
|
352
250
|
import doctest
|