edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. edsl/Base.py +169 -116
  2. edsl/__init__.py +14 -6
  3. edsl/__version__.py +1 -1
  4. edsl/agents/Agent.py +358 -146
  5. edsl/agents/AgentList.py +211 -73
  6. edsl/agents/Invigilator.py +88 -36
  7. edsl/agents/InvigilatorBase.py +59 -70
  8. edsl/agents/PromptConstructor.py +117 -219
  9. edsl/agents/QuestionInstructionPromptBuilder.py +128 -0
  10. edsl/agents/QuestionOptionProcessor.py +172 -0
  11. edsl/agents/QuestionTemplateReplacementsBuilder.py +137 -0
  12. edsl/agents/__init__.py +0 -1
  13. edsl/agents/prompt_helpers.py +3 -3
  14. edsl/config.py +22 -2
  15. edsl/conversation/car_buying.py +2 -1
  16. edsl/coop/CoopFunctionsMixin.py +15 -0
  17. edsl/coop/ExpectedParrotKeyHandler.py +125 -0
  18. edsl/coop/PriceFetcher.py +1 -1
  19. edsl/coop/coop.py +104 -42
  20. edsl/coop/utils.py +14 -14
  21. edsl/data/Cache.py +21 -14
  22. edsl/data/CacheEntry.py +12 -15
  23. edsl/data/CacheHandler.py +33 -12
  24. edsl/data/__init__.py +4 -3
  25. edsl/data_transfer_models.py +2 -1
  26. edsl/enums.py +20 -0
  27. edsl/exceptions/__init__.py +50 -50
  28. edsl/exceptions/agents.py +12 -0
  29. edsl/exceptions/inference_services.py +5 -0
  30. edsl/exceptions/questions.py +24 -6
  31. edsl/exceptions/scenarios.py +7 -0
  32. edsl/inference_services/AnthropicService.py +0 -3
  33. edsl/inference_services/AvailableModelCacheHandler.py +184 -0
  34. edsl/inference_services/AvailableModelFetcher.py +209 -0
  35. edsl/inference_services/AwsBedrock.py +0 -2
  36. edsl/inference_services/AzureAI.py +0 -2
  37. edsl/inference_services/GoogleService.py +2 -11
  38. edsl/inference_services/InferenceServiceABC.py +18 -85
  39. edsl/inference_services/InferenceServicesCollection.py +105 -80
  40. edsl/inference_services/MistralAIService.py +0 -3
  41. edsl/inference_services/OpenAIService.py +1 -4
  42. edsl/inference_services/PerplexityService.py +0 -3
  43. edsl/inference_services/ServiceAvailability.py +135 -0
  44. edsl/inference_services/TestService.py +11 -8
  45. edsl/inference_services/data_structures.py +62 -0
  46. edsl/jobs/AnswerQuestionFunctionConstructor.py +188 -0
  47. edsl/jobs/Answers.py +1 -14
  48. edsl/jobs/FetchInvigilator.py +40 -0
  49. edsl/jobs/InterviewTaskManager.py +98 -0
  50. edsl/jobs/InterviewsConstructor.py +48 -0
  51. edsl/jobs/Jobs.py +102 -243
  52. edsl/jobs/JobsChecks.py +35 -10
  53. edsl/jobs/JobsComponentConstructor.py +189 -0
  54. edsl/jobs/JobsPrompts.py +5 -3
  55. edsl/jobs/JobsRemoteInferenceHandler.py +128 -80
  56. edsl/jobs/JobsRemoteInferenceLogger.py +239 -0
  57. edsl/jobs/RequestTokenEstimator.py +30 -0
  58. edsl/jobs/buckets/BucketCollection.py +44 -3
  59. edsl/jobs/buckets/TokenBucket.py +53 -21
  60. edsl/jobs/buckets/TokenBucketAPI.py +211 -0
  61. edsl/jobs/buckets/TokenBucketClient.py +191 -0
  62. edsl/jobs/decorators.py +35 -0
  63. edsl/jobs/interviews/Interview.py +77 -380
  64. edsl/jobs/jobs_status_enums.py +9 -0
  65. edsl/jobs/loggers/HTMLTableJobLogger.py +304 -0
  66. edsl/jobs/runners/JobsRunnerAsyncio.py +4 -49
  67. edsl/jobs/tasks/QuestionTaskCreator.py +21 -19
  68. edsl/jobs/tasks/TaskHistory.py +14 -15
  69. edsl/jobs/tasks/task_status_enum.py +0 -2
  70. edsl/language_models/ComputeCost.py +63 -0
  71. edsl/language_models/LanguageModel.py +137 -234
  72. edsl/language_models/ModelList.py +11 -13
  73. edsl/language_models/PriceManager.py +127 -0
  74. edsl/language_models/RawResponseHandler.py +106 -0
  75. edsl/language_models/ServiceDataSources.py +0 -0
  76. edsl/language_models/__init__.py +0 -1
  77. edsl/language_models/key_management/KeyLookup.py +63 -0
  78. edsl/language_models/key_management/KeyLookupBuilder.py +273 -0
  79. edsl/language_models/key_management/KeyLookupCollection.py +38 -0
  80. edsl/language_models/key_management/__init__.py +0 -0
  81. edsl/language_models/key_management/models.py +131 -0
  82. edsl/language_models/registry.py +49 -59
  83. edsl/language_models/repair.py +2 -2
  84. edsl/language_models/utilities.py +5 -4
  85. edsl/notebooks/Notebook.py +19 -14
  86. edsl/notebooks/NotebookToLaTeX.py +142 -0
  87. edsl/prompts/Prompt.py +29 -39
  88. edsl/questions/AnswerValidatorMixin.py +47 -2
  89. edsl/questions/ExceptionExplainer.py +77 -0
  90. edsl/questions/HTMLQuestion.py +103 -0
  91. edsl/questions/LoopProcessor.py +149 -0
  92. edsl/questions/QuestionBase.py +37 -192
  93. edsl/questions/QuestionBaseGenMixin.py +52 -48
  94. edsl/questions/QuestionBasePromptsMixin.py +7 -3
  95. edsl/questions/QuestionCheckBox.py +1 -1
  96. edsl/questions/QuestionExtract.py +1 -1
  97. edsl/questions/QuestionFreeText.py +1 -2
  98. edsl/questions/QuestionList.py +3 -5
  99. edsl/questions/QuestionMatrix.py +265 -0
  100. edsl/questions/QuestionMultipleChoice.py +66 -22
  101. edsl/questions/QuestionNumerical.py +1 -3
  102. edsl/questions/QuestionRank.py +6 -16
  103. edsl/questions/ResponseValidatorABC.py +37 -11
  104. edsl/questions/ResponseValidatorFactory.py +28 -0
  105. edsl/questions/SimpleAskMixin.py +4 -3
  106. edsl/questions/__init__.py +1 -0
  107. edsl/questions/derived/QuestionLinearScale.py +6 -3
  108. edsl/questions/derived/QuestionTopK.py +1 -1
  109. edsl/questions/descriptors.py +17 -3
  110. edsl/questions/question_registry.py +1 -1
  111. edsl/questions/templates/matrix/__init__.py +1 -0
  112. edsl/questions/templates/matrix/answering_instructions.jinja +5 -0
  113. edsl/questions/templates/matrix/question_presentation.jinja +20 -0
  114. edsl/results/CSSParameterizer.py +1 -1
  115. edsl/results/Dataset.py +170 -7
  116. edsl/results/DatasetExportMixin.py +224 -302
  117. edsl/results/DatasetTree.py +28 -8
  118. edsl/results/MarkdownToDocx.py +122 -0
  119. edsl/results/MarkdownToPDF.py +111 -0
  120. edsl/results/Result.py +192 -206
  121. edsl/results/Results.py +120 -113
  122. edsl/results/ResultsExportMixin.py +2 -0
  123. edsl/results/Selector.py +23 -13
  124. edsl/results/TableDisplay.py +98 -171
  125. edsl/results/TextEditor.py +50 -0
  126. edsl/results/__init__.py +1 -1
  127. edsl/results/smart_objects.py +96 -0
  128. edsl/results/table_data_class.py +12 -0
  129. edsl/results/table_renderers.py +118 -0
  130. edsl/scenarios/ConstructDownloadLink.py +109 -0
  131. edsl/scenarios/DirectoryScanner.py +96 -0
  132. edsl/scenarios/DocumentChunker.py +102 -0
  133. edsl/scenarios/DocxScenario.py +16 -0
  134. edsl/scenarios/FileStore.py +118 -239
  135. edsl/scenarios/PdfExtractor.py +40 -0
  136. edsl/scenarios/Scenario.py +90 -193
  137. edsl/scenarios/ScenarioHtmlMixin.py +4 -3
  138. edsl/scenarios/ScenarioJoin.py +10 -6
  139. edsl/scenarios/ScenarioList.py +383 -240
  140. edsl/scenarios/ScenarioListExportMixin.py +0 -7
  141. edsl/scenarios/ScenarioListPdfMixin.py +15 -37
  142. edsl/scenarios/ScenarioSelector.py +156 -0
  143. edsl/scenarios/__init__.py +1 -2
  144. edsl/scenarios/file_methods.py +85 -0
  145. edsl/scenarios/handlers/__init__.py +13 -0
  146. edsl/scenarios/handlers/csv.py +38 -0
  147. edsl/scenarios/handlers/docx.py +76 -0
  148. edsl/scenarios/handlers/html.py +37 -0
  149. edsl/scenarios/handlers/json.py +111 -0
  150. edsl/scenarios/handlers/latex.py +5 -0
  151. edsl/scenarios/handlers/md.py +51 -0
  152. edsl/scenarios/handlers/pdf.py +68 -0
  153. edsl/scenarios/handlers/png.py +39 -0
  154. edsl/scenarios/handlers/pptx.py +105 -0
  155. edsl/scenarios/handlers/py.py +294 -0
  156. edsl/scenarios/handlers/sql.py +313 -0
  157. edsl/scenarios/handlers/sqlite.py +149 -0
  158. edsl/scenarios/handlers/txt.py +33 -0
  159. edsl/study/ObjectEntry.py +1 -1
  160. edsl/study/SnapShot.py +1 -1
  161. edsl/study/Study.py +5 -12
  162. edsl/surveys/ConstructDAG.py +92 -0
  163. edsl/surveys/EditSurvey.py +221 -0
  164. edsl/surveys/InstructionHandler.py +100 -0
  165. edsl/surveys/MemoryManagement.py +72 -0
  166. edsl/surveys/Rule.py +5 -4
  167. edsl/surveys/RuleCollection.py +25 -27
  168. edsl/surveys/RuleManager.py +172 -0
  169. edsl/surveys/Simulator.py +75 -0
  170. edsl/surveys/Survey.py +199 -771
  171. edsl/surveys/SurveyCSS.py +20 -8
  172. edsl/surveys/{SurveyFlowVisualizationMixin.py → SurveyFlowVisualization.py} +11 -9
  173. edsl/surveys/SurveyToApp.py +141 -0
  174. edsl/surveys/__init__.py +4 -2
  175. edsl/surveys/descriptors.py +6 -2
  176. edsl/surveys/instructions/ChangeInstruction.py +1 -2
  177. edsl/surveys/instructions/Instruction.py +4 -13
  178. edsl/surveys/instructions/InstructionCollection.py +11 -6
  179. edsl/templates/error_reporting/interview_details.html +1 -1
  180. edsl/templates/error_reporting/report.html +1 -1
  181. edsl/tools/plotting.py +1 -1
  182. edsl/utilities/PrettyList.py +56 -0
  183. edsl/utilities/is_notebook.py +18 -0
  184. edsl/utilities/is_valid_variable_name.py +11 -0
  185. edsl/utilities/remove_edsl_version.py +24 -0
  186. edsl/utilities/utilities.py +35 -23
  187. {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/METADATA +12 -10
  188. edsl-0.1.39.dev2.dist-info/RECORD +352 -0
  189. edsl/language_models/KeyLookup.py +0 -30
  190. edsl/language_models/unused/ReplicateBase.py +0 -83
  191. edsl/results/ResultsDBMixin.py +0 -238
  192. edsl-0.1.39.dev1.dist-info/RECORD +0 -277
  193. {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/LICENSE +0 -0
  194. {edsl-0.1.39.dev1.dist-info → edsl-0.1.39.dev2.dist-info}/WHEEL +0 -0
@@ -1,21 +1,22 @@
1
1
  from abc import ABC, abstractmethod
2
2
  import asyncio
3
- from typing import Coroutine, Dict, Any, Optional
3
+ from typing import Coroutine, Dict, Any, Optional, TYPE_CHECKING
4
4
 
5
- from edsl.prompts.Prompt import Prompt
6
5
  from edsl.utilities.decorators import jupyter_nb_handler
7
6
  from edsl.data_transfer_models import AgentResponseDict
8
7
 
9
- from edsl.data.Cache import Cache
10
-
11
- from edsl.questions.QuestionBase import QuestionBase
12
- from edsl.scenarios.Scenario import Scenario
13
- from edsl.surveys.MemoryPlan import MemoryPlan
14
- from edsl.language_models.LanguageModel import LanguageModel
8
+ if TYPE_CHECKING:
9
+ from edsl.prompts.Prompt import Prompt
10
+ from edsl.data.Cache import Cache
11
+ from edsl.questions.QuestionBase import QuestionBase
12
+ from edsl.scenarios.Scenario import Scenario
13
+ from edsl.surveys.MemoryPlan import MemoryPlan
14
+ from edsl.language_models.LanguageModel import LanguageModel
15
+ from edsl.surveys.Survey import Survey
16
+ from edsl.agents.Agent import Agent
15
17
 
16
18
  from edsl.data_transfer_models import EDSLResultObjectInput
17
19
  from edsl.agents.PromptConstructor import PromptConstructor
18
-
19
20
  from edsl.agents.prompt_helpers import PromptPlan
20
21
 
21
22
 
@@ -29,23 +30,20 @@ class InvigilatorBase(ABC):
29
30
  'Failed to get response'
30
31
 
31
32
  This returns an empty prompt because there is no memory the agent needs to have at q0.
32
-
33
-
34
33
  """
35
34
 
36
35
  def __init__(
37
36
  self,
38
37
  agent: "Agent",
39
- question: QuestionBase,
40
- scenario: Scenario,
41
- model: LanguageModel,
42
- memory_plan: MemoryPlan,
38
+ question: "QuestionBase",
39
+ scenario: "Scenario",
40
+ model: "LanguageModel",
41
+ memory_plan: "MemoryPlan",
43
42
  current_answers: dict,
44
43
  survey: Optional["Survey"],
45
- cache: Optional[Cache] = None,
44
+ cache: Optional["Cache"] = None,
46
45
  iteration: Optional[int] = 1,
47
46
  additional_prompt_data: Optional[dict] = None,
48
- sidecar_model: Optional[LanguageModel] = None,
49
47
  raise_validation_errors: Optional[bool] = True,
50
48
  prompt_plan: Optional["PromptPlan"] = None,
51
49
  ):
@@ -59,24 +57,23 @@ class InvigilatorBase(ABC):
59
57
  self.iteration = iteration
60
58
  self.additional_prompt_data = additional_prompt_data
61
59
  self.cache = cache
62
- self.sidecar_model = sidecar_model
63
60
  self.survey = survey
64
61
  self.raise_validation_errors = raise_validation_errors
62
+
65
63
  if prompt_plan is None:
66
64
  self.prompt_plan = PromptPlan()
67
65
  else:
68
66
  self.prompt_plan = prompt_plan
69
67
 
70
- self.raw_model_response = (
71
- None # placeholder for the raw response from the model
72
- )
68
+ # placeholder to store the raw model response
69
+ self.raw_model_response = None
73
70
 
74
71
  @property
75
72
  def prompt_constructor(self) -> PromptConstructor:
76
73
  """Return the prompt constructor."""
77
74
  return PromptConstructor(self, prompt_plan=self.prompt_plan)
78
75
 
79
- def to_dict(self):
76
+ def to_dict(self, include_cache=False) -> Dict[str, Any]:
80
77
  attributes = [
81
78
  "agent",
82
79
  "question",
@@ -86,10 +83,10 @@ class InvigilatorBase(ABC):
86
83
  "current_answers",
87
84
  "iteration",
88
85
  "additional_prompt_data",
89
- "cache",
90
- "sidecar_model",
91
86
  "survey",
92
87
  ]
88
+ if include_cache:
89
+ attributes.append("cache")
93
90
 
94
91
  def serialize_attribute(attr):
95
92
  value = getattr(self, attr)
@@ -104,43 +101,37 @@ class InvigilatorBase(ABC):
104
101
  return {attr: serialize_attribute(attr) for attr in attributes}
105
102
 
106
103
  @classmethod
107
- def from_dict(cls, data):
104
+ def from_dict(cls, data) -> "InvigilatorBase":
108
105
  from edsl.agents.Agent import Agent
109
106
  from edsl.questions import QuestionBase
110
107
  from edsl.scenarios.Scenario import Scenario
111
108
  from edsl.surveys.MemoryPlan import MemoryPlan
112
109
  from edsl.language_models.LanguageModel import LanguageModel
113
110
  from edsl.surveys.Survey import Survey
111
+ from edsl.data.Cache import Cache
112
+
113
+ attributes_to_classes = {
114
+ "agent": Agent,
115
+ "question": QuestionBase,
116
+ "scenario": Scenario,
117
+ "model": LanguageModel,
118
+ "memory_plan": MemoryPlan,
119
+ "survey": Survey,
120
+ "cache": Cache,
121
+ }
122
+ d = {}
123
+ for attr, cls_ in attributes_to_classes.items():
124
+ if attr in data and data[attr] is not None:
125
+ if attr not in data:
126
+ d[attr] = {}
127
+ else:
128
+ d[attr] = cls_.from_dict(data[attr])
114
129
 
115
- agent = Agent.from_dict(data["agent"])
116
- question = QuestionBase.from_dict(data["question"])
117
- scenario = Scenario.from_dict(data["scenario"])
118
- model = LanguageModel.from_dict(data["model"])
119
- memory_plan = MemoryPlan.from_dict(data["memory_plan"])
120
- survey = Survey.from_dict(data["survey"])
121
- current_answers = data["current_answers"]
122
- iteration = data["iteration"]
123
- additional_prompt_data = data["additional_prompt_data"]
124
- cache = Cache.from_dict(data["cache"])
125
-
126
- if data["sidecar_model"] is None:
127
- sidecar_model = None
128
- else:
129
- sidecar_model = LanguageModel.from_dict(data["sidecar_model"])
130
+ d["current_answers"] = data["current_answers"]
131
+ d["iteration"] = data["iteration"]
132
+ d["additional_prompt_data"] = data["additional_prompt_data"]
130
133
 
131
- return cls(
132
- agent=agent,
133
- question=question,
134
- scenario=scenario,
135
- model=model,
136
- memory_plan=memory_plan,
137
- current_answers=current_answers,
138
- survey=survey,
139
- iteration=iteration,
140
- additional_prompt_data=additional_prompt_data,
141
- cache=cache,
142
- sidecar_model=sidecar_model,
143
- )
134
+ d = cls(**d)
144
135
 
145
136
  def __repr__(self) -> str:
146
137
  """Return a string representation of the Invigilator.
@@ -149,9 +140,9 @@ class InvigilatorBase(ABC):
149
140
  'InvigilatorExample(...)'
150
141
 
151
142
  """
152
- return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)}, sidecarmodel={repr(self.sidecar_model)})"
143
+ return f"{self.__class__.__name__}(agent={repr(self.agent)}, question={repr(self.question)}, scneario={repr(self.scenario)}, model={repr(self.model)}, memory_plan={repr(self.memory_plan)}, current_answers={repr(self.current_answers)}, iteration{repr(self.iteration)}, additional_prompt_data={repr(self.additional_prompt_data)}, cache={repr(self.cache)})"
153
144
 
154
- def get_failed_task_result(self, failure_reason) -> EDSLResultObjectInput:
145
+ def get_failed_task_result(self, failure_reason: str) -> EDSLResultObjectInput:
155
146
  """Return an AgentResponseDict used in case the question-asking fails.
156
147
 
157
148
  Possible reasons include:
@@ -172,8 +163,9 @@ class InvigilatorBase(ABC):
172
163
  }
173
164
  return EDSLResultObjectInput(**data)
174
165
 
175
- def get_prompts(self) -> Dict[str, Prompt]:
166
+ def get_prompts(self) -> Dict[str, "Prompt"]:
176
167
  """Return the prompt used."""
168
+ from edsl.prompts.Prompt import Prompt
177
169
 
178
170
  return {
179
171
  "user_prompt": Prompt("NA"),
@@ -205,24 +197,25 @@ class InvigilatorBase(ABC):
205
197
  >>> InvigilatorBase.example()
206
198
  InvigilatorExample(...)
207
199
 
200
+ >>> InvigilatorBase.example().answer_question()
201
+ {'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
202
+
203
+ >>> InvigilatorBase.example(throw_an_exception=True).answer_question()
204
+ Traceback (most recent call last):
205
+ ...
206
+ Exception: This is a test error
208
207
  """
209
208
  from edsl.agents.Agent import Agent
210
- from edsl.questions import QuestionMultipleChoice
211
209
  from edsl.scenarios.Scenario import Scenario
212
- from edsl.language_models import LanguageModel
213
210
  from edsl.surveys.MemoryPlan import MemoryPlan
214
-
215
- from edsl.enums import InferenceServiceType
216
-
217
- from edsl import Model
211
+ from edsl.language_models.registry import Model
212
+ from edsl.surveys.Survey import Survey
218
213
 
219
214
  model = Model("test", canned_response="SPAM!")
220
215
 
221
216
  if throw_an_exception:
222
- model.throw_an_exception = True
217
+ model.throw_exception = True
223
218
  agent = Agent.example()
224
- # question = QuestionMultipleChoice.example()
225
- from edsl.surveys import Survey
226
219
 
227
220
  if not survey:
228
221
  survey = Survey.example()
@@ -232,14 +225,10 @@ class InvigilatorBase(ABC):
232
225
 
233
226
  question = question or survey.questions[0]
234
227
  scenario = scenario or Scenario.example()
235
- # memory_plan = None #memory_plan = MemoryPlan()
236
- from edsl import Survey
237
-
238
228
  memory_plan = MemoryPlan(survey=survey)
239
229
  current_answers = None
240
- from edsl.agents.PromptConstructor import PromptConstructor
241
230
 
242
- class InvigilatorExample(InvigilatorBase):
231
+ class InvigilatorExample(cls):
243
232
  """An example invigilator."""
244
233
 
245
234
  async def async_answer_question(self):
@@ -1,10 +1,18 @@
1
1
  from __future__ import annotations
2
- from typing import Dict, Any, Optional, Set
3
-
4
- from jinja2 import Environment, meta
2
+ from typing import Dict, Any, Optional, Set, Union, TYPE_CHECKING
3
+ from functools import cached_property
5
4
 
6
5
  from edsl.prompts.Prompt import Prompt
7
- from edsl.agents.prompt_helpers import PromptPlan
6
+
7
+ from .prompt_helpers import PromptPlan
8
+ from .QuestionTemplateReplacementsBuilder import (
9
+ QuestionTemplateReplacementsBuilder,
10
+ )
11
+ from .QuestionOptionProcessor import QuestionOptionProcessor
12
+
13
+ if TYPE_CHECKING:
14
+ from edsl.agents.InvigilatorBase import InvigilatorBase
15
+ from edsl.questions.QuestionBase import QuestionBase
8
16
 
9
17
 
10
18
  class PlaceholderAnswer:
@@ -18,39 +26,34 @@ class PlaceholderAnswer:
18
26
  return ""
19
27
 
20
28
  def __str__(self):
21
- return "<<PlaceholderAnswer>>"
29
+ return f"<<{self.__class__.__name__}>>"
22
30
 
23
31
  def __repr__(self):
24
- return "<<PlaceholderAnswer>>"
32
+ return f"<<{self.__class__.__name__}>>"
25
33
 
26
34
 
27
- def get_jinja2_variables(template_str: str) -> Set[str]:
28
- """
29
- Extracts all variable names from a Jinja2 template using Jinja2's built-in parsing.
35
+ class PlaceholderComment(PlaceholderAnswer):
36
+ pass
30
37
 
31
- Args:
32
- template_str (str): The Jinja2 template string
33
38
 
34
- Returns:
35
- Set[str]: A set of variable names found in the template
36
- """
37
- env = Environment()
38
- ast = env.parse(template_str)
39
- return meta.find_undeclared_variables(ast)
39
+ class PlaceholderGeneratedTokens(PlaceholderAnswer):
40
+ pass
40
41
 
41
42
 
42
43
  class PromptConstructor:
43
44
  """
45
+ This class constructs the prompts for the language model.
46
+
44
47
  The pieces of a prompt are:
45
48
  - The agent instructions - "You are answering questions as if you were a human. Do not break character."
46
49
  - The persona prompt - "You are an agent with the following persona: {'age': 22, 'hair': 'brown', 'height': 5.5}"
47
50
  - The question instructions - "You are being asked the following question: Do you like school? The options are 0: yes 1: no Return a valid JSON formatted like this, selecting only the number of the option: {"answer": <put answer code here>, "comment": "<put explanation here>"} Only 1 option may be selected."
48
51
  - The memory prompt - "Before the question you are now answering, you already answered the following question(s): Question: Do you like school? Answer: Prior answer"
49
-
50
- This is mixed into the Invigilator class.
51
52
  """
52
53
 
53
- def __init__(self, invigilator, prompt_plan: Optional["PromptPlan"] = None):
54
+ def __init__(
55
+ self, invigilator: "InvigilatorBase", prompt_plan: Optional["PromptPlan"] = None
56
+ ):
54
57
  self.invigilator = invigilator
55
58
  self.agent = invigilator.agent
56
59
  self.question = invigilator.question
@@ -61,20 +64,11 @@ class PromptConstructor:
61
64
  self.memory_plan = invigilator.memory_plan
62
65
  self.prompt_plan = prompt_plan or PromptPlan()
63
66
 
64
- @property
65
- def scenario_file_keys(self) -> list:
66
- """We need to find all the keys in the scenario that refer to FileStore objects.
67
- These will be used to append to the prompt a list of files that are part of the scenario.
68
- """
69
- from edsl.scenarios.FileStore import FileStore
70
-
71
- file_entries = []
72
- for key, value in self.scenario.items():
73
- if isinstance(value, FileStore):
74
- file_entries.append(key)
75
- return file_entries
67
+ def get_question_options(self, question_data):
68
+ """Get the question options."""
69
+ return QuestionOptionProcessor(self).get_question_options(question_data)
76
70
 
77
- @property
71
+ @cached_property
78
72
  def agent_instructions_prompt(self) -> Prompt:
79
73
  """
80
74
  >>> from edsl.agents.InvigilatorBase import InvigilatorBase
@@ -82,14 +76,14 @@ class PromptConstructor:
82
76
  >>> i.prompt_constructor.agent_instructions_prompt
83
77
  Prompt(text=\"""You are answering questions as if you were a human. Do not break character.\""")
84
78
  """
85
- from edsl import Agent
79
+ from edsl.agents.Agent import Agent
86
80
 
87
81
  if self.agent == Agent(): # if agent is empty, then return an empty prompt
88
82
  return Prompt(text="")
89
83
 
90
84
  return Prompt(text=self.agent.instruction)
91
85
 
92
- @property
86
+ @cached_property
93
87
  def agent_persona_prompt(self) -> Prompt:
94
88
  """
95
89
  >>> from edsl.agents.InvigilatorBase import InvigilatorBase
@@ -97,159 +91,93 @@ class PromptConstructor:
97
91
  >>> i.prompt_constructor.agent_persona_prompt
98
92
  Prompt(text=\"""Your traits: {'age': 22, 'hair': 'brown', 'height': 5.5}\""")
99
93
  """
100
- from edsl import Agent
94
+ from edsl.agents.Agent import Agent
101
95
 
102
96
  if self.agent == Agent(): # if agent is empty, then return an empty prompt
103
97
  return Prompt(text="")
104
98
 
105
99
  return self.agent.prompt()
106
100
 
107
- def prior_answers_dict(self) -> dict:
108
- # this is all questions
109
- d = self.survey.question_names_to_questions()
110
- # This attaches the answer to the question
111
- for question in d:
112
- if question in self.current_answers:
113
- d[question].answer = self.current_answers[question]
114
- else:
115
- d[question].answer = PlaceholderAnswer()
101
+ def prior_answers_dict(self) -> dict[str, "QuestionBase"]:
102
+ """This is a dictionary of prior answers, if they exist."""
103
+ return self._add_answers(
104
+ self.survey.question_names_to_questions(), self.current_answers
105
+ )
116
106
 
117
- # if (new_question := question.split("_comment")[0]) in d:
118
- # d[new_question].comment = answer
119
- # d[question].answer = PlaceholderAnswer()
107
+ @staticmethod
108
+ def _extract_quetion_and_entry_type(key_entry) -> tuple[str, str]:
109
+ """
110
+ Extracts the question name and type for the current answer dictionary key entry.
111
+
112
+ >>> PromptConstructor._extract_quetion_and_entry_type("q0")
113
+ ('q0', 'answer')
114
+ >>> PromptConstructor._extract_quetion_and_entry_type("q0_comment")
115
+ ('q0', 'comment')
116
+ >>> PromptConstructor._extract_quetion_and_entry_type("q0_alternate_generated_tokens")
117
+ ('q0_alternate', 'generated_tokens')
118
+ """
119
+ split_list = key_entry.rsplit("_", maxsplit=1)
120
+ if len(split_list) == 1:
121
+ question_name = split_list[0]
122
+ entry_type = "answer"
123
+ else:
124
+ if split_list[1] == "comment":
125
+ question_name = split_list[0]
126
+ entry_type = "comment"
127
+ elif split_list[1] == "tokens": # it's actually 'generated_tokens'
128
+ question_name = key_entry.replace("_generated_tokens", "")
129
+ entry_type = "generated_tokens"
130
+ else:
131
+ question_name = key_entry
132
+ entry_type = "answer"
133
+ return question_name, entry_type
120
134
 
121
- # breakpoint()
135
+ @staticmethod
136
+ def _augmented_answers_dict(current_answers: dict):
137
+ """
138
+ >>> PromptConstructor._augmented_answers_dict({"q0": "LOVE IT!", "q0_comment": "I love school!"})
139
+ {'q0': {'answer': 'LOVE IT!', 'comment': 'I love school!'}}
140
+ """
141
+ d = {}
142
+ for key, value in current_answers.items():
143
+ (
144
+ question_name,
145
+ entry_type,
146
+ ) = PromptConstructor._extract_quetion_and_entry_type(key)
147
+ if question_name not in d:
148
+ d[question_name] = {}
149
+ d[question_name][entry_type] = value
122
150
  return d
123
151
 
124
- @property
125
- def question_file_keys(self):
126
- raw_question_text = self.question.question_text
127
- variables = get_jinja2_variables(raw_question_text)
128
- question_file_keys = []
129
- for var in variables:
130
- if var in self.scenario_file_keys:
131
- question_file_keys.append(var)
132
- return question_file_keys
133
-
134
- def build_replacement_dict(self, question_data: dict):
152
+ @staticmethod
153
+ def _add_answers(answer_dict: dict, current_answers) -> dict[str, "QuestionBase"]:
135
154
  """
136
- Builds a dictionary of replacement values by combining multiple data sources.
155
+ >>> from edsl import QuestionFreeText
156
+ >>> d = {"q0": QuestionFreeText(question_text="Do you like school?", question_name = "q0")}
157
+ >>> current_answers = {"q0": "LOVE IT!"}
158
+ >>> PromptConstructor._add_answers(d, current_answers)['q0'].answer
159
+ 'LOVE IT!'
137
160
  """
138
- # File references dictionary
139
- file_refs = {key: f"<see file {key}>" for key in self.scenario_file_keys}
140
-
141
- # Scenario items excluding file keys
142
- scenario_items = {
143
- k: v for k, v in self.scenario.items() if k not in self.scenario_file_keys
144
- }
145
-
146
- # Question settings with defaults
147
- question_settings = {
148
- "use_code": getattr(self.question, "_use_code", True),
149
- "include_comment": getattr(self.question, "_include_comment", False),
150
- }
151
-
152
- # Combine all dictionaries using dict.update() for clarity
153
- replacement_dict = {}
154
- for d in [
155
- file_refs,
156
- question_data,
157
- scenario_items,
158
- self.prior_answers_dict(),
159
- {"agent": self.agent},
160
- question_settings,
161
- ]:
162
- replacement_dict.update(d)
163
-
164
- return replacement_dict
165
-
166
- def _get_question_options(self, question_data):
167
- question_options_entry = question_data.get("question_options", None)
168
- question_options = question_options_entry
169
-
170
- placeholder = ["<< Option 1 - Placholder >>", "<< Option 2 - Placholder >>"]
171
-
172
- # print("Question options entry: ", question_options_entry)
173
-
174
- if isinstance(question_options_entry, str):
175
- env = Environment()
176
- parsed_content = env.parse(question_options_entry)
177
- question_option_key = list(meta.find_undeclared_variables(parsed_content))[
178
- 0
179
- ]
180
- if isinstance(self.scenario.get(question_option_key), list):
181
- question_options = self.scenario.get(question_option_key)
182
-
183
- # might be getting it from the prior answers
184
- if self.prior_answers_dict().get(question_option_key) is not None:
185
- prior_question = self.prior_answers_dict().get(question_option_key)
186
- if hasattr(prior_question, "answer"):
187
- if isinstance(prior_question.answer, list):
188
- question_options = prior_question.answer
189
- else:
190
- question_options = placeholder
191
- else:
192
- question_options = placeholder
193
-
194
- return question_options
195
-
196
- def build_question_instructions_prompt(self):
197
- """Buils the question instructions prompt."""
198
-
199
- question_prompt = Prompt(self.question.get_instructions(model=self.model.model))
200
-
201
- # Get the data for the question - this is a dictionary of the question data
202
- # e.g., {'question_text': 'Do you like school?', 'question_name': 'q0', 'question_options': ['yes', 'no']}
203
- question_data = self.question.data.copy()
204
-
205
- if (
206
- "question_options" in question_data
207
- ): # is this a question with question options?
208
- question_options = self._get_question_options(question_data)
209
- question_data["question_options"] = question_options
161
+ augmented_answers = PromptConstructor._augmented_answers_dict(current_answers)
210
162
 
211
- replacement_dict = self.build_replacement_dict(question_data)
212
- rendered_instructions = question_prompt.render(replacement_dict)
213
-
214
- # is there anything left to render?
215
- undefined_template_variables = (
216
- rendered_instructions.undefined_template_variables({})
217
- )
218
-
219
- # Check if it's the name of a question in the survey
220
- for question_name in self.survey.question_names:
221
- if question_name in undefined_template_variables:
222
- print(
223
- "Question name found in undefined_template_variables: ",
224
- question_name,
225
- )
226
-
227
- if undefined_template_variables:
228
- msg = f"Question instructions still has variables: {undefined_template_variables}."
229
- import warnings
230
-
231
- warnings.warn(msg)
232
- # raise QuestionScenarioRenderError(
233
- # f"Question instructions still has variables: {undefined_template_variables}."
234
- # )
235
-
236
- # Check if question has instructions - these are instructions in a Survey that can apply to multiple follow-on questions
237
- relevant_instructions = self.survey.relevant_instructions(
238
- self.question.question_name
239
- )
240
-
241
- if relevant_instructions != []:
242
- # preamble_text = Prompt(
243
- # text="You were given the following instructions: "
244
- # )
245
- preamble_text = Prompt(text="")
246
- for instruction in relevant_instructions:
247
- preamble_text += instruction.text
248
- rendered_instructions = preamble_text + rendered_instructions
249
-
250
- return rendered_instructions
163
+ for question in answer_dict:
164
+ if question in augmented_answers:
165
+ for entry_type, value in augmented_answers[question].items():
166
+ setattr(answer_dict[question], entry_type, value)
167
+ else:
168
+ answer_dict[question].answer = PlaceholderAnswer()
169
+ answer_dict[question].comment = PlaceholderComment()
170
+ answer_dict[question].generated_tokens = PlaceholderGeneratedTokens()
171
+ return answer_dict
172
+
173
+ @cached_property
174
+ def question_file_keys(self) -> list:
175
+ """Extracts the file keys from the question text.
176
+ It checks if the variables in the question text are in the scenario file keys.
177
+ """
178
+ return QuestionTemplateReplacementsBuilder(self).question_file_keys()
251
179
 
252
- @property
180
+ @cached_property
253
181
  def question_instructions_prompt(self) -> Prompt:
254
182
  """
255
183
  >>> from edsl.agents.InvigilatorBase import InvigilatorBase
@@ -258,25 +186,24 @@ class PromptConstructor:
258
186
  Prompt(text=\"""...
259
187
  ...
260
188
  """
261
- if not hasattr(self, "_question_instructions_prompt"):
262
- self._question_instructions_prompt = (
263
- self.build_question_instructions_prompt()
264
- )
189
+ return self.build_question_instructions_prompt()
190
+
191
+ def build_question_instructions_prompt(self) -> Prompt:
192
+ """Buils the question instructions prompt."""
193
+ from edsl.agents.QuestionInstructionPromptBuilder import (
194
+ QuestionInstructionPromptBuilder,
195
+ )
265
196
 
266
- return self._question_instructions_prompt
197
+ return QuestionInstructionPromptBuilder(self).build()
267
198
 
268
- @property
199
+ @cached_property
269
200
  def prior_question_memory_prompt(self) -> Prompt:
270
- if not hasattr(self, "_prior_question_memory_prompt"):
271
- from edsl.prompts.Prompt import Prompt
272
-
273
- memory_prompt = Prompt(text="")
274
- if self.memory_plan is not None:
275
- memory_prompt += self.create_memory_prompt(
276
- self.question.question_name
277
- ).render(self.scenario | self.prior_answers_dict())
278
- self._prior_question_memory_prompt = memory_prompt
279
- return self._prior_question_memory_prompt
201
+ memory_prompt = Prompt(text="")
202
+ if self.memory_plan is not None:
203
+ memory_prompt += self.create_memory_prompt(
204
+ self.question.question_name
205
+ ).render(self.scenario | self.prior_answers_dict())
206
+ return memory_prompt
280
207
 
281
208
  def create_memory_prompt(self, question_name: str) -> Prompt:
282
209
  """Create a memory for the agent.
@@ -295,24 +222,6 @@ class PromptConstructor:
295
222
  question_name, self.current_answers
296
223
  )
297
224
 
298
- def construct_system_prompt(self) -> Prompt:
299
- """Construct the system prompt for the LLM call."""
300
- import warnings
301
-
302
- warnings.warn(
303
- "This method is deprecated. Use get_prompts instead.", DeprecationWarning
304
- )
305
- return self.get_prompts()["system_prompt"]
306
-
307
- def construct_user_prompt(self) -> Prompt:
308
- """Construct the user prompt for the LLM call."""
309
- import warnings
310
-
311
- warnings.warn(
312
- "This method is deprecated. Use get_prompts instead.", DeprecationWarning
313
- )
314
- return self.get_prompts()["user_prompt"]
315
-
316
225
  def get_prompts(self) -> Dict[str, Prompt]:
317
226
  """Get both prompts for the LLM call.
318
227
 
@@ -323,7 +232,6 @@ class PromptConstructor:
323
232
  >>> i.get_prompts()
324
233
  {'user_prompt': ..., 'system_prompt': ...}
325
234
  """
326
- # breakpoint()
327
235
  prompts = self.prompt_plan.get_prompts(
328
236
  agent_instructions=self.agent_instructions_prompt,
329
237
  agent_persona=self.agent_persona_prompt,
@@ -337,16 +245,6 @@ class PromptConstructor:
337
245
  prompts["files_list"] = files_list
338
246
  return prompts
339
247
 
340
- def _get_scenario_with_image(self) -> Scenario:
341
- """This is a helper function to get a scenario with an image, for testing purposes."""
342
- from edsl import Scenario
343
-
344
- try:
345
- scenario = Scenario.from_image("../../static/logo.png")
346
- except FileNotFoundError:
347
- scenario = Scenario.from_image("static/logo.png")
348
- return scenario
349
-
350
248
 
351
249
  if __name__ == "__main__":
352
250
  import doctest