edsl 0.1.37.dev2__py3-none-any.whl → 0.1.37.dev4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (257) hide show
  1. edsl/Base.py +303 -303
  2. edsl/BaseDiff.py +260 -260
  3. edsl/TemplateLoader.py +24 -24
  4. edsl/__init__.py +48 -48
  5. edsl/__version__.py +1 -1
  6. edsl/agents/Agent.py +804 -804
  7. edsl/agents/AgentList.py +345 -345
  8. edsl/agents/Invigilator.py +222 -222
  9. edsl/agents/InvigilatorBase.py +305 -305
  10. edsl/agents/PromptConstructor.py +312 -312
  11. edsl/agents/__init__.py +3 -3
  12. edsl/agents/descriptors.py +86 -86
  13. edsl/agents/prompt_helpers.py +129 -129
  14. edsl/auto/AutoStudy.py +117 -117
  15. edsl/auto/StageBase.py +230 -230
  16. edsl/auto/StageGenerateSurvey.py +178 -178
  17. edsl/auto/StageLabelQuestions.py +125 -125
  18. edsl/auto/StagePersona.py +61 -61
  19. edsl/auto/StagePersonaDimensionValueRanges.py +88 -88
  20. edsl/auto/StagePersonaDimensionValues.py +74 -74
  21. edsl/auto/StagePersonaDimensions.py +69 -69
  22. edsl/auto/StageQuestions.py +73 -73
  23. edsl/auto/SurveyCreatorPipeline.py +21 -21
  24. edsl/auto/utilities.py +224 -224
  25. edsl/base/Base.py +289 -289
  26. edsl/config.py +149 -149
  27. edsl/conjure/AgentConstructionMixin.py +152 -152
  28. edsl/conjure/Conjure.py +62 -62
  29. edsl/conjure/InputData.py +659 -659
  30. edsl/conjure/InputDataCSV.py +48 -48
  31. edsl/conjure/InputDataMixinQuestionStats.py +182 -182
  32. edsl/conjure/InputDataPyRead.py +91 -91
  33. edsl/conjure/InputDataSPSS.py +8 -8
  34. edsl/conjure/InputDataStata.py +8 -8
  35. edsl/conjure/QuestionOptionMixin.py +76 -76
  36. edsl/conjure/QuestionTypeMixin.py +23 -23
  37. edsl/conjure/RawQuestion.py +65 -65
  38. edsl/conjure/SurveyResponses.py +7 -7
  39. edsl/conjure/__init__.py +9 -9
  40. edsl/conjure/naming_utilities.py +263 -263
  41. edsl/conjure/utilities.py +201 -201
  42. edsl/conversation/Conversation.py +238 -238
  43. edsl/conversation/car_buying.py +58 -58
  44. edsl/conversation/mug_negotiation.py +81 -81
  45. edsl/conversation/next_speaker_utilities.py +93 -93
  46. edsl/coop/PriceFetcher.py +54 -54
  47. edsl/coop/__init__.py +2 -2
  48. edsl/coop/coop.py +827 -824
  49. edsl/coop/utils.py +131 -131
  50. edsl/data/Cache.py +527 -527
  51. edsl/data/CacheEntry.py +228 -228
  52. edsl/data/CacheHandler.py +149 -149
  53. edsl/data/RemoteCacheSync.py +97 -97
  54. edsl/data/SQLiteDict.py +292 -292
  55. edsl/data/__init__.py +4 -4
  56. edsl/data/orm.py +10 -10
  57. edsl/data_transfer_models.py +73 -73
  58. edsl/enums.py +173 -173
  59. edsl/exceptions/__init__.py +50 -50
  60. edsl/exceptions/agents.py +40 -40
  61. edsl/exceptions/configuration.py +16 -16
  62. edsl/exceptions/coop.py +10 -10
  63. edsl/exceptions/data.py +14 -14
  64. edsl/exceptions/general.py +34 -34
  65. edsl/exceptions/jobs.py +33 -33
  66. edsl/exceptions/language_models.py +63 -63
  67. edsl/exceptions/prompts.py +15 -15
  68. edsl/exceptions/questions.py +91 -91
  69. edsl/exceptions/results.py +26 -26
  70. edsl/exceptions/surveys.py +34 -34
  71. edsl/inference_services/AnthropicService.py +87 -87
  72. edsl/inference_services/AwsBedrock.py +120 -115
  73. edsl/inference_services/AzureAI.py +217 -217
  74. edsl/inference_services/DeepInfraService.py +18 -18
  75. edsl/inference_services/GoogleService.py +156 -156
  76. edsl/inference_services/GroqService.py +20 -20
  77. edsl/inference_services/InferenceServiceABC.py +147 -147
  78. edsl/inference_services/InferenceServicesCollection.py +74 -74
  79. edsl/inference_services/MistralAIService.py +123 -123
  80. edsl/inference_services/OllamaService.py +18 -18
  81. edsl/inference_services/OpenAIService.py +224 -224
  82. edsl/inference_services/TestService.py +89 -89
  83. edsl/inference_services/TogetherAIService.py +170 -170
  84. edsl/inference_services/models_available_cache.py +118 -118
  85. edsl/inference_services/rate_limits_cache.py +25 -25
  86. edsl/inference_services/registry.py +39 -39
  87. edsl/inference_services/write_available.py +10 -10
  88. edsl/jobs/Answers.py +56 -56
  89. edsl/jobs/Jobs.py +1135 -1112
  90. edsl/jobs/__init__.py +1 -1
  91. edsl/jobs/buckets/BucketCollection.py +63 -63
  92. edsl/jobs/buckets/ModelBuckets.py +65 -65
  93. edsl/jobs/buckets/TokenBucket.py +248 -248
  94. edsl/jobs/interviews/Interview.py +661 -661
  95. edsl/jobs/interviews/InterviewExceptionCollection.py +99 -99
  96. edsl/jobs/interviews/InterviewExceptionEntry.py +182 -182
  97. edsl/jobs/interviews/InterviewStatistic.py +63 -63
  98. edsl/jobs/interviews/InterviewStatisticsCollection.py +25 -25
  99. edsl/jobs/interviews/InterviewStatusDictionary.py +78 -78
  100. edsl/jobs/interviews/InterviewStatusLog.py +92 -92
  101. edsl/jobs/interviews/ReportErrors.py +66 -66
  102. edsl/jobs/interviews/interview_status_enum.py +9 -9
  103. edsl/jobs/runners/JobsRunnerAsyncio.py +338 -338
  104. edsl/jobs/runners/JobsRunnerStatus.py +332 -332
  105. edsl/jobs/tasks/QuestionTaskCreator.py +242 -242
  106. edsl/jobs/tasks/TaskCreators.py +64 -64
  107. edsl/jobs/tasks/TaskHistory.py +441 -441
  108. edsl/jobs/tasks/TaskStatusLog.py +23 -23
  109. edsl/jobs/tasks/task_status_enum.py +163 -163
  110. edsl/jobs/tokens/InterviewTokenUsage.py +27 -27
  111. edsl/jobs/tokens/TokenUsage.py +34 -34
  112. edsl/language_models/LanguageModel.py +718 -718
  113. edsl/language_models/ModelList.py +102 -102
  114. edsl/language_models/RegisterLanguageModelsMeta.py +184 -184
  115. edsl/language_models/__init__.py +2 -2
  116. edsl/language_models/fake_openai_call.py +15 -15
  117. edsl/language_models/fake_openai_service.py +61 -61
  118. edsl/language_models/registry.py +137 -137
  119. edsl/language_models/repair.py +156 -156
  120. edsl/language_models/unused/ReplicateBase.py +83 -83
  121. edsl/language_models/utilities.py +64 -64
  122. edsl/notebooks/Notebook.py +259 -259
  123. edsl/notebooks/__init__.py +1 -1
  124. edsl/prompts/Prompt.py +353 -353
  125. edsl/prompts/__init__.py +2 -2
  126. edsl/questions/AnswerValidatorMixin.py +289 -289
  127. edsl/questions/QuestionBase.py +616 -616
  128. edsl/questions/QuestionBaseGenMixin.py +161 -161
  129. edsl/questions/QuestionBasePromptsMixin.py +266 -266
  130. edsl/questions/QuestionBudget.py +227 -227
  131. edsl/questions/QuestionCheckBox.py +359 -359
  132. edsl/questions/QuestionExtract.py +183 -183
  133. edsl/questions/QuestionFreeText.py +114 -114
  134. edsl/questions/QuestionFunctional.py +159 -159
  135. edsl/questions/QuestionList.py +231 -231
  136. edsl/questions/QuestionMultipleChoice.py +286 -286
  137. edsl/questions/QuestionNumerical.py +153 -153
  138. edsl/questions/QuestionRank.py +324 -324
  139. edsl/questions/Quick.py +41 -41
  140. edsl/questions/RegisterQuestionsMeta.py +71 -71
  141. edsl/questions/ResponseValidatorABC.py +174 -174
  142. edsl/questions/SimpleAskMixin.py +73 -73
  143. edsl/questions/__init__.py +26 -26
  144. edsl/questions/compose_questions.py +98 -98
  145. edsl/questions/decorators.py +21 -21
  146. edsl/questions/derived/QuestionLikertFive.py +76 -76
  147. edsl/questions/derived/QuestionLinearScale.py +87 -87
  148. edsl/questions/derived/QuestionTopK.py +91 -91
  149. edsl/questions/derived/QuestionYesNo.py +82 -82
  150. edsl/questions/descriptors.py +418 -418
  151. edsl/questions/prompt_templates/question_budget.jinja +13 -13
  152. edsl/questions/prompt_templates/question_checkbox.jinja +32 -32
  153. edsl/questions/prompt_templates/question_extract.jinja +11 -11
  154. edsl/questions/prompt_templates/question_free_text.jinja +3 -3
  155. edsl/questions/prompt_templates/question_linear_scale.jinja +11 -11
  156. edsl/questions/prompt_templates/question_list.jinja +17 -17
  157. edsl/questions/prompt_templates/question_multiple_choice.jinja +33 -33
  158. edsl/questions/prompt_templates/question_numerical.jinja +36 -36
  159. edsl/questions/question_registry.py +147 -147
  160. edsl/questions/settings.py +12 -12
  161. edsl/questions/templates/budget/answering_instructions.jinja +7 -7
  162. edsl/questions/templates/budget/question_presentation.jinja +7 -7
  163. edsl/questions/templates/checkbox/answering_instructions.jinja +10 -10
  164. edsl/questions/templates/checkbox/question_presentation.jinja +22 -22
  165. edsl/questions/templates/extract/answering_instructions.jinja +7 -7
  166. edsl/questions/templates/likert_five/answering_instructions.jinja +10 -10
  167. edsl/questions/templates/likert_five/question_presentation.jinja +11 -11
  168. edsl/questions/templates/linear_scale/answering_instructions.jinja +5 -5
  169. edsl/questions/templates/linear_scale/question_presentation.jinja +5 -5
  170. edsl/questions/templates/list/answering_instructions.jinja +3 -3
  171. edsl/questions/templates/list/question_presentation.jinja +5 -5
  172. edsl/questions/templates/multiple_choice/answering_instructions.jinja +9 -9
  173. edsl/questions/templates/multiple_choice/question_presentation.jinja +11 -11
  174. edsl/questions/templates/numerical/answering_instructions.jinja +6 -6
  175. edsl/questions/templates/numerical/question_presentation.jinja +6 -6
  176. edsl/questions/templates/rank/answering_instructions.jinja +11 -11
  177. edsl/questions/templates/rank/question_presentation.jinja +15 -15
  178. edsl/questions/templates/top_k/answering_instructions.jinja +8 -8
  179. edsl/questions/templates/top_k/question_presentation.jinja +22 -22
  180. edsl/questions/templates/yes_no/answering_instructions.jinja +6 -6
  181. edsl/questions/templates/yes_no/question_presentation.jinja +11 -11
  182. edsl/results/Dataset.py +293 -293
  183. edsl/results/DatasetExportMixin.py +693 -693
  184. edsl/results/DatasetTree.py +145 -145
  185. edsl/results/Result.py +435 -435
  186. edsl/results/Results.py +1160 -1160
  187. edsl/results/ResultsDBMixin.py +238 -238
  188. edsl/results/ResultsExportMixin.py +43 -43
  189. edsl/results/ResultsFetchMixin.py +33 -33
  190. edsl/results/ResultsGGMixin.py +121 -121
  191. edsl/results/ResultsToolsMixin.py +98 -98
  192. edsl/results/Selector.py +118 -118
  193. edsl/results/__init__.py +2 -2
  194. edsl/results/tree_explore.py +115 -115
  195. edsl/scenarios/FileStore.py +458 -458
  196. edsl/scenarios/Scenario.py +510 -510
  197. edsl/scenarios/ScenarioHtmlMixin.py +59 -59
  198. edsl/scenarios/ScenarioList.py +1101 -1101
  199. edsl/scenarios/ScenarioListExportMixin.py +52 -52
  200. edsl/scenarios/ScenarioListPdfMixin.py +261 -261
  201. edsl/scenarios/__init__.py +4 -4
  202. edsl/shared.py +1 -1
  203. edsl/study/ObjectEntry.py +173 -173
  204. edsl/study/ProofOfWork.py +113 -113
  205. edsl/study/SnapShot.py +80 -80
  206. edsl/study/Study.py +528 -528
  207. edsl/study/__init__.py +4 -4
  208. edsl/surveys/DAG.py +148 -148
  209. edsl/surveys/Memory.py +31 -31
  210. edsl/surveys/MemoryPlan.py +244 -244
  211. edsl/surveys/Rule.py +324 -324
  212. edsl/surveys/RuleCollection.py +387 -387
  213. edsl/surveys/Survey.py +1772 -1772
  214. edsl/surveys/SurveyCSS.py +261 -261
  215. edsl/surveys/SurveyExportMixin.py +259 -259
  216. edsl/surveys/SurveyFlowVisualizationMixin.py +121 -121
  217. edsl/surveys/SurveyQualtricsImport.py +284 -284
  218. edsl/surveys/__init__.py +3 -3
  219. edsl/surveys/base.py +53 -53
  220. edsl/surveys/descriptors.py +56 -56
  221. edsl/surveys/instructions/ChangeInstruction.py +47 -47
  222. edsl/surveys/instructions/Instruction.py +51 -51
  223. edsl/surveys/instructions/InstructionCollection.py +77 -77
  224. edsl/templates/error_reporting/base.html +23 -23
  225. edsl/templates/error_reporting/exceptions_by_model.html +34 -34
  226. edsl/templates/error_reporting/exceptions_by_question_name.html +16 -16
  227. edsl/templates/error_reporting/exceptions_by_type.html +16 -16
  228. edsl/templates/error_reporting/interview_details.html +115 -115
  229. edsl/templates/error_reporting/interviews.html +9 -9
  230. edsl/templates/error_reporting/overview.html +4 -4
  231. edsl/templates/error_reporting/performance_plot.html +1 -1
  232. edsl/templates/error_reporting/report.css +73 -73
  233. edsl/templates/error_reporting/report.html +117 -117
  234. edsl/templates/error_reporting/report.js +25 -25
  235. edsl/tools/__init__.py +1 -1
  236. edsl/tools/clusters.py +192 -192
  237. edsl/tools/embeddings.py +27 -27
  238. edsl/tools/embeddings_plotting.py +118 -118
  239. edsl/tools/plotting.py +112 -112
  240. edsl/tools/summarize.py +18 -18
  241. edsl/utilities/SystemInfo.py +28 -28
  242. edsl/utilities/__init__.py +22 -22
  243. edsl/utilities/ast_utilities.py +25 -25
  244. edsl/utilities/data/Registry.py +6 -6
  245. edsl/utilities/data/__init__.py +1 -1
  246. edsl/utilities/data/scooter_results.json +1 -1
  247. edsl/utilities/decorators.py +77 -77
  248. edsl/utilities/gcp_bucket/cloud_storage.py +96 -96
  249. edsl/utilities/interface.py +627 -627
  250. edsl/utilities/repair_functions.py +28 -28
  251. edsl/utilities/restricted_python.py +70 -70
  252. edsl/utilities/utilities.py +391 -391
  253. {edsl-0.1.37.dev2.dist-info → edsl-0.1.37.dev4.dist-info}/LICENSE +21 -21
  254. {edsl-0.1.37.dev2.dist-info → edsl-0.1.37.dev4.dist-info}/METADATA +1 -1
  255. edsl-0.1.37.dev4.dist-info/RECORD +279 -0
  256. edsl-0.1.37.dev2.dist-info/RECORD +0 -279
  257. {edsl-0.1.37.dev2.dist-info → edsl-0.1.37.dev4.dist-info}/WHEEL +0 -0
edsl/results/Result.py CHANGED
@@ -1,435 +1,435 @@
1
- # """This module contains the Result class, which captures the result of one interview."""
2
- from __future__ import annotations
3
- from collections import UserDict
4
- from typing import Any, Type, Callable, Optional
5
- from collections import UserDict
6
- from edsl.Base import Base
7
- from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
8
-
9
-
10
- class PromptDict(UserDict):
11
- """A dictionary that is used to store the prompt for a given result."""
12
-
13
- def rich_print(self):
14
- """Display an object as a table."""
15
- from rich.table import Table
16
-
17
- table = Table(title="")
18
- table.add_column("Attribute", style="bold")
19
- table.add_column("Value")
20
-
21
- to_display = self
22
- for attr_name, attr_value in to_display.items():
23
- table.add_row(attr_name, repr(attr_value))
24
-
25
- return table
26
-
27
-
28
- def agent_namer_closure():
29
- """Return a function that can be used to name an agent."""
30
- agent_dict = {}
31
-
32
- def agent_namer(agent):
33
- """Return a name for an agent. If the agent has been named before, return the same name. Otherwise, return a new name."""
34
- nonlocal agent_dict
35
- agent_count = len(agent_dict)
36
- if id(agent) in agent_dict:
37
- return agent_dict[id(agent)]
38
- else:
39
- agent_dict[id(agent)] = f"Agent_{agent_count}"
40
- return agent_dict[id(agent)]
41
-
42
- return agent_namer
43
-
44
-
45
- agent_namer = agent_namer_closure()
46
-
47
-
48
- class Result(Base, UserDict):
49
- """
50
- This class captures the result of one interview.
51
-
52
- The answer dictionary has the structure:
53
-
54
- >>> import warnings
55
- >>> warnings.simplefilter("ignore", UserWarning)
56
- >>> Result.example().answer == {'how_feeling_yesterday': 'Great', 'how_feeling': 'OK'}
57
- True
58
-
59
- Its main data is an Agent, a Scenario, a Model, an Iteration, and an Answer.
60
- These are stored both in the UserDict and as attributes.
61
-
62
-
63
- """
64
-
65
- def __init__(
66
- self,
67
- agent: "Agent",
68
- scenario: "Scenario",
69
- model: Type["LanguageModel"],
70
- iteration: int,
71
- answer: str,
72
- prompt: dict[str, str] = None,
73
- raw_model_response=None,
74
- survey: Optional["Survey"] = None,
75
- question_to_attributes: Optional[dict] = None,
76
- generated_tokens: Optional[dict] = None,
77
- comments_dict: Optional[dict] = None,
78
- cache_used_dict: Optional[dict] = None,
79
- ):
80
- """Initialize a Result object.
81
-
82
- :param agent: The Agent object.
83
- :param scenario: The Scenario object.
84
- :param model: The LanguageModel object.
85
- :param iteration: The iteration number.
86
- :param answer: The answer string.
87
- :param prompt: A dictionary of prompts.
88
- :param raw_model_response: The raw model response.
89
-
90
- """
91
- if question_to_attributes is not None:
92
- question_to_attributes = question_to_attributes
93
- else:
94
- question_to_attributes = {}
95
-
96
- if survey is not None:
97
- question_to_attributes = {
98
- q.question_name: {
99
- "question_text": q.question_text,
100
- "question_type": q.question_type,
101
- "question_options": (
102
- None
103
- if not hasattr(q, "question_options")
104
- else q.question_options
105
- ),
106
- }
107
- for q in survey.questions
108
- }
109
-
110
- data = {
111
- "agent": agent,
112
- "scenario": scenario,
113
- "model": model,
114
- "iteration": iteration,
115
- "answer": answer,
116
- "prompt": prompt or {},
117
- "raw_model_response": raw_model_response or {},
118
- "question_to_attributes": question_to_attributes,
119
- "generated_tokens": generated_tokens or {},
120
- "comments_dict": comments_dict or {},
121
- }
122
- super().__init__(**data)
123
- # but also store the data as attributes
124
- self.agent = agent
125
- self.scenario = scenario
126
- self.model = model
127
- self.iteration = iteration
128
- self.answer = answer
129
- self.prompt = prompt or {}
130
- self.raw_model_response = raw_model_response or {}
131
- self.survey = survey
132
- self.question_to_attributes = question_to_attributes
133
- self.generated_tokens = generated_tokens
134
- self.comments_dict = comments_dict or {}
135
- self.cache_used_dict = cache_used_dict or {}
136
-
137
- self._combined_dict = None
138
- self._problem_keys = None
139
-
140
- ###############
141
- # Used in Results
142
- ###############
143
- @property
144
- def sub_dicts(self) -> dict[str, dict]:
145
- """Return a dictionary where keys are strings for each of the main class attributes/objects."""
146
- if self.agent.name is None:
147
- agent_name = agent_namer(self.agent)
148
- else:
149
- agent_name = self.agent.name
150
-
151
- # comments_dict = {k: v for k, v in self.answer.items() if k.endswith("_comment")}
152
- question_text_dict = {}
153
- question_options_dict = {}
154
- question_type_dict = {}
155
- for key, _ in self.answer.items():
156
- if key in self.question_to_attributes:
157
- # You might be tempted to just use the naked key
158
- # but this is a bad idea because it pollutes the namespace
159
- question_text_dict[
160
- key + "_question_text"
161
- ] = self.question_to_attributes[key]["question_text"]
162
- question_options_dict[
163
- key + "_question_options"
164
- ] = self.question_to_attributes[key]["question_options"]
165
- question_type_dict[
166
- key + "_question_type"
167
- ] = self.question_to_attributes[key]["question_type"]
168
-
169
- return {
170
- "agent": self.agent.traits
171
- | {"agent_name": agent_name}
172
- | {"agent_instruction": self.agent.instruction},
173
- "scenario": self.scenario,
174
- "model": self.model.parameters | {"model": self.model.model},
175
- "answer": self.answer,
176
- "prompt": self.prompt,
177
- "raw_model_response": self.raw_model_response,
178
- "iteration": {"iteration": self.iteration},
179
- "question_text": question_text_dict,
180
- "question_options": question_options_dict,
181
- "question_type": question_type_dict,
182
- "comment": self.comments_dict,
183
- "generated_tokens": self.generated_tokens,
184
- }
185
-
186
- def check_expression(self, expression) -> None:
187
- for key in self.problem_keys:
188
- if key in expression and not key + "." in expression:
189
- raise ValueError(
190
- f"Key by iself {key} is problematic. Use the full key {key + '.' + key} name instead."
191
- )
192
- return None
193
-
194
- def code(self):
195
- """Return a string of code that can be used to recreate the Result object."""
196
- raise NotImplementedError
197
-
198
- @property
199
- def problem_keys(self):
200
- """Return a list of keys that are problematic."""
201
- return self._problem_keys
202
-
203
- def _compute_combined_dict_and_problem_keys(self) -> None:
204
- combined = {}
205
- problem_keys = []
206
- for key, sub_dict in self.sub_dicts.items():
207
- combined.update(sub_dict)
208
- # in some cases, the sub_dict might have keys that conflict with the main dict
209
- if key in combined:
210
- # The key is already in the combined dict
211
- problem_keys = problem_keys + [key]
212
-
213
- combined.update({key: sub_dict})
214
- # I *think* this allows us to do do things like "answer.how_feelling" i.e., that the evaluator can use
215
- # dot notation to access the subdicts.
216
- self._combined_dict = combined
217
- self._problem_keys = problem_keys
218
-
219
- @property
220
- def combined_dict(self) -> dict[str, Any]:
221
- """Return a dictionary that includes all sub_dicts, but also puts the key-value pairs in each sub_dict as a key_value pair in the combined dictionary.
222
-
223
- >>> r = Result.example()
224
- >>> r.combined_dict['how_feeling']
225
- 'OK'
226
- """
227
- if self._combined_dict is None or self._problem_keys is None:
228
- self._compute_combined_dict_and_problem_keys()
229
- return self._combined_dict
230
-
231
- @property
232
- def problem_keys(self):
233
- """Return a list of keys that are problematic."""
234
- if self._combined_dict is None or self._problem_keys is None:
235
- self._compute_combined_dict_and_problem_keys()
236
- return self._problem_keys
237
-
238
- def get_value(self, data_type: str, key: str) -> Any:
239
- """Return the value for a given data type and key.
240
-
241
- >>> r = Result.example()
242
- >>> r.get_value("answer", "how_feeling")
243
- 'OK'
244
-
245
- - data types can be "agent", "scenario", "model", or "answer"
246
- - keys are relevant attributes of the Objects the data types represent
247
- """
248
- return self.sub_dicts[data_type][key]
249
-
250
- @property
251
- def key_to_data_type(self) -> dict[str, str]:
252
- """Return a dictionary where keys are object attributes and values are the data type (object) that the attribute is associated with.
253
-
254
- >>> r = Result.example()
255
- >>> r.key_to_data_type["how_feeling"]
256
- 'answer'
257
-
258
- """
259
- d = {}
260
- data_types = self.sub_dicts.keys()
261
- for data_type in data_types:
262
- for key in self.sub_dicts[data_type]:
263
- d[key] = data_type
264
- return d
265
-
266
- def rows(self, index) -> tuple[int, str, str, str]:
267
- """Return a generator of rows for the Result object."""
268
- for data_type, subdict in self.sub_dicts.items():
269
- for key, value in subdict.items():
270
- yield (index, data_type, key, str(value))
271
-
272
- def leaves(self):
273
- leaves = []
274
- for question_name, answer in self.answer.items():
275
- if not question_name.endswith("_comment"):
276
- leaves.append(
277
- {
278
- "question": f"({question_name}): "
279
- + str(
280
- self.question_to_attributes[question_name]["question_text"]
281
- ),
282
- "answer": answer,
283
- "comment": self.answer.get(question_name + "_comment", ""),
284
- "scenario": repr(self.scenario),
285
- "agent": repr(self.agent),
286
- "model": repr(self.model),
287
- "iteration": self.iteration,
288
- }
289
- )
290
- return leaves
291
-
292
- ###############
293
- # Useful
294
- ###############
295
- def copy(self) -> Result:
296
- """Return a copy of the Result object."""
297
- return Result.from_dict(self.to_dict())
298
-
299
- def __eq__(self, other) -> bool:
300
- """Return True if the Result object is equal to another Result object.
301
-
302
- >>> r = Result.example()
303
- >>> r == r
304
- True
305
-
306
- """
307
- return self.to_dict() == other.to_dict()
308
-
309
- ###############
310
- # Serialization
311
- ###############
312
- def _to_dict(self) -> dict[str, Any]:
313
- """Return a dictionary representation of the Result object."""
314
- d = {}
315
- for key, value in self.items():
316
- if hasattr(value, "to_dict"):
317
- d[key] = value.to_dict()
318
- else:
319
- d[key] = value
320
- if key == "prompt":
321
- new_prompt_dict = {}
322
- for prompt_name, prompt_obj in value.items():
323
- new_prompt_dict[prompt_name] = (
324
- prompt_obj
325
- if not hasattr(prompt_obj, "to_dict")
326
- else prompt_obj.to_dict()
327
- )
328
- d[key] = new_prompt_dict
329
- return d
330
-
331
- @add_edsl_version
332
- def to_dict(self) -> dict[str, Any]:
333
- """Return a dictionary representation of the Result object.
334
-
335
- >>> r = Result.example()
336
- >>> r.to_dict()['scenario']
337
- {'period': 'morning', 'edsl_version': '...', 'edsl_class_name': 'Scenario'}
338
- """
339
- return self._to_dict()
340
-
341
- def __hash__(self):
342
- """Return a hash of the Result object."""
343
- from edsl.utilities.utilities import dict_hash
344
-
345
- return dict_hash(self._to_dict())
346
-
347
- @classmethod
348
- @remove_edsl_version
349
- def from_dict(self, json_dict: dict) -> Result:
350
- """Return a Result object from a dictionary representation."""
351
-
352
- from edsl import Agent
353
- from edsl import Scenario
354
- from edsl.language_models.LanguageModel import LanguageModel
355
- from edsl.prompts.Prompt import Prompt
356
-
357
- prompt_data = json_dict.get("prompt", {})
358
- prompt_d = {}
359
- for prompt_name, prompt_obj in prompt_data.items():
360
- prompt_d[prompt_name] = Prompt.from_dict(prompt_obj)
361
-
362
- result = Result(
363
- agent=Agent.from_dict(json_dict["agent"]),
364
- scenario=Scenario.from_dict(json_dict["scenario"]),
365
- model=LanguageModel.from_dict(json_dict["model"]),
366
- iteration=json_dict["iteration"],
367
- answer=json_dict["answer"],
368
- prompt=prompt_d, # json_dict["prompt"],
369
- raw_model_response=json_dict.get(
370
- "raw_model_response", {"raw_model_response": "No raw model response"}
371
- ),
372
- question_to_attributes=json_dict.get("question_to_attributes", None),
373
- generated_tokens=json_dict.get("generated_tokens", {}),
374
- comments_dict=json_dict.get("comments_dict", {}),
375
- )
376
- return result
377
-
378
- def rich_print(self) -> None:
379
- """Display an object as a table."""
380
- # from edsl.utilities import print_dict_with_rich
381
- from rich import print
382
- from rich.table import Table
383
-
384
- table = Table(title="Result")
385
- table.add_column("Attribute", style="bold")
386
- table.add_column("Value")
387
-
388
- to_display = self.__dict__.copy()
389
- data = to_display.pop("data", None)
390
- for attr_name, attr_value in to_display.items():
391
- if hasattr(attr_value, "rich_print"):
392
- table.add_row(attr_name, attr_value.rich_print())
393
- elif isinstance(attr_value, dict):
394
- a = PromptDict(attr_value)
395
- table.add_row(attr_name, a.rich_print())
396
- else:
397
- table.add_row(attr_name, repr(attr_value))
398
- return table
399
-
400
- def __repr__(self):
401
- """Return a string representation of the Result object."""
402
- return f"Result(agent={repr(self.agent)}, scenario={repr(self.scenario)}, model={repr(self.model)}, iteration={self.iteration}, answer={repr(self.answer)}, prompt={repr(self.prompt)})"
403
-
404
- @classmethod
405
- def example(cls):
406
- """Return an example Result object."""
407
- from edsl.results.Results import Results
408
-
409
- return Results.example()[0]
410
-
411
- def score(self, scoring_function: Callable) -> Any:
412
- """Score the result using a passed-in scoring function.
413
-
414
- >>> def f(status): return 1 if status == 'Joyful' else 0
415
- >>> Result.example().score(f)
416
- 1
417
- """
418
- import inspect
419
-
420
- signature = inspect.signature(scoring_function)
421
- params = {}
422
- for k, v in signature.parameters.items():
423
- if k in self.combined_dict:
424
- params[k] = self.combined_dict[k]
425
- elif v.default is not v.empty:
426
- params[k] = v.default
427
- else:
428
- raise ValueError(f"Parameter {k} not found in Result object")
429
- return scoring_function(**params)
430
-
431
-
432
- if __name__ == "__main__":
433
- import doctest
434
-
435
- doctest.testmod(optionflags=doctest.ELLIPSIS)
1
+ # """This module contains the Result class, which captures the result of one interview."""
2
+ from __future__ import annotations
3
+ from collections import UserDict
4
+ from typing import Any, Type, Callable, Optional
5
+ from collections import UserDict
6
+ from edsl.Base import Base
7
+ from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
8
+
9
+
10
+ class PromptDict(UserDict):
11
+ """A dictionary that is used to store the prompt for a given result."""
12
+
13
+ def rich_print(self):
14
+ """Display an object as a table."""
15
+ from rich.table import Table
16
+
17
+ table = Table(title="")
18
+ table.add_column("Attribute", style="bold")
19
+ table.add_column("Value")
20
+
21
+ to_display = self
22
+ for attr_name, attr_value in to_display.items():
23
+ table.add_row(attr_name, repr(attr_value))
24
+
25
+ return table
26
+
27
+
28
+ def agent_namer_closure():
29
+ """Return a function that can be used to name an agent."""
30
+ agent_dict = {}
31
+
32
+ def agent_namer(agent):
33
+ """Return a name for an agent. If the agent has been named before, return the same name. Otherwise, return a new name."""
34
+ nonlocal agent_dict
35
+ agent_count = len(agent_dict)
36
+ if id(agent) in agent_dict:
37
+ return agent_dict[id(agent)]
38
+ else:
39
+ agent_dict[id(agent)] = f"Agent_{agent_count}"
40
+ return agent_dict[id(agent)]
41
+
42
+ return agent_namer
43
+
44
+
45
+ agent_namer = agent_namer_closure()
46
+
47
+
48
+ class Result(Base, UserDict):
49
+ """
50
+ This class captures the result of one interview.
51
+
52
+ The answer dictionary has the structure:
53
+
54
+ >>> import warnings
55
+ >>> warnings.simplefilter("ignore", UserWarning)
56
+ >>> Result.example().answer == {'how_feeling_yesterday': 'Great', 'how_feeling': 'OK'}
57
+ True
58
+
59
+ Its main data is an Agent, a Scenario, a Model, an Iteration, and an Answer.
60
+ These are stored both in the UserDict and as attributes.
61
+
62
+
63
+ """
64
+
65
+ def __init__(
66
+ self,
67
+ agent: "Agent",
68
+ scenario: "Scenario",
69
+ model: Type["LanguageModel"],
70
+ iteration: int,
71
+ answer: str,
72
+ prompt: dict[str, str] = None,
73
+ raw_model_response=None,
74
+ survey: Optional["Survey"] = None,
75
+ question_to_attributes: Optional[dict] = None,
76
+ generated_tokens: Optional[dict] = None,
77
+ comments_dict: Optional[dict] = None,
78
+ cache_used_dict: Optional[dict] = None,
79
+ ):
80
+ """Initialize a Result object.
81
+
82
+ :param agent: The Agent object.
83
+ :param scenario: The Scenario object.
84
+ :param model: The LanguageModel object.
85
+ :param iteration: The iteration number.
86
+ :param answer: The answer string.
87
+ :param prompt: A dictionary of prompts.
88
+ :param raw_model_response: The raw model response.
89
+
90
+ """
91
+ if question_to_attributes is not None:
92
+ question_to_attributes = question_to_attributes
93
+ else:
94
+ question_to_attributes = {}
95
+
96
+ if survey is not None:
97
+ question_to_attributes = {
98
+ q.question_name: {
99
+ "question_text": q.question_text,
100
+ "question_type": q.question_type,
101
+ "question_options": (
102
+ None
103
+ if not hasattr(q, "question_options")
104
+ else q.question_options
105
+ ),
106
+ }
107
+ for q in survey.questions
108
+ }
109
+
110
+ data = {
111
+ "agent": agent,
112
+ "scenario": scenario,
113
+ "model": model,
114
+ "iteration": iteration,
115
+ "answer": answer,
116
+ "prompt": prompt or {},
117
+ "raw_model_response": raw_model_response or {},
118
+ "question_to_attributes": question_to_attributes,
119
+ "generated_tokens": generated_tokens or {},
120
+ "comments_dict": comments_dict or {},
121
+ }
122
+ super().__init__(**data)
123
+ # but also store the data as attributes
124
+ self.agent = agent
125
+ self.scenario = scenario
126
+ self.model = model
127
+ self.iteration = iteration
128
+ self.answer = answer
129
+ self.prompt = prompt or {}
130
+ self.raw_model_response = raw_model_response or {}
131
+ self.survey = survey
132
+ self.question_to_attributes = question_to_attributes
133
+ self.generated_tokens = generated_tokens
134
+ self.comments_dict = comments_dict or {}
135
+ self.cache_used_dict = cache_used_dict or {}
136
+
137
+ self._combined_dict = None
138
+ self._problem_keys = None
139
+
140
+ ###############
141
+ # Used in Results
142
+ ###############
143
+ @property
144
+ def sub_dicts(self) -> dict[str, dict]:
145
+ """Return a dictionary where keys are strings for each of the main class attributes/objects."""
146
+ if self.agent.name is None:
147
+ agent_name = agent_namer(self.agent)
148
+ else:
149
+ agent_name = self.agent.name
150
+
151
+ # comments_dict = {k: v for k, v in self.answer.items() if k.endswith("_comment")}
152
+ question_text_dict = {}
153
+ question_options_dict = {}
154
+ question_type_dict = {}
155
+ for key, _ in self.answer.items():
156
+ if key in self.question_to_attributes:
157
+ # You might be tempted to just use the naked key
158
+ # but this is a bad idea because it pollutes the namespace
159
+ question_text_dict[
160
+ key + "_question_text"
161
+ ] = self.question_to_attributes[key]["question_text"]
162
+ question_options_dict[
163
+ key + "_question_options"
164
+ ] = self.question_to_attributes[key]["question_options"]
165
+ question_type_dict[
166
+ key + "_question_type"
167
+ ] = self.question_to_attributes[key]["question_type"]
168
+
169
+ return {
170
+ "agent": self.agent.traits
171
+ | {"agent_name": agent_name}
172
+ | {"agent_instruction": self.agent.instruction},
173
+ "scenario": self.scenario,
174
+ "model": self.model.parameters | {"model": self.model.model},
175
+ "answer": self.answer,
176
+ "prompt": self.prompt,
177
+ "raw_model_response": self.raw_model_response,
178
+ "iteration": {"iteration": self.iteration},
179
+ "question_text": question_text_dict,
180
+ "question_options": question_options_dict,
181
+ "question_type": question_type_dict,
182
+ "comment": self.comments_dict,
183
+ "generated_tokens": self.generated_tokens,
184
+ }
185
+
186
+ def check_expression(self, expression) -> None:
187
+ for key in self.problem_keys:
188
+ if key in expression and not key + "." in expression:
189
+ raise ValueError(
190
+ f"Key by iself {key} is problematic. Use the full key {key + '.' + key} name instead."
191
+ )
192
+ return None
193
+
194
+ def code(self):
195
+ """Return a string of code that can be used to recreate the Result object."""
196
+ raise NotImplementedError
197
+
198
+ @property
199
+ def problem_keys(self):
200
+ """Return a list of keys that are problematic."""
201
+ return self._problem_keys
202
+
203
+ def _compute_combined_dict_and_problem_keys(self) -> None:
204
+ combined = {}
205
+ problem_keys = []
206
+ for key, sub_dict in self.sub_dicts.items():
207
+ combined.update(sub_dict)
208
+ # in some cases, the sub_dict might have keys that conflict with the main dict
209
+ if key in combined:
210
+ # The key is already in the combined dict
211
+ problem_keys = problem_keys + [key]
212
+
213
+ combined.update({key: sub_dict})
214
+ # I *think* this allows us to do do things like "answer.how_feelling" i.e., that the evaluator can use
215
+ # dot notation to access the subdicts.
216
+ self._combined_dict = combined
217
+ self._problem_keys = problem_keys
218
+
219
+ @property
220
+ def combined_dict(self) -> dict[str, Any]:
221
+ """Return a dictionary that includes all sub_dicts, but also puts the key-value pairs in each sub_dict as a key_value pair in the combined dictionary.
222
+
223
+ >>> r = Result.example()
224
+ >>> r.combined_dict['how_feeling']
225
+ 'OK'
226
+ """
227
+ if self._combined_dict is None or self._problem_keys is None:
228
+ self._compute_combined_dict_and_problem_keys()
229
+ return self._combined_dict
230
+
231
+ @property
232
+ def problem_keys(self):
233
+ """Return a list of keys that are problematic."""
234
+ if self._combined_dict is None or self._problem_keys is None:
235
+ self._compute_combined_dict_and_problem_keys()
236
+ return self._problem_keys
237
+
238
+ def get_value(self, data_type: str, key: str) -> Any:
239
+ """Return the value for a given data type and key.
240
+
241
+ >>> r = Result.example()
242
+ >>> r.get_value("answer", "how_feeling")
243
+ 'OK'
244
+
245
+ - data types can be "agent", "scenario", "model", or "answer"
246
+ - keys are relevant attributes of the Objects the data types represent
247
+ """
248
+ return self.sub_dicts[data_type][key]
249
+
250
+ @property
251
+ def key_to_data_type(self) -> dict[str, str]:
252
+ """Return a dictionary where keys are object attributes and values are the data type (object) that the attribute is associated with.
253
+
254
+ >>> r = Result.example()
255
+ >>> r.key_to_data_type["how_feeling"]
256
+ 'answer'
257
+
258
+ """
259
+ d = {}
260
+ data_types = self.sub_dicts.keys()
261
+ for data_type in data_types:
262
+ for key in self.sub_dicts[data_type]:
263
+ d[key] = data_type
264
+ return d
265
+
266
+ def rows(self, index) -> tuple[int, str, str, str]:
267
+ """Return a generator of rows for the Result object."""
268
+ for data_type, subdict in self.sub_dicts.items():
269
+ for key, value in subdict.items():
270
+ yield (index, data_type, key, str(value))
271
+
272
+ def leaves(self):
273
+ leaves = []
274
+ for question_name, answer in self.answer.items():
275
+ if not question_name.endswith("_comment"):
276
+ leaves.append(
277
+ {
278
+ "question": f"({question_name}): "
279
+ + str(
280
+ self.question_to_attributes[question_name]["question_text"]
281
+ ),
282
+ "answer": answer,
283
+ "comment": self.answer.get(question_name + "_comment", ""),
284
+ "scenario": repr(self.scenario),
285
+ "agent": repr(self.agent),
286
+ "model": repr(self.model),
287
+ "iteration": self.iteration,
288
+ }
289
+ )
290
+ return leaves
291
+
292
+ ###############
293
+ # Useful
294
+ ###############
295
+ def copy(self) -> Result:
296
+ """Return a copy of the Result object."""
297
+ return Result.from_dict(self.to_dict())
298
+
299
+ def __eq__(self, other) -> bool:
300
+ """Return True if the Result object is equal to another Result object.
301
+
302
+ >>> r = Result.example()
303
+ >>> r == r
304
+ True
305
+
306
+ """
307
+ return self.to_dict() == other.to_dict()
308
+
309
+ ###############
310
+ # Serialization
311
+ ###############
312
+ def _to_dict(self) -> dict[str, Any]:
313
+ """Return a dictionary representation of the Result object."""
314
+ d = {}
315
+ for key, value in self.items():
316
+ if hasattr(value, "to_dict"):
317
+ d[key] = value.to_dict()
318
+ else:
319
+ d[key] = value
320
+ if key == "prompt":
321
+ new_prompt_dict = {}
322
+ for prompt_name, prompt_obj in value.items():
323
+ new_prompt_dict[prompt_name] = (
324
+ prompt_obj
325
+ if not hasattr(prompt_obj, "to_dict")
326
+ else prompt_obj.to_dict()
327
+ )
328
+ d[key] = new_prompt_dict
329
+ return d
330
+
331
+ @add_edsl_version
332
+ def to_dict(self) -> dict[str, Any]:
333
+ """Return a dictionary representation of the Result object.
334
+
335
+ >>> r = Result.example()
336
+ >>> r.to_dict()['scenario']
337
+ {'period': 'morning', 'edsl_version': '...', 'edsl_class_name': 'Scenario'}
338
+ """
339
+ return self._to_dict()
340
+
341
+ def __hash__(self):
342
+ """Return a hash of the Result object."""
343
+ from edsl.utilities.utilities import dict_hash
344
+
345
+ return dict_hash(self._to_dict())
346
+
347
+ @classmethod
348
+ @remove_edsl_version
349
+ def from_dict(self, json_dict: dict) -> Result:
350
+ """Return a Result object from a dictionary representation."""
351
+
352
+ from edsl import Agent
353
+ from edsl import Scenario
354
+ from edsl.language_models.LanguageModel import LanguageModel
355
+ from edsl.prompts.Prompt import Prompt
356
+
357
+ prompt_data = json_dict.get("prompt", {})
358
+ prompt_d = {}
359
+ for prompt_name, prompt_obj in prompt_data.items():
360
+ prompt_d[prompt_name] = Prompt.from_dict(prompt_obj)
361
+
362
+ result = Result(
363
+ agent=Agent.from_dict(json_dict["agent"]),
364
+ scenario=Scenario.from_dict(json_dict["scenario"]),
365
+ model=LanguageModel.from_dict(json_dict["model"]),
366
+ iteration=json_dict["iteration"],
367
+ answer=json_dict["answer"],
368
+ prompt=prompt_d, # json_dict["prompt"],
369
+ raw_model_response=json_dict.get(
370
+ "raw_model_response", {"raw_model_response": "No raw model response"}
371
+ ),
372
+ question_to_attributes=json_dict.get("question_to_attributes", None),
373
+ generated_tokens=json_dict.get("generated_tokens", {}),
374
+ comments_dict=json_dict.get("comments_dict", {}),
375
+ )
376
+ return result
377
+
378
+ def rich_print(self) -> None:
379
+ """Display an object as a table."""
380
+ # from edsl.utilities import print_dict_with_rich
381
+ from rich import print
382
+ from rich.table import Table
383
+
384
+ table = Table(title="Result")
385
+ table.add_column("Attribute", style="bold")
386
+ table.add_column("Value")
387
+
388
+ to_display = self.__dict__.copy()
389
+ data = to_display.pop("data", None)
390
+ for attr_name, attr_value in to_display.items():
391
+ if hasattr(attr_value, "rich_print"):
392
+ table.add_row(attr_name, attr_value.rich_print())
393
+ elif isinstance(attr_value, dict):
394
+ a = PromptDict(attr_value)
395
+ table.add_row(attr_name, a.rich_print())
396
+ else:
397
+ table.add_row(attr_name, repr(attr_value))
398
+ return table
399
+
400
+ def __repr__(self):
401
+ """Return a string representation of the Result object."""
402
+ return f"Result(agent={repr(self.agent)}, scenario={repr(self.scenario)}, model={repr(self.model)}, iteration={self.iteration}, answer={repr(self.answer)}, prompt={repr(self.prompt)})"
403
+
404
+ @classmethod
405
+ def example(cls):
406
+ """Return an example Result object."""
407
+ from edsl.results.Results import Results
408
+
409
+ return Results.example()[0]
410
+
411
+ def score(self, scoring_function: Callable) -> Any:
412
+ """Score the result using a passed-in scoring function.
413
+
414
+ >>> def f(status): return 1 if status == 'Joyful' else 0
415
+ >>> Result.example().score(f)
416
+ 1
417
+ """
418
+ import inspect
419
+
420
+ signature = inspect.signature(scoring_function)
421
+ params = {}
422
+ for k, v in signature.parameters.items():
423
+ if k in self.combined_dict:
424
+ params[k] = self.combined_dict[k]
425
+ elif v.default is not v.empty:
426
+ params[k] = v.default
427
+ else:
428
+ raise ValueError(f"Parameter {k} not found in Result object")
429
+ return scoring_function(**params)
430
+
431
+
432
+ if __name__ == "__main__":
433
+ import doctest
434
+
435
+ doctest.testmod(optionflags=doctest.ELLIPSIS)