edsl 0.1.39__py3-none-any.whl → 0.1.39.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/Base.py +116 -197
- edsl/__init__.py +7 -15
- edsl/__version__.py +1 -1
- edsl/agents/Agent.py +147 -351
- edsl/agents/AgentList.py +73 -211
- edsl/agents/Invigilator.py +50 -101
- edsl/agents/InvigilatorBase.py +70 -62
- edsl/agents/PromptConstructor.py +225 -143
- edsl/agents/__init__.py +1 -0
- edsl/agents/prompt_helpers.py +3 -3
- edsl/auto/AutoStudy.py +5 -18
- edsl/auto/StageBase.py +40 -53
- edsl/auto/StageQuestions.py +1 -2
- edsl/auto/utilities.py +6 -0
- edsl/config.py +2 -22
- edsl/conversation/car_buying.py +1 -2
- edsl/coop/PriceFetcher.py +1 -1
- edsl/coop/coop.py +47 -125
- edsl/coop/utils.py +14 -14
- edsl/data/Cache.py +27 -45
- edsl/data/CacheEntry.py +15 -12
- edsl/data/CacheHandler.py +12 -31
- edsl/data/RemoteCacheSync.py +46 -154
- edsl/data/__init__.py +3 -4
- edsl/data_transfer_models.py +1 -2
- edsl/enums.py +0 -27
- edsl/exceptions/__init__.py +50 -50
- edsl/exceptions/agents.py +0 -12
- edsl/exceptions/questions.py +6 -24
- edsl/exceptions/scenarios.py +0 -7
- edsl/inference_services/AnthropicService.py +19 -38
- edsl/inference_services/AwsBedrock.py +2 -0
- edsl/inference_services/AzureAI.py +2 -0
- edsl/inference_services/GoogleService.py +12 -7
- edsl/inference_services/InferenceServiceABC.py +85 -18
- edsl/inference_services/InferenceServicesCollection.py +79 -120
- edsl/inference_services/MistralAIService.py +3 -0
- edsl/inference_services/OpenAIService.py +35 -47
- edsl/inference_services/PerplexityService.py +3 -0
- edsl/inference_services/TestService.py +10 -11
- edsl/inference_services/TogetherAIService.py +3 -5
- edsl/jobs/Answers.py +14 -1
- edsl/jobs/Jobs.py +431 -356
- edsl/jobs/JobsChecks.py +10 -35
- edsl/jobs/JobsPrompts.py +4 -6
- edsl/jobs/JobsRemoteInferenceHandler.py +133 -205
- edsl/jobs/buckets/BucketCollection.py +3 -44
- edsl/jobs/buckets/TokenBucket.py +21 -53
- edsl/jobs/interviews/Interview.py +408 -143
- edsl/jobs/runners/JobsRunnerAsyncio.py +403 -88
- edsl/jobs/runners/JobsRunnerStatus.py +165 -133
- edsl/jobs/tasks/QuestionTaskCreator.py +19 -21
- edsl/jobs/tasks/TaskHistory.py +18 -38
- edsl/jobs/tasks/task_status_enum.py +2 -0
- edsl/language_models/KeyLookup.py +30 -0
- edsl/language_models/LanguageModel.py +236 -194
- edsl/language_models/ModelList.py +19 -28
- edsl/language_models/__init__.py +2 -1
- edsl/language_models/registry.py +190 -0
- edsl/language_models/repair.py +2 -2
- edsl/language_models/unused/ReplicateBase.py +83 -0
- edsl/language_models/utilities.py +4 -5
- edsl/notebooks/Notebook.py +14 -19
- edsl/prompts/Prompt.py +39 -29
- edsl/questions/{answer_validator_mixin.py → AnswerValidatorMixin.py} +2 -47
- edsl/questions/QuestionBase.py +214 -68
- edsl/questions/{question_base_gen_mixin.py → QuestionBaseGenMixin.py} +50 -57
- edsl/questions/QuestionBasePromptsMixin.py +3 -7
- edsl/questions/QuestionBudget.py +1 -1
- edsl/questions/QuestionCheckBox.py +3 -3
- edsl/questions/QuestionExtract.py +7 -5
- edsl/questions/QuestionFreeText.py +3 -2
- edsl/questions/QuestionList.py +18 -10
- edsl/questions/QuestionMultipleChoice.py +23 -67
- edsl/questions/QuestionNumerical.py +4 -2
- edsl/questions/QuestionRank.py +17 -7
- edsl/questions/{response_validator_abc.py → ResponseValidatorABC.py} +26 -40
- edsl/questions/SimpleAskMixin.py +3 -4
- edsl/questions/__init__.py +1 -2
- edsl/questions/derived/QuestionLinearScale.py +3 -6
- edsl/questions/derived/QuestionTopK.py +1 -1
- edsl/questions/descriptors.py +3 -17
- edsl/questions/question_registry.py +1 -1
- edsl/results/CSSParameterizer.py +1 -1
- edsl/results/Dataset.py +7 -170
- edsl/results/DatasetExportMixin.py +305 -168
- edsl/results/DatasetTree.py +8 -28
- edsl/results/Result.py +206 -298
- edsl/results/Results.py +131 -149
- edsl/results/ResultsDBMixin.py +238 -0
- edsl/results/ResultsExportMixin.py +0 -2
- edsl/results/{results_selector.py → Selector.py} +13 -23
- edsl/results/TableDisplay.py +171 -98
- edsl/results/__init__.py +1 -1
- edsl/scenarios/FileStore.py +239 -150
- edsl/scenarios/Scenario.py +193 -90
- edsl/scenarios/ScenarioHtmlMixin.py +3 -4
- edsl/scenarios/{scenario_join.py → ScenarioJoin.py} +6 -10
- edsl/scenarios/ScenarioList.py +244 -415
- edsl/scenarios/ScenarioListExportMixin.py +7 -0
- edsl/scenarios/ScenarioListPdfMixin.py +37 -15
- edsl/scenarios/__init__.py +2 -1
- edsl/study/ObjectEntry.py +1 -1
- edsl/study/SnapShot.py +1 -1
- edsl/study/Study.py +12 -5
- edsl/surveys/Rule.py +4 -5
- edsl/surveys/RuleCollection.py +27 -25
- edsl/surveys/Survey.py +791 -270
- edsl/surveys/SurveyCSS.py +8 -20
- edsl/surveys/{SurveyFlowVisualization.py → SurveyFlowVisualizationMixin.py} +9 -11
- edsl/surveys/__init__.py +2 -4
- edsl/surveys/descriptors.py +2 -6
- edsl/surveys/instructions/ChangeInstruction.py +2 -1
- edsl/surveys/instructions/Instruction.py +13 -4
- edsl/surveys/instructions/InstructionCollection.py +6 -11
- edsl/templates/error_reporting/interview_details.html +1 -1
- edsl/templates/error_reporting/report.html +1 -1
- edsl/tools/plotting.py +1 -1
- edsl/utilities/utilities.py +23 -35
- {edsl-0.1.39.dist-info → edsl-0.1.39.dev1.dist-info}/METADATA +10 -12
- edsl-0.1.39.dev1.dist-info/RECORD +277 -0
- {edsl-0.1.39.dist-info → edsl-0.1.39.dev1.dist-info}/WHEEL +1 -1
- edsl/agents/QuestionInstructionPromptBuilder.py +0 -128
- edsl/agents/QuestionTemplateReplacementsBuilder.py +0 -137
- edsl/agents/question_option_processor.py +0 -172
- edsl/coop/CoopFunctionsMixin.py +0 -15
- edsl/coop/ExpectedParrotKeyHandler.py +0 -125
- edsl/exceptions/inference_services.py +0 -5
- edsl/inference_services/AvailableModelCacheHandler.py +0 -184
- edsl/inference_services/AvailableModelFetcher.py +0 -215
- edsl/inference_services/ServiceAvailability.py +0 -135
- edsl/inference_services/data_structures.py +0 -134
- edsl/jobs/AnswerQuestionFunctionConstructor.py +0 -223
- edsl/jobs/FetchInvigilator.py +0 -47
- edsl/jobs/InterviewTaskManager.py +0 -98
- edsl/jobs/InterviewsConstructor.py +0 -50
- edsl/jobs/JobsComponentConstructor.py +0 -189
- edsl/jobs/JobsRemoteInferenceLogger.py +0 -239
- edsl/jobs/RequestTokenEstimator.py +0 -30
- edsl/jobs/async_interview_runner.py +0 -138
- edsl/jobs/buckets/TokenBucketAPI.py +0 -211
- edsl/jobs/buckets/TokenBucketClient.py +0 -191
- edsl/jobs/check_survey_scenario_compatibility.py +0 -85
- edsl/jobs/data_structures.py +0 -120
- edsl/jobs/decorators.py +0 -35
- edsl/jobs/jobs_status_enums.py +0 -9
- edsl/jobs/loggers/HTMLTableJobLogger.py +0 -304
- edsl/jobs/results_exceptions_handler.py +0 -98
- edsl/language_models/ComputeCost.py +0 -63
- edsl/language_models/PriceManager.py +0 -127
- edsl/language_models/RawResponseHandler.py +0 -106
- edsl/language_models/ServiceDataSources.py +0 -0
- edsl/language_models/key_management/KeyLookup.py +0 -63
- edsl/language_models/key_management/KeyLookupBuilder.py +0 -273
- edsl/language_models/key_management/KeyLookupCollection.py +0 -38
- edsl/language_models/key_management/__init__.py +0 -0
- edsl/language_models/key_management/models.py +0 -131
- edsl/language_models/model.py +0 -256
- edsl/notebooks/NotebookToLaTeX.py +0 -142
- edsl/questions/ExceptionExplainer.py +0 -77
- edsl/questions/HTMLQuestion.py +0 -103
- edsl/questions/QuestionMatrix.py +0 -265
- edsl/questions/data_structures.py +0 -20
- edsl/questions/loop_processor.py +0 -149
- edsl/questions/response_validator_factory.py +0 -34
- edsl/questions/templates/matrix/__init__.py +0 -1
- edsl/questions/templates/matrix/answering_instructions.jinja +0 -5
- edsl/questions/templates/matrix/question_presentation.jinja +0 -20
- edsl/results/MarkdownToDocx.py +0 -122
- edsl/results/MarkdownToPDF.py +0 -111
- edsl/results/TextEditor.py +0 -50
- edsl/results/file_exports.py +0 -252
- edsl/results/smart_objects.py +0 -96
- edsl/results/table_data_class.py +0 -12
- edsl/results/table_renderers.py +0 -118
- edsl/scenarios/ConstructDownloadLink.py +0 -109
- edsl/scenarios/DocumentChunker.py +0 -102
- edsl/scenarios/DocxScenario.py +0 -16
- edsl/scenarios/PdfExtractor.py +0 -40
- edsl/scenarios/directory_scanner.py +0 -96
- edsl/scenarios/file_methods.py +0 -85
- edsl/scenarios/handlers/__init__.py +0 -13
- edsl/scenarios/handlers/csv.py +0 -49
- edsl/scenarios/handlers/docx.py +0 -76
- edsl/scenarios/handlers/html.py +0 -37
- edsl/scenarios/handlers/json.py +0 -111
- edsl/scenarios/handlers/latex.py +0 -5
- edsl/scenarios/handlers/md.py +0 -51
- edsl/scenarios/handlers/pdf.py +0 -68
- edsl/scenarios/handlers/png.py +0 -39
- edsl/scenarios/handlers/pptx.py +0 -105
- edsl/scenarios/handlers/py.py +0 -294
- edsl/scenarios/handlers/sql.py +0 -313
- edsl/scenarios/handlers/sqlite.py +0 -149
- edsl/scenarios/handlers/txt.py +0 -33
- edsl/scenarios/scenario_selector.py +0 -156
- edsl/surveys/ConstructDAG.py +0 -92
- edsl/surveys/EditSurvey.py +0 -221
- edsl/surveys/InstructionHandler.py +0 -100
- edsl/surveys/MemoryManagement.py +0 -72
- edsl/surveys/RuleManager.py +0 -172
- edsl/surveys/Simulator.py +0 -75
- edsl/surveys/SurveyToApp.py +0 -141
- edsl/utilities/PrettyList.py +0 -56
- edsl/utilities/is_notebook.py +0 -18
- edsl/utilities/is_valid_variable_name.py +0 -11
- edsl/utilities/remove_edsl_version.py +0 -24
- edsl-0.1.39.dist-info/RECORD +0 -358
- /edsl/questions/{register_questions_meta.py → RegisterQuestionsMeta.py} +0 -0
- /edsl/results/{results_fetch_mixin.py → ResultsFetchMixin.py} +0 -0
- /edsl/results/{results_tools_mixin.py → ResultsToolsMixin.py} +0 -0
- {edsl-0.1.39.dist-info → edsl-0.1.39.dev1.dist-info}/LICENSE +0 -0
edsl/agents/AgentList.py
CHANGED
@@ -1,41 +1,38 @@
|
|
1
|
-
"""A list of
|
1
|
+
"""A list of Agent objects.
|
2
|
+
|
3
|
+
Example usage:
|
4
|
+
|
5
|
+
.. code-block:: python
|
6
|
+
|
7
|
+
al = AgentList([Agent.example(), Agent.example()])
|
8
|
+
len(al)
|
9
|
+
2
|
10
|
+
|
2
11
|
"""
|
3
12
|
|
4
13
|
from __future__ import annotations
|
5
14
|
import csv
|
6
|
-
import
|
15
|
+
import json
|
7
16
|
from collections import UserList
|
8
|
-
from collections.abc import Iterable
|
9
|
-
|
10
17
|
from typing import Any, List, Optional, Union, TYPE_CHECKING
|
18
|
+
from rich import print_json
|
19
|
+
from rich.table import Table
|
20
|
+
from simpleeval import EvalWithCompoundTypes
|
21
|
+
from edsl.Base import Base
|
22
|
+
from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
|
11
23
|
|
12
|
-
from
|
24
|
+
from collections.abc import Iterable
|
13
25
|
|
14
|
-
from edsl.Base import Base
|
15
|
-
from edsl.utilities.remove_edsl_version import remove_edsl_version
|
16
26
|
from edsl.exceptions.agents import AgentListError
|
17
|
-
from edsl.utilities.is_notebook import is_notebook
|
18
|
-
from edsl.results.ResultsExportMixin import ResultsExportMixin
|
19
|
-
import logging
|
20
|
-
|
21
|
-
logger = logging.getLogger(__name__)
|
22
27
|
|
23
28
|
if TYPE_CHECKING:
|
24
29
|
from edsl.scenarios.ScenarioList import ScenarioList
|
25
|
-
from edsl.agents.Agent import Agent
|
26
|
-
from pandas import DataFrame
|
27
30
|
|
28
31
|
|
29
32
|
def is_iterable(obj):
|
30
33
|
return isinstance(obj, Iterable)
|
31
34
|
|
32
35
|
|
33
|
-
class EmptyAgentList:
|
34
|
-
def __repr__(self):
|
35
|
-
return "Empty AgentList"
|
36
|
-
|
37
|
-
|
38
|
-
# ResultsExportMixin,
|
39
36
|
class AgentList(UserList, Base):
|
40
37
|
"""A list of Agents."""
|
41
38
|
|
@@ -53,15 +50,14 @@ class AgentList(UserList, Base):
|
|
53
50
|
else:
|
54
51
|
super().__init__()
|
55
52
|
|
56
|
-
def shuffle(self, seed: Optional[str] =
|
53
|
+
def shuffle(self, seed: Optional[str] = "edsl") -> AgentList:
|
57
54
|
"""Shuffle the AgentList.
|
58
55
|
|
59
56
|
:param seed: The seed for the random number generator.
|
60
57
|
"""
|
61
58
|
import random
|
62
59
|
|
63
|
-
|
64
|
-
random.seed(seed)
|
60
|
+
random.seed(seed)
|
65
61
|
random.shuffle(self.data)
|
66
62
|
return self
|
67
63
|
|
@@ -77,60 +73,22 @@ class AgentList(UserList, Base):
|
|
77
73
|
random.seed(seed)
|
78
74
|
return AgentList(random.sample(self.data, n))
|
79
75
|
|
80
|
-
def to_pandas(self)
|
81
|
-
"""Return a pandas DataFrame.
|
82
|
-
|
83
|
-
>>> from edsl.agents.Agent import Agent
|
84
|
-
>>> al = AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5})])
|
85
|
-
>>> al.to_pandas()
|
86
|
-
age hair height
|
87
|
-
0 22 brown 5.5
|
88
|
-
1 22 brown 5.5
|
89
|
-
"""
|
76
|
+
def to_pandas(self):
|
77
|
+
"""Return a pandas DataFrame."""
|
90
78
|
return self.to_scenario_list().to_pandas()
|
91
79
|
|
92
|
-
def tally(
|
93
|
-
self
|
94
|
-
) -> Union[dict, "Dataset"]:
|
95
|
-
"""Tally the values of a field or perform a cross-tab of multiple fields.
|
96
|
-
|
97
|
-
:param fields: The field(s) to tally, multiple fields for cross-tabulation.
|
80
|
+
def tally(self):
|
81
|
+
return self.to_scenario_list().tally()
|
98
82
|
|
99
|
-
|
100
|
-
>>> al.tally('age')
|
101
|
-
Dataset([{'age': [22]}, {'count': [2]}])
|
102
|
-
"""
|
103
|
-
return self.to_scenario_list().tally(*fields, top_n=top_n, output=output)
|
104
|
-
|
105
|
-
def duplicate(self):
|
106
|
-
"""Duplicate the AgentList.
|
107
|
-
|
108
|
-
>>> al = AgentList.example()
|
109
|
-
>>> al2 = al.duplicate()
|
110
|
-
>>> al2 == al
|
111
|
-
True
|
112
|
-
>>> id(al2) == id(al)
|
113
|
-
False
|
114
|
-
"""
|
115
|
-
return AgentList([a.duplicate() for a in self.data])
|
116
|
-
|
117
|
-
def rename(self, old_name, new_name) -> AgentList:
|
83
|
+
def rename(self, old_name, new_name):
|
118
84
|
"""Rename a trait in the AgentList.
|
119
85
|
|
120
86
|
:param old_name: The old name of the trait.
|
121
87
|
:param new_name: The new name of the trait.
|
122
|
-
:param inplace: Whether to rename the trait in place.
|
123
|
-
|
124
|
-
>>> from edsl.agents.Agent import Agent
|
125
|
-
>>> al = AgentList([Agent(traits = {'a': 1, 'b': 1}), Agent(traits = {'a': 1, 'b': 2})])
|
126
|
-
>>> al2 = al.rename('a', 'c')
|
127
|
-
>>> assert al2 == AgentList([Agent(traits = {'c': 1, 'b': 1}), Agent(traits = {'c': 1, 'b': 2})])
|
128
|
-
>>> assert al != al2
|
129
88
|
"""
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
return AgentList(newagents)
|
89
|
+
for agent in self.data:
|
90
|
+
agent.rename(old_name, new_name)
|
91
|
+
return self
|
134
92
|
|
135
93
|
def select(self, *traits) -> AgentList:
|
136
94
|
"""Selects agents with only the references traits.
|
@@ -165,36 +123,19 @@ class AgentList(UserList, Base):
|
|
165
123
|
"""
|
166
124
|
return EvalWithCompoundTypes(names=agent.traits)
|
167
125
|
|
168
|
-
# iterates through all the results and evaluates the expression
|
169
|
-
|
170
126
|
try:
|
127
|
+
# iterates through all the results and evaluates the expression
|
171
128
|
new_data = [
|
172
129
|
agent for agent in self.data if create_evaluator(agent).eval(expression)
|
173
130
|
]
|
174
|
-
except
|
175
|
-
|
176
|
-
|
177
|
-
print(e, file=sys.stderr)
|
178
|
-
else:
|
179
|
-
raise e
|
180
|
-
|
181
|
-
return EmptyAgentList()
|
182
|
-
|
183
|
-
if len(new_data) == 0:
|
184
|
-
return EmptyAgentList()
|
131
|
+
except Exception as e:
|
132
|
+
print(f"Exception:{e}")
|
133
|
+
raise AgentListError(f"Error in filter. Exception:{e}")
|
185
134
|
|
186
135
|
return AgentList(new_data)
|
187
136
|
|
188
137
|
@property
|
189
|
-
def all_traits(self)
|
190
|
-
"""Return all traits in the AgentList.
|
191
|
-
>>> from edsl.agents.Agent import Agent
|
192
|
-
>>> agent_1 = Agent(traits = {'age': 22})
|
193
|
-
>>> agent_2 = Agent(traits = {'hair': 'brown'})
|
194
|
-
>>> al = AgentList([agent_1, agent_2])
|
195
|
-
>>> al.all_traits
|
196
|
-
['age', 'hair']
|
197
|
-
"""
|
138
|
+
def all_traits(self):
|
198
139
|
d = {}
|
199
140
|
for agent in self:
|
200
141
|
d.update(agent.traits)
|
@@ -239,20 +180,14 @@ class AgentList(UserList, Base):
|
|
239
180
|
agent_list.append(Agent(traits=row))
|
240
181
|
return cls(agent_list)
|
241
182
|
|
242
|
-
def translate_traits(self,
|
183
|
+
def translate_traits(self, values_codebook: dict[str, str]):
|
243
184
|
"""Translate traits to a new codebook.
|
244
185
|
|
245
186
|
:param codebook: The new codebook.
|
246
|
-
|
247
|
-
>>> al = AgentList.example()
|
248
|
-
>>> codebook = {'hair': {'brown':'Secret word for green'}}
|
249
|
-
>>> al.translate_traits(codebook)
|
250
|
-
AgentList([Agent(traits = {'age': 22, 'hair': 'Secret word for green', 'height': 5.5}), Agent(traits = {'age': 22, 'hair': 'Secret word for green', 'height': 5.5})])
|
251
187
|
"""
|
252
|
-
new_agents = []
|
253
188
|
for agent in self.data:
|
254
|
-
|
255
|
-
return
|
189
|
+
agent.translate_traits(codebook)
|
190
|
+
return self
|
256
191
|
|
257
192
|
def remove_trait(self, trait: str):
|
258
193
|
"""Remove traits from the AgentList.
|
@@ -263,21 +198,20 @@ class AgentList(UserList, Base):
|
|
263
198
|
>>> al.remove_trait('age')
|
264
199
|
AgentList([Agent(traits = {'hair': 'brown', 'height': 5.5}), Agent(traits = {'hair': 'brown', 'height': 5.5})])
|
265
200
|
"""
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
agents.append(agent.remove_trait(trait))
|
270
|
-
return AgentList(agents)
|
201
|
+
for agent in self.data:
|
202
|
+
_ = agent.remove_trait(trait)
|
203
|
+
return self
|
271
204
|
|
272
|
-
def add_trait(self, trait
|
205
|
+
def add_trait(self, trait, values):
|
273
206
|
"""Adds a new trait to every agent, with values taken from values.
|
274
207
|
|
275
208
|
:param trait: The name of the trait.
|
276
209
|
:param values: The valeues(s) of the trait. If a single value is passed, it is used for all agents.
|
277
210
|
|
278
211
|
>>> al = AgentList.example()
|
279
|
-
>>>
|
280
|
-
|
212
|
+
>>> al.add_trait('new_trait', 1)
|
213
|
+
AgentList([Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1}), Agent(traits = {'age': 22, 'hair': 'brown', 'height': 5.5, 'new_trait': 1})])
|
214
|
+
>>> al.select('new_trait').to_scenario_list().to_list()
|
281
215
|
[1, 1]
|
282
216
|
>>> al.add_trait('new_trait', [1, 2, 3])
|
283
217
|
Traceback (most recent call last):
|
@@ -286,24 +220,18 @@ class AgentList(UserList, Base):
|
|
286
220
|
...
|
287
221
|
"""
|
288
222
|
if not is_iterable(values):
|
289
|
-
new_agents = []
|
290
223
|
value = values
|
291
224
|
for agent in self.data:
|
292
|
-
|
293
|
-
return
|
225
|
+
agent.add_trait(trait, value)
|
226
|
+
return self
|
294
227
|
|
295
228
|
if len(values) != len(self):
|
296
|
-
|
229
|
+
raise AgentListError(
|
297
230
|
"The passed values have to be the same length as the agent list."
|
298
231
|
)
|
299
|
-
if is_notebook():
|
300
|
-
print(e, file=sys.stderr)
|
301
|
-
else:
|
302
|
-
raise e
|
303
|
-
new_agents = []
|
304
232
|
for agent, value in zip(self.data, values):
|
305
|
-
|
306
|
-
return
|
233
|
+
agent.add_trait(trait, value)
|
234
|
+
return self
|
307
235
|
|
308
236
|
@staticmethod
|
309
237
|
def get_codebook(file_path: str):
|
@@ -316,23 +244,12 @@ class AgentList(UserList, Base):
|
|
316
244
|
return {field: None for field in reader.fieldnames}
|
317
245
|
|
318
246
|
def __hash__(self) -> int:
|
319
|
-
"""Return the hash of the AgentList.
|
320
|
-
|
321
|
-
>>> al = AgentList.example()
|
322
|
-
>>> hash(al)
|
323
|
-
1681154913465662422
|
324
|
-
"""
|
325
247
|
from edsl.utilities.utilities import dict_hash
|
326
248
|
|
327
249
|
return dict_hash(self.to_dict(add_edsl_version=False, sorted=True))
|
328
250
|
|
329
251
|
def to_dict(self, sorted=False, add_edsl_version=True):
|
330
|
-
"""Serialize the AgentList to a dictionary.
|
331
|
-
|
332
|
-
>>> AgentList.example().to_dict(add_edsl_version=False)
|
333
|
-
{'agent_list': [{'traits': {'age': 22, 'hair': 'brown', 'height': 5.5}}, {'traits': {'age': 22, 'hair': 'brown', 'height': 5.5}}]}
|
334
|
-
|
335
|
-
"""
|
252
|
+
"""Serialize the AgentList to a dictionary."""
|
336
253
|
if sorted:
|
337
254
|
data = self.data[:]
|
338
255
|
data.sort(key=lambda x: hash(x))
|
@@ -362,26 +279,15 @@ class AgentList(UserList, Base):
|
|
362
279
|
|
363
280
|
def _summary(self):
|
364
281
|
return {
|
365
|
-
"
|
282
|
+
"EDSL Class": "AgentList",
|
283
|
+
"Number of agents": len(self),
|
284
|
+
"Agent trait fields": self.all_traits,
|
366
285
|
}
|
367
286
|
|
368
|
-
def
|
369
|
-
"""
|
370
|
-
|
371
|
-
|
372
|
-
>>> a = Agent(traits = {'hair': 'brown'})
|
373
|
-
>>> al = AgentList([a, a])
|
374
|
-
>>> _ = al.set_codebook({'hair': "Color of hair on driver's license"})
|
375
|
-
>>> al[0].codebook
|
376
|
-
{'hair': "Color of hair on driver's license"}
|
377
|
-
|
378
|
-
|
379
|
-
:param codebook: The codebook.
|
380
|
-
"""
|
381
|
-
for agent in self.data:
|
382
|
-
agent.codebook = codebook
|
383
|
-
|
384
|
-
return self
|
287
|
+
def _repr_html_(self):
|
288
|
+
"""Return an HTML representation of the AgentList."""
|
289
|
+
footer = f"<a href={self.__documentation__}>(docs)</a>"
|
290
|
+
return str(self.summary(format="html")) + footer
|
385
291
|
|
386
292
|
def to_csv(self, file_path: str):
|
387
293
|
"""Save the AgentList to a CSV file.
|
@@ -394,33 +300,19 @@ class AgentList(UserList, Base):
|
|
394
300
|
"""Return a list of tuples."""
|
395
301
|
return self.to_scenario_list(include_agent_name).to_list()
|
396
302
|
|
397
|
-
def to_scenario_list(
|
398
|
-
|
399
|
-
) -> ScenarioList:
|
400
|
-
"""Converts the agent to a scenario list."""
|
303
|
+
def to_scenario_list(self, include_agent_name=False) -> ScenarioList:
|
304
|
+
"""Return a list of scenarios."""
|
401
305
|
from edsl.scenarios.ScenarioList import ScenarioList
|
402
306
|
from edsl.scenarios.Scenario import Scenario
|
403
307
|
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
d["instruction"] = agent.instruction
|
413
|
-
scenario_list.append(Scenario(d))
|
414
|
-
return scenario_list
|
415
|
-
|
416
|
-
# if include_agent_name:
|
417
|
-
# return ScenarioList(
|
418
|
-
# [
|
419
|
-
# Scenario(agent.traits | {"agent_name": agent.name} | })
|
420
|
-
# for agent in self.data
|
421
|
-
# ]
|
422
|
-
# )
|
423
|
-
# return ScenarioList([Scenario(agent.traits) for agent in self.data])
|
308
|
+
if include_agent_name:
|
309
|
+
return ScenarioList(
|
310
|
+
[
|
311
|
+
Scenario(agent.traits | {"agent_name": agent.name})
|
312
|
+
for agent in self.data
|
313
|
+
]
|
314
|
+
)
|
315
|
+
return ScenarioList([Scenario(agent.traits) for agent in self.data])
|
424
316
|
|
425
317
|
def table(
|
426
318
|
self,
|
@@ -428,50 +320,12 @@ class AgentList(UserList, Base):
|
|
428
320
|
tablefmt: Optional[str] = None,
|
429
321
|
pretty_labels: Optional[dict] = None,
|
430
322
|
) -> Table:
|
431
|
-
if len(self) == 0:
|
432
|
-
e = AgentListError("Cannot create a table from an empty AgentList.")
|
433
|
-
if is_notebook():
|
434
|
-
print(e, file=sys.stderr)
|
435
|
-
return None
|
436
|
-
else:
|
437
|
-
raise e
|
438
323
|
return (
|
439
324
|
self.to_scenario_list()
|
440
325
|
.to_dataset()
|
441
326
|
.table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
|
442
327
|
)
|
443
328
|
|
444
|
-
def to_dataset(self, traits_only: bool = True):
|
445
|
-
"""
|
446
|
-
Convert the AgentList to a Dataset.
|
447
|
-
|
448
|
-
>>> from edsl.agents.AgentList import AgentList
|
449
|
-
>>> al = AgentList.example()
|
450
|
-
>>> al.to_dataset()
|
451
|
-
Dataset([{'age': [22, 22]}, {'hair': ['brown', 'brown']}, {'height': [5.5, 5.5]}])
|
452
|
-
>>> al.to_dataset(traits_only = False)
|
453
|
-
Dataset([{'age': [22, 22]}, {'hair': ['brown', 'brown']}, {'height': [5.5, 5.5]}, {'agent_parameters': [{'instruction': 'You are answering questions as if you were a human. Do not break character.', 'agent_name': None}, {'instruction': 'You are answering questions as if you were a human. Do not break character.', 'agent_name': None}]}])
|
454
|
-
"""
|
455
|
-
from edsl.results.Dataset import Dataset
|
456
|
-
from collections import defaultdict
|
457
|
-
|
458
|
-
agent_trait_keys = []
|
459
|
-
for agent in self:
|
460
|
-
agent_keys = list(agent.traits.keys())
|
461
|
-
for key in agent_keys:
|
462
|
-
if key not in agent_trait_keys:
|
463
|
-
agent_trait_keys.append(key)
|
464
|
-
|
465
|
-
data = defaultdict(list)
|
466
|
-
for agent in self:
|
467
|
-
for trait_key in agent_trait_keys:
|
468
|
-
data[trait_key].append(agent.traits.get(trait_key, None))
|
469
|
-
if not traits_only:
|
470
|
-
data["agent_parameters"].append(
|
471
|
-
{"instruction": agent.instruction, "agent_name": agent.name}
|
472
|
-
)
|
473
|
-
return Dataset([{key: entry} for key, entry in data.items()])
|
474
|
-
|
475
329
|
def tree(self, node_order: Optional[List[str]] = None):
|
476
330
|
return self.to_scenario_list().tree(node_order)
|
477
331
|
|
@@ -544,6 +398,14 @@ class AgentList(UserList, Base):
|
|
544
398
|
return "\n".join(lines)
|
545
399
|
return lines
|
546
400
|
|
401
|
+
def rich_print(self) -> Table:
|
402
|
+
"""Display an object as a rich table."""
|
403
|
+
table = Table(title="AgentList")
|
404
|
+
table.add_column("Agents", style="bold")
|
405
|
+
for agent in self.data:
|
406
|
+
table.add_row(agent.rich_print())
|
407
|
+
return table
|
408
|
+
|
547
409
|
|
548
410
|
if __name__ == "__main__":
|
549
411
|
import doctest
|
edsl/agents/Invigilator.py
CHANGED
@@ -1,29 +1,38 @@
|
|
1
1
|
"""Module for creating Invigilators, which are objects to administer a question to an Agent."""
|
2
2
|
|
3
|
-
from typing import Dict, Any, Optional
|
3
|
+
from typing import Dict, Any, Optional
|
4
4
|
|
5
|
-
from edsl.
|
5
|
+
from edsl.prompts.Prompt import Prompt
|
6
|
+
from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
|
7
|
+
|
8
|
+
# from edsl.prompts.registry import get_classes as prompt_lookup
|
6
9
|
from edsl.exceptions.questions import QuestionAnswerValidationError
|
7
10
|
from edsl.agents.InvigilatorBase import InvigilatorBase
|
8
11
|
from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
|
9
|
-
|
10
|
-
if TYPE_CHECKING:
|
11
|
-
from edsl.prompts.Prompt import Prompt
|
12
|
-
from edsl.scenarios.Scenario import Scenario
|
13
|
-
from edsl.surveys.Survey import Survey
|
12
|
+
from edsl.agents.PromptConstructor import PromptConstructor
|
14
13
|
|
15
14
|
|
16
|
-
|
15
|
+
class NotApplicable(str):
|
16
|
+
def __new__(cls):
|
17
|
+
instance = super().__new__(cls, "Not Applicable")
|
18
|
+
instance.literal = "Not Applicable"
|
19
|
+
return instance
|
17
20
|
|
18
21
|
|
19
22
|
class InvigilatorAI(InvigilatorBase):
|
20
23
|
"""An invigilator that uses an AI model to answer questions."""
|
21
24
|
|
22
|
-
def get_prompts(self) -> Dict[str,
|
25
|
+
def get_prompts(self) -> Dict[str, Prompt]:
|
23
26
|
"""Return the prompts used."""
|
24
27
|
return self.prompt_constructor.get_prompts()
|
25
28
|
|
26
|
-
async def
|
29
|
+
async def async_answer_question(self) -> AgentResponseDict:
|
30
|
+
"""Answer a question using the AI model.
|
31
|
+
|
32
|
+
>>> i = InvigilatorAI.example()
|
33
|
+
>>> i.answer_question()
|
34
|
+
{'message': [{'text': 'SPAM!'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
|
35
|
+
"""
|
27
36
|
prompts = self.get_prompts()
|
28
37
|
params = {
|
29
38
|
"user_prompt": prompts["user_prompt"].text,
|
@@ -31,95 +40,33 @@ class InvigilatorAI(InvigilatorBase):
|
|
31
40
|
}
|
32
41
|
if "encoded_image" in prompts:
|
33
42
|
params["encoded_image"] = prompts["encoded_image"]
|
34
|
-
raise NotImplementedError("encoded_image not implemented")
|
35
|
-
|
36
43
|
if "files_list" in prompts:
|
37
44
|
params["files_list"] = prompts["files_list"]
|
38
45
|
|
39
46
|
params.update({"iteration": self.iteration, "cache": self.cache})
|
40
|
-
params.update({"invigilator": self})
|
41
|
-
|
42
|
-
if self.key_lookup:
|
43
|
-
self.model.set_key_lookup(self.key_lookup)
|
44
47
|
|
45
|
-
|
48
|
+
params.update({"invigilator": self})
|
49
|
+
# if hasattr(self.question, "answer_template"):
|
50
|
+
# breakpoint()
|
46
51
|
|
47
|
-
|
48
|
-
|
52
|
+
agent_response_dict: AgentResponseDict = await self.model.async_get_response(
|
53
|
+
**params
|
54
|
+
)
|
55
|
+
# store to self in case validation failure
|
49
56
|
self.raw_model_response = agent_response_dict.model_outputs.response
|
50
57
|
self.generated_tokens = agent_response_dict.edsl_dict.generated_tokens
|
51
58
|
|
52
|
-
|
53
|
-
"""Answer a question using the AI model.
|
54
|
-
|
55
|
-
>>> i = InvigilatorAI.example()
|
56
|
-
"""
|
57
|
-
agent_response_dict = await self.async_get_agent_response()
|
58
|
-
self.store_response(agent_response_dict)
|
59
|
-
return self._extract_edsl_result_entry_and_validate(agent_response_dict)
|
59
|
+
return self.extract_edsl_result_entry_and_validate(agent_response_dict)
|
60
60
|
|
61
61
|
def _remove_from_cache(self, cache_key) -> None:
|
62
62
|
"""Remove an entry from the cache."""
|
63
63
|
if cache_key:
|
64
64
|
del self.cache.data[cache_key]
|
65
65
|
|
66
|
-
def
|
67
|
-
|
68
|
-
|
69
|
-
>>> i = InvigilatorAI.example()
|
70
|
-
>>> i._determine_answer("SPAM!")
|
71
|
-
'SPAM!'
|
72
|
-
|
73
|
-
>>> from edsl.questions import QuestionMultipleChoice
|
74
|
-
>>> q = QuestionMultipleChoice(question_text = "How are you?", question_name = "how_are_you", question_options = ["Good", "Bad"], use_code = True)
|
75
|
-
>>> i = InvigilatorAI.example(question = q)
|
76
|
-
>>> i._determine_answer("1")
|
77
|
-
'Bad'
|
78
|
-
>>> i._determine_answer("0")
|
79
|
-
'Good'
|
80
|
-
|
81
|
-
This shows how the answer can depend on scenario details
|
82
|
-
|
83
|
-
>>> from edsl import Scenario
|
84
|
-
>>> s = Scenario({'feeling_options':['Good', 'Bad']})
|
85
|
-
>>> q = QuestionMultipleChoice(question_text = "How are you?", question_name = "how_are_you", question_options = "{{ feeling_options }}", use_code = True)
|
86
|
-
>>> i = InvigilatorAI.example(question = q, scenario = s)
|
87
|
-
>>> i._determine_answer("1")
|
88
|
-
'Bad'
|
89
|
-
|
90
|
-
>>> from edsl import QuestionList, QuestionMultipleChoice, Survey
|
91
|
-
>>> q1 = QuestionList(question_name = "favs", question_text = "What are your top 3 colors?")
|
92
|
-
>>> q2 = QuestionMultipleChoice(question_text = "What is your favorite color?", question_name = "best", question_options = "{{ favs.answer }}", use_code = True)
|
93
|
-
>>> survey = Survey([q1, q2])
|
94
|
-
>>> i = InvigilatorAI.example(question = q2, scenario = s, survey = survey)
|
95
|
-
>>> i.current_answers = {"favs": ["Green", "Blue", "Red"]}
|
96
|
-
>>> i._determine_answer("2")
|
97
|
-
'Red'
|
98
|
-
"""
|
99
|
-
substitution_dict = self._prepare_substitution_dict(
|
100
|
-
self.survey, self.current_answers, self.scenario
|
101
|
-
)
|
102
|
-
return self.question._translate_answer_code_to_answer(
|
103
|
-
raw_answer, substitution_dict
|
104
|
-
)
|
105
|
-
|
106
|
-
@staticmethod
|
107
|
-
def _prepare_substitution_dict(
|
108
|
-
survey: "Survey", current_answers: dict, scenario: "Scenario"
|
109
|
-
) -> Dict[str, Any]:
|
110
|
-
"""Prepares a substitution dictionary for the question based on the survey, current answers, and scenario.
|
111
|
-
|
112
|
-
This is necessary beause sometimes the model's answer to a question could depend on details in
|
113
|
-
the prompt that were provided by the answer to a previous question or a scenario detail.
|
114
|
-
|
115
|
-
Note that the question object is getting the answer & a the comment appended to it, as the
|
116
|
-
jinja2 template might be referencing these values with a dot notation.
|
117
|
-
|
118
|
-
"""
|
119
|
-
question_dict = survey.duplicate().question_names_to_questions()
|
120
|
-
|
66
|
+
def determine_answer(self, raw_answer: str) -> Any:
|
67
|
+
question_dict = self.survey.question_names_to_questions()
|
121
68
|
# iterates through the current answers and updates the question_dict (which is all questions)
|
122
|
-
for other_question, answer in current_answers.items():
|
69
|
+
for other_question, answer in self.current_answers.items():
|
123
70
|
if other_question in question_dict:
|
124
71
|
question_dict[other_question].answer = answer
|
125
72
|
else:
|
@@ -129,12 +76,13 @@ class InvigilatorAI(InvigilatorBase):
|
|
129
76
|
) in question_dict:
|
130
77
|
question_dict[new_question].comment = answer
|
131
78
|
|
132
|
-
|
79
|
+
combined_dict = {**question_dict, **self.scenario}
|
80
|
+
# sometimes the answer is a code, so we need to translate it
|
81
|
+
return self.question._translate_answer_code_to_answer(raw_answer, combined_dict)
|
133
82
|
|
134
|
-
def
|
83
|
+
def extract_edsl_result_entry_and_validate(
|
135
84
|
self, agent_response_dict: AgentResponseDict
|
136
85
|
) -> EDSLResultObjectInput:
|
137
|
-
"""Extract the EDSL result entry and validate it."""
|
138
86
|
edsl_dict = agent_response_dict.edsl_dict._asdict()
|
139
87
|
exception_occurred = None
|
140
88
|
validated = False
|
@@ -146,8 +94,10 @@ class InvigilatorAI(InvigilatorBase):
|
|
146
94
|
# question options have be treated differently because of dynamic question
|
147
95
|
# this logic is all in the prompt constructor
|
148
96
|
if "question_options" in self.question.data:
|
149
|
-
new_question_options =
|
150
|
-
self.
|
97
|
+
new_question_options = (
|
98
|
+
self.prompt_constructor._get_question_options(
|
99
|
+
self.question.data
|
100
|
+
)
|
151
101
|
)
|
152
102
|
if new_question_options != self.question.data["question_options"]:
|
153
103
|
# I don't love this direct writing but it seems to work
|
@@ -160,8 +110,9 @@ class InvigilatorAI(InvigilatorBase):
|
|
160
110
|
else:
|
161
111
|
question_with_validators = self.question
|
162
112
|
|
113
|
+
# breakpoint()
|
163
114
|
validated_edsl_dict = question_with_validators._validate_answer(edsl_dict)
|
164
|
-
answer = self.
|
115
|
+
answer = self.determine_answer(validated_edsl_dict["answer"])
|
165
116
|
comment = validated_edsl_dict.get("comment", "")
|
166
117
|
validated = True
|
167
118
|
except QuestionAnswerValidationError as e:
|
@@ -231,13 +182,13 @@ class InvigilatorHuman(InvigilatorBase):
|
|
231
182
|
exception_occurred = e
|
232
183
|
finally:
|
233
184
|
data = {
|
234
|
-
"generated_tokens":
|
185
|
+
"generated_tokens": NotApplicable(),
|
235
186
|
"question_name": self.question.question_name,
|
236
187
|
"prompts": self.get_prompts(),
|
237
|
-
"cached_response":
|
238
|
-
"raw_model_response":
|
239
|
-
"cache_used":
|
240
|
-
"cache_key":
|
188
|
+
"cached_response": NotApplicable(),
|
189
|
+
"raw_model_response": NotApplicable(),
|
190
|
+
"cache_used": NotApplicable(),
|
191
|
+
"cache_key": NotApplicable(),
|
241
192
|
"answer": answer,
|
242
193
|
"comment": comment,
|
243
194
|
"validated": validated,
|
@@ -258,19 +209,17 @@ class InvigilatorFunctional(InvigilatorBase):
|
|
258
209
|
generated_tokens=str(answer),
|
259
210
|
question_name=self.question.question_name,
|
260
211
|
prompts=self.get_prompts(),
|
261
|
-
cached_response=
|
262
|
-
raw_model_response=
|
263
|
-
cache_used=
|
264
|
-
cache_key=
|
212
|
+
cached_response=NotApplicable(),
|
213
|
+
raw_model_response=NotApplicable(),
|
214
|
+
cache_used=NotApplicable(),
|
215
|
+
cache_key=NotApplicable(),
|
265
216
|
answer=answer["answer"],
|
266
217
|
comment="This is the result of a functional question.",
|
267
218
|
validated=True,
|
268
219
|
exception_occurred=None,
|
269
220
|
)
|
270
221
|
|
271
|
-
def get_prompts(self) -> Dict[str,
|
272
|
-
from edsl.prompts.Prompt import Prompt
|
273
|
-
|
222
|
+
def get_prompts(self) -> Dict[str, Prompt]:
|
274
223
|
"""Return the prompts used."""
|
275
224
|
return {
|
276
225
|
"user_prompt": Prompt("NA"),
|