edsl 0.1.31__py3-none-any.whl → 0.1.31.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__version__.py +1 -1
- edsl/agents/Invigilator.py +2 -7
- edsl/agents/PromptConstructionMixin.py +1 -18
- edsl/config.py +0 -4
- edsl/conjure/Conjure.py +0 -6
- edsl/coop/coop.py +0 -4
- edsl/data/CacheHandler.py +4 -3
- edsl/enums.py +0 -2
- edsl/inference_services/DeepInfraService.py +91 -6
- edsl/inference_services/InferenceServicesCollection.py +5 -13
- edsl/inference_services/OpenAIService.py +21 -64
- edsl/inference_services/registry.py +1 -2
- edsl/jobs/Jobs.py +33 -80
- edsl/jobs/buckets/TokenBucket.py +4 -12
- edsl/jobs/interviews/Interview.py +9 -31
- edsl/jobs/interviews/InterviewTaskBuildingMixin.py +33 -49
- edsl/jobs/interviews/interview_exception_tracking.py +10 -68
- edsl/jobs/runners/JobsRunnerAsyncio.py +81 -112
- edsl/jobs/runners/JobsRunnerStatusData.py +237 -0
- edsl/jobs/runners/JobsRunnerStatusMixin.py +35 -291
- edsl/jobs/tasks/TaskCreators.py +2 -8
- edsl/jobs/tasks/TaskHistory.py +1 -145
- edsl/language_models/LanguageModel.py +74 -127
- edsl/language_models/registry.py +0 -4
- edsl/questions/QuestionMultipleChoice.py +0 -1
- edsl/questions/QuestionNumerical.py +1 -0
- edsl/results/DatasetExportMixin.py +3 -12
- edsl/scenarios/Scenario.py +0 -14
- edsl/scenarios/ScenarioList.py +2 -15
- edsl/scenarios/ScenarioListExportMixin.py +4 -15
- edsl/scenarios/ScenarioListPdfMixin.py +0 -3
- {edsl-0.1.31.dist-info → edsl-0.1.31.dev1.dist-info}/METADATA +2 -3
- {edsl-0.1.31.dist-info → edsl-0.1.31.dev1.dist-info}/RECORD +35 -37
- edsl/inference_services/GroqService.py +0 -18
- edsl/jobs/interviews/InterviewExceptionEntry.py +0 -101
- {edsl-0.1.31.dist-info → edsl-0.1.31.dev1.dist-info}/LICENSE +0 -0
- {edsl-0.1.31.dist-info → edsl-0.1.31.dev1.dist-info}/WHEEL +0 -0
edsl/jobs/tasks/TaskHistory.py
CHANGED
@@ -11,8 +11,6 @@ class TaskHistory:
|
|
11
11
|
|
12
12
|
[Interview.exceptions, Interview.exceptions, Interview.exceptions, ...]
|
13
13
|
|
14
|
-
>>> _ = TaskHistory.example()
|
15
|
-
...
|
16
14
|
"""
|
17
15
|
|
18
16
|
self.total_interviews = interviews
|
@@ -20,26 +18,8 @@ class TaskHistory:
|
|
20
18
|
|
21
19
|
self._interviews = {index: i for index, i in enumerate(self.total_interviews)}
|
22
20
|
|
23
|
-
@classmethod
|
24
|
-
def example(cls):
|
25
|
-
from edsl.jobs.interviews.Interview import Interview
|
26
|
-
|
27
|
-
from edsl.jobs.Jobs import Jobs
|
28
|
-
|
29
|
-
j = Jobs.example(throw_exception_probability=1, test_model=True)
|
30
|
-
|
31
|
-
from edsl.config import CONFIG
|
32
|
-
|
33
|
-
results = j.run(print_exceptions=False, skip_retry=True, cache = False)
|
34
|
-
|
35
|
-
return cls(results.task_history.total_interviews)
|
36
|
-
|
37
21
|
@property
|
38
22
|
def exceptions(self):
|
39
|
-
"""
|
40
|
-
>>> len(TaskHistory.example().exceptions)
|
41
|
-
4
|
42
|
-
"""
|
43
23
|
return [i.exceptions for k, i in self._interviews.items() if i.exceptions != {}]
|
44
24
|
|
45
25
|
@property
|
@@ -62,12 +42,7 @@ class TaskHistory:
|
|
62
42
|
|
63
43
|
@property
|
64
44
|
def has_exceptions(self) -> bool:
|
65
|
-
"""Return True if there are any exceptions.
|
66
|
-
|
67
|
-
>>> TaskHistory.example().has_exceptions
|
68
|
-
True
|
69
|
-
|
70
|
-
"""
|
45
|
+
"""Return True if there are any exceptions."""
|
71
46
|
return len(self.exceptions) > 0
|
72
47
|
|
73
48
|
def _repr_html_(self):
|
@@ -241,47 +216,6 @@ class TaskHistory:
|
|
241
216
|
}
|
242
217
|
"""
|
243
218
|
|
244
|
-
@property
|
245
|
-
def exceptions_by_type(self) -> dict:
|
246
|
-
"""Return a dictionary of exceptions by type."""
|
247
|
-
exceptions_by_type = {}
|
248
|
-
for interview in self.total_interviews:
|
249
|
-
for question_name, exceptions in interview.exceptions.items():
|
250
|
-
for exception in exceptions:
|
251
|
-
exception_type = exception["exception"]
|
252
|
-
if exception_type in exceptions_by_type:
|
253
|
-
exceptions_by_type[exception_type] += 1
|
254
|
-
else:
|
255
|
-
exceptions_by_type[exception_type] = 1
|
256
|
-
return exceptions_by_type
|
257
|
-
|
258
|
-
@property
|
259
|
-
def exceptions_by_question_name(self) -> dict:
|
260
|
-
"""Return a dictionary of exceptions tallied by question name."""
|
261
|
-
exceptions_by_question_name = {}
|
262
|
-
for interview in self.total_interviews:
|
263
|
-
for question_name, exceptions in interview.exceptions.items():
|
264
|
-
if question_name not in exceptions_by_question_name:
|
265
|
-
exceptions_by_question_name[question_name] = 0
|
266
|
-
exceptions_by_question_name[question_name] += len(exceptions)
|
267
|
-
|
268
|
-
for question in self.total_interviews[0].survey.questions:
|
269
|
-
if question.question_name not in exceptions_by_question_name:
|
270
|
-
exceptions_by_question_name[question.question_name] = 0
|
271
|
-
return exceptions_by_question_name
|
272
|
-
|
273
|
-
@property
|
274
|
-
def exceptions_by_model(self) -> dict:
|
275
|
-
"""Return a dictionary of exceptions tallied by model and question name."""
|
276
|
-
exceptions_by_model = {}
|
277
|
-
for interview in self.total_interviews:
|
278
|
-
model = interview.model
|
279
|
-
if model not in exceptions_by_model:
|
280
|
-
exceptions_by_model[model.model] = 0
|
281
|
-
if interview.exceptions != {}:
|
282
|
-
exceptions_by_model[model.model] += len(interview.exceptions)
|
283
|
-
return exceptions_by_model
|
284
|
-
|
285
219
|
def html(
|
286
220
|
self,
|
287
221
|
filename: Optional[str] = None,
|
@@ -302,8 +236,6 @@ class TaskHistory:
|
|
302
236
|
if css is None:
|
303
237
|
css = self.css()
|
304
238
|
|
305
|
-
models_used = set([i.model for index, i in self._interviews.items()])
|
306
|
-
|
307
239
|
template = Template(
|
308
240
|
"""
|
309
241
|
<!DOCTYPE html>
|
@@ -317,69 +249,6 @@ class TaskHistory:
|
|
317
249
|
</style>
|
318
250
|
</head>
|
319
251
|
<body>
|
320
|
-
<h1>Overview</h1>
|
321
|
-
<p>There were {{ interviews|length }} total interviews. The number of interviews with exceptions was {{ num_exceptions }}.</p>
|
322
|
-
<p>The models used were: {{ models_used }}.</p>
|
323
|
-
<p>For documentation on dealing with exceptions on Expected Parrot,
|
324
|
-
see <a href="https://docs.expectedparrot.com/en/latest/exceptions.html">here</a>.</p>
|
325
|
-
|
326
|
-
<h2>Exceptions by Type</h2>
|
327
|
-
<table>
|
328
|
-
<thead>
|
329
|
-
<tr>
|
330
|
-
<th>Exception Type</th>
|
331
|
-
<th>Number</th>
|
332
|
-
</tr>
|
333
|
-
</thead>
|
334
|
-
<tbody>
|
335
|
-
{% for exception_type, exceptions in exceptions_by_type.items() %}
|
336
|
-
<tr>
|
337
|
-
<td>{{ exception_type }}</td>
|
338
|
-
<td>{{ exceptions }}</td>
|
339
|
-
</tr>
|
340
|
-
{% endfor %}
|
341
|
-
</tbody>
|
342
|
-
</table>
|
343
|
-
|
344
|
-
|
345
|
-
<h2>Exceptions by Model</h2>
|
346
|
-
<table>
|
347
|
-
<thead>
|
348
|
-
<tr>
|
349
|
-
<th>Model</th>
|
350
|
-
<th>Number</th>
|
351
|
-
</tr>
|
352
|
-
</thead>
|
353
|
-
<tbody>
|
354
|
-
{% for model, exceptions in exceptions_by_model.items() %}
|
355
|
-
<tr>
|
356
|
-
<td>{{ model }}</td>
|
357
|
-
<td>{{ exceptions }}</td>
|
358
|
-
</tr>
|
359
|
-
{% endfor %}
|
360
|
-
</tbody>
|
361
|
-
</table>
|
362
|
-
|
363
|
-
|
364
|
-
<h2>Exceptions by Question Name</h2>
|
365
|
-
<table>
|
366
|
-
<thead>
|
367
|
-
<tr>
|
368
|
-
<th>Question Name</th>
|
369
|
-
<th>Number of Exceptions</th>
|
370
|
-
</tr>
|
371
|
-
</thead>
|
372
|
-
<tbody>
|
373
|
-
{% for question_name, exception_count in exceptions_by_question_name.items() %}
|
374
|
-
<tr>
|
375
|
-
<td>{{ question_name }}</td>
|
376
|
-
<td>{{ exception_count }}</td>
|
377
|
-
</tr>
|
378
|
-
{% endfor %}
|
379
|
-
</tbody>
|
380
|
-
</table>
|
381
|
-
|
382
|
-
|
383
252
|
{% for index, interview in interviews.items() %}
|
384
253
|
{% if interview.exceptions != {} %}
|
385
254
|
<div class="interview">Interview: {{ index }} </div>
|
@@ -427,18 +296,11 @@ class TaskHistory:
|
|
427
296
|
"""
|
428
297
|
)
|
429
298
|
|
430
|
-
# breakpoint()
|
431
|
-
|
432
299
|
# Render the template with data
|
433
300
|
output = template.render(
|
434
301
|
interviews=self._interviews,
|
435
302
|
css=css,
|
436
|
-
num_exceptions=len(self.exceptions),
|
437
303
|
performance_plot_html=performance_plot_html,
|
438
|
-
exceptions_by_type=self.exceptions_by_type,
|
439
|
-
exceptions_by_question_name=self.exceptions_by_question_name,
|
440
|
-
exceptions_by_model=self.exceptions_by_model,
|
441
|
-
models_used=models_used,
|
442
304
|
)
|
443
305
|
|
444
306
|
# Save the rendered output to a file
|
@@ -482,9 +344,3 @@ class TaskHistory:
|
|
482
344
|
|
483
345
|
if return_link:
|
484
346
|
return filename
|
485
|
-
|
486
|
-
|
487
|
-
if __name__ == "__main__":
|
488
|
-
import doctest
|
489
|
-
|
490
|
-
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
@@ -7,37 +7,9 @@ import asyncio
|
|
7
7
|
import json
|
8
8
|
import time
|
9
9
|
import os
|
10
|
-
import hashlib
|
11
10
|
from typing import Coroutine, Any, Callable, Type, List, get_type_hints
|
12
11
|
from abc import ABC, abstractmethod
|
13
12
|
|
14
|
-
|
15
|
-
class IntendedModelCallOutcome:
|
16
|
-
"This is a tuple-like class that holds the response, cache_used, and cache_key."
|
17
|
-
|
18
|
-
def __init__(self, response: dict, cache_used: bool, cache_key: str):
|
19
|
-
self.response = response
|
20
|
-
self.cache_used = cache_used
|
21
|
-
self.cache_key = cache_key
|
22
|
-
|
23
|
-
def __iter__(self):
|
24
|
-
"""Iterate over the class attributes.
|
25
|
-
|
26
|
-
>>> a, b, c = IntendedModelCallOutcome({'answer': "yes"}, True, 'x1289')
|
27
|
-
>>> a
|
28
|
-
{'answer': 'yes'}
|
29
|
-
"""
|
30
|
-
yield self.response
|
31
|
-
yield self.cache_used
|
32
|
-
yield self.cache_key
|
33
|
-
|
34
|
-
def __len__(self):
|
35
|
-
return 3
|
36
|
-
|
37
|
-
def __repr__(self):
|
38
|
-
return f"IntendedModelCallOutcome(response = {self.response}, cache_used = {self.cache_used}, cache_key = '{self.cache_key}')"
|
39
|
-
|
40
|
-
|
41
13
|
from edsl.config import CONFIG
|
42
14
|
|
43
15
|
from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
|
@@ -124,11 +96,6 @@ class LanguageModel(
|
|
124
96
|
# Skip the API key check. Sometimes this is useful for testing.
|
125
97
|
self._api_token = None
|
126
98
|
|
127
|
-
def ask_question(self, question):
|
128
|
-
user_prompt = question.get_instructions().render(question.data).text
|
129
|
-
system_prompt = "You are a helpful agent pretending to be a human."
|
130
|
-
return self.execute_model_call(user_prompt, system_prompt)
|
131
|
-
|
132
99
|
@property
|
133
100
|
def api_token(self) -> str:
|
134
101
|
if not hasattr(self, "_api_token"):
|
@@ -182,7 +149,7 @@ class LanguageModel(
|
|
182
149
|
key_value = os.getenv(key_name)
|
183
150
|
return key_value is not None
|
184
151
|
|
185
|
-
def __hash__(self)
|
152
|
+
def __hash__(self):
|
186
153
|
"""Allow the model to be used as a key in a dictionary."""
|
187
154
|
from edsl.utilities.utilities import dict_hash
|
188
155
|
|
@@ -249,25 +216,19 @@ class LanguageModel(
|
|
249
216
|
>>> LanguageModel._overide_default_parameters(passed_parameter_dict={"temperature": 0.5}, default_parameter_dict={"temperature":0.9, "max_tokens": 1000})
|
250
217
|
{'temperature': 0.5, 'max_tokens': 1000}
|
251
218
|
"""
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
def __call__(self, user_prompt: str, system_prompt: str):
|
260
|
-
return self.execute_model_call(user_prompt, system_prompt)
|
219
|
+
parameters = dict({})
|
220
|
+
for parameter, default_value in default_parameter_dict.items():
|
221
|
+
if parameter in passed_parameter_dict:
|
222
|
+
parameters[parameter] = passed_parameter_dict[parameter]
|
223
|
+
else:
|
224
|
+
parameters[parameter] = default_value
|
225
|
+
return parameters
|
261
226
|
|
262
227
|
@abstractmethod
|
263
228
|
async def async_execute_model_call(user_prompt: str, system_prompt: str):
|
264
|
-
"""Execute the model call and returns a coroutine.
|
229
|
+
"""Execute the model call and returns the result as a coroutine.
|
265
230
|
|
266
231
|
>>> m = LanguageModel.example(test_model = True)
|
267
|
-
>>> async def test(): return await m.async_execute_model_call("Hello, model!", "You are a helpful agent.")
|
268
|
-
>>> asyncio.run(test())
|
269
|
-
{'message': '{"answer": "Hello world"}'}
|
270
|
-
|
271
232
|
>>> m.execute_model_call("Hello, model!", "You are a helpful agent.")
|
272
233
|
{'message': '{"answer": "Hello world"}'}
|
273
234
|
|
@@ -313,38 +274,11 @@ class LanguageModel(
|
|
313
274
|
|
314
275
|
What is returned by the API is model-specific and often includes meta-data that we do not need.
|
315
276
|
For example, here is the results from a call to GPT-4:
|
316
|
-
To actually
|
277
|
+
To actually tract the response, we need to grab
|
317
278
|
data["choices[0]"]["message"]["content"].
|
318
279
|
"""
|
319
280
|
raise NotImplementedError
|
320
281
|
|
321
|
-
async def _async_prepare_response(
|
322
|
-
self, model_call_outcome: IntendedModelCallOutcome, cache: "Cache"
|
323
|
-
) -> dict:
|
324
|
-
"""Prepare the response for return."""
|
325
|
-
|
326
|
-
model_response = {
|
327
|
-
"cache_used": model_call_outcome.cache_used,
|
328
|
-
"cache_key": model_call_outcome.cache_key,
|
329
|
-
"usage": model_call_outcome.response.get("usage", {}),
|
330
|
-
"raw_model_response": model_call_outcome.response,
|
331
|
-
}
|
332
|
-
|
333
|
-
answer_portion = self.parse_response(model_call_outcome.response)
|
334
|
-
try:
|
335
|
-
answer_dict = json.loads(answer_portion)
|
336
|
-
except json.JSONDecodeError as e:
|
337
|
-
# TODO: Turn into logs to generate issues
|
338
|
-
answer_dict, success = await repair(
|
339
|
-
bad_json=answer_portion, error_message=str(e), cache=cache
|
340
|
-
)
|
341
|
-
if not success:
|
342
|
-
raise Exception(
|
343
|
-
f"""Even the repair failed. The error was: {e}. The response was: {answer_portion}."""
|
344
|
-
)
|
345
|
-
|
346
|
-
return {**model_response, **answer_dict}
|
347
|
-
|
348
282
|
async def async_get_raw_response(
|
349
283
|
self,
|
350
284
|
user_prompt: str,
|
@@ -352,28 +286,7 @@ class LanguageModel(
|
|
352
286
|
cache: "Cache",
|
353
287
|
iteration: int = 0,
|
354
288
|
encoded_image=None,
|
355
|
-
) ->
|
356
|
-
import warnings
|
357
|
-
|
358
|
-
warnings.warn(
|
359
|
-
"This method is deprecated. Use async_get_intended_model_call_outcome."
|
360
|
-
)
|
361
|
-
return await self._async_get_intended_model_call_outcome(
|
362
|
-
user_prompt=user_prompt,
|
363
|
-
system_prompt=system_prompt,
|
364
|
-
cache=cache,
|
365
|
-
iteration=iteration,
|
366
|
-
encoded_image=encoded_image,
|
367
|
-
)
|
368
|
-
|
369
|
-
async def _async_get_intended_model_call_outcome(
|
370
|
-
self,
|
371
|
-
user_prompt: str,
|
372
|
-
system_prompt: str,
|
373
|
-
cache: "Cache",
|
374
|
-
iteration: int = 0,
|
375
|
-
encoded_image=None,
|
376
|
-
) -> IntendedModelCallOutcome:
|
289
|
+
) -> tuple[dict, bool, str]:
|
377
290
|
"""Handle caching of responses.
|
378
291
|
|
379
292
|
:param user_prompt: The user's prompt.
|
@@ -391,49 +304,52 @@ class LanguageModel(
|
|
391
304
|
|
392
305
|
>>> from edsl import Cache
|
393
306
|
>>> m = LanguageModel.example(test_model = True)
|
394
|
-
>>> m.
|
395
|
-
|
307
|
+
>>> m.get_raw_response(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
|
308
|
+
({'message': '{"answer": "Hello world"}'}, False, '24ff6ac2bc2f1729f817f261e0792577')
|
396
309
|
"""
|
397
|
-
|
398
|
-
if encoded_image:
|
399
|
-
# the image has is appended to the user_prompt for hash-lookup purposes
|
400
|
-
image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
|
310
|
+
start_time = time.time()
|
401
311
|
|
402
312
|
cache_call_params = {
|
403
313
|
"model": str(self.model),
|
404
314
|
"parameters": self.parameters,
|
405
315
|
"system_prompt": system_prompt,
|
406
|
-
"user_prompt": user_prompt
|
316
|
+
"user_prompt": user_prompt,
|
407
317
|
"iteration": iteration,
|
408
318
|
}
|
409
|
-
cached_response, cache_key = cache.fetch(**cache_call_params)
|
410
319
|
|
411
|
-
if
|
320
|
+
if encoded_image:
|
321
|
+
import hashlib
|
322
|
+
|
323
|
+
image_hash = hashlib.md5(encoded_image.encode()).hexdigest()
|
324
|
+
cache_call_params["user_prompt"] = f"{user_prompt} {image_hash}"
|
325
|
+
|
326
|
+
cached_response, cache_key = cache.fetch(**cache_call_params)
|
327
|
+
if cached_response:
|
412
328
|
response = json.loads(cached_response)
|
329
|
+
cache_used = True
|
413
330
|
else:
|
331
|
+
remote_call = hasattr(self, "remote") and self.remote
|
414
332
|
f = (
|
415
333
|
self.remote_async_execute_model_call
|
416
|
-
if
|
334
|
+
if remote_call
|
417
335
|
else self.async_execute_model_call
|
418
336
|
)
|
419
|
-
params = {
|
420
|
-
|
421
|
-
"
|
422
|
-
**({"encoded_image": encoded_image} if encoded_image else {}),
|
423
|
-
}
|
337
|
+
params = {"user_prompt": user_prompt, "system_prompt": system_prompt}
|
338
|
+
if encoded_image:
|
339
|
+
params["encoded_image"] = encoded_image
|
424
340
|
response = await f(**params)
|
425
341
|
new_cache_key = cache.store(
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
342
|
+
user_prompt=user_prompt,
|
343
|
+
model=str(self.model),
|
344
|
+
parameters=self.parameters,
|
345
|
+
system_prompt=system_prompt,
|
346
|
+
response=response,
|
347
|
+
iteration=iteration,
|
348
|
+
)
|
349
|
+
assert new_cache_key == cache_key
|
350
|
+
cache_used = False
|
433
351
|
|
434
|
-
|
435
|
-
_async_get_intended_model_call_outcome
|
436
|
-
)
|
352
|
+
return response, cache_used, cache_key
|
437
353
|
|
438
354
|
get_raw_response = sync_wrapper(async_get_raw_response)
|
439
355
|
|
@@ -454,7 +370,7 @@ class LanguageModel(
|
|
454
370
|
self,
|
455
371
|
user_prompt: str,
|
456
372
|
system_prompt: str,
|
457
|
-
cache:
|
373
|
+
cache: Cache,
|
458
374
|
iteration: int = 1,
|
459
375
|
encoded_image=None,
|
460
376
|
) -> dict:
|
@@ -472,10 +388,36 @@ class LanguageModel(
|
|
472
388
|
"system_prompt": system_prompt,
|
473
389
|
"iteration": iteration,
|
474
390
|
"cache": cache,
|
475
|
-
**({"encoded_image": encoded_image} if encoded_image else {}),
|
476
391
|
}
|
477
|
-
|
478
|
-
|
392
|
+
if encoded_image:
|
393
|
+
params["encoded_image"] = encoded_image
|
394
|
+
|
395
|
+
raw_response, cache_used, cache_key = await self.async_get_raw_response(
|
396
|
+
**params
|
397
|
+
)
|
398
|
+
response = self.parse_response(raw_response)
|
399
|
+
|
400
|
+
try:
|
401
|
+
dict_response = json.loads(response)
|
402
|
+
except json.JSONDecodeError as e:
|
403
|
+
# TODO: Turn into logs to generate issues
|
404
|
+
dict_response, success = await repair(
|
405
|
+
bad_json=response, error_message=str(e), cache=cache
|
406
|
+
)
|
407
|
+
if not success:
|
408
|
+
raise Exception(
|
409
|
+
f"""Even the repair failed. The error was: {e}. The response was: {response}."""
|
410
|
+
)
|
411
|
+
|
412
|
+
dict_response.update(
|
413
|
+
{
|
414
|
+
"cache_used": cache_used,
|
415
|
+
"cache_key": cache_key,
|
416
|
+
"usage": raw_response.get("usage", {}),
|
417
|
+
"raw_model_response": raw_response,
|
418
|
+
}
|
419
|
+
)
|
420
|
+
return dict_response
|
479
421
|
|
480
422
|
get_response = sync_wrapper(async_get_response)
|
481
423
|
|
@@ -601,3 +543,8 @@ if __name__ == "__main__":
|
|
601
543
|
import doctest
|
602
544
|
|
603
545
|
doctest.testmod(optionflags=doctest.ELLIPSIS)
|
546
|
+
|
547
|
+
# from edsl.language_models import LanguageModel
|
548
|
+
|
549
|
+
# from edsl.language_models import LanguageModel
|
550
|
+
# print(LanguageModel.example())
|
edsl/language_models/registry.py
CHANGED
@@ -36,10 +36,6 @@ class Model(metaclass=Meta):
|
|
36
36
|
from edsl.inference_services.registry import default
|
37
37
|
|
38
38
|
registry = registry or default
|
39
|
-
|
40
|
-
if isinstance(model_name, int):
|
41
|
-
model_name = cls.available(name_only=True)[model_name]
|
42
|
-
|
43
39
|
factory = registry.create_model_factory(model_name)
|
44
40
|
return factory(*args, **kwargs)
|
45
41
|
|
@@ -26,6 +26,7 @@ class QuestionNumerical(QuestionBase):
|
|
26
26
|
|
27
27
|
:param question_name: The name of the question.
|
28
28
|
:param question_text: The text of the question.
|
29
|
+
:param instructions: Instructions for the question. If not provided, the default instructions are used. To view them, run `QuestionNumerical.default_instructions`.
|
29
30
|
:param min_value: The minimum value of the answer.
|
30
31
|
:param max_value: The maximum value of the answer.
|
31
32
|
"""
|
@@ -27,10 +27,6 @@ class DatasetExportMixin:
|
|
27
27
|
>>> d.relevant_columns(remove_prefix=True)
|
28
28
|
['b']
|
29
29
|
|
30
|
-
>>> d = Dataset([{'a':[1,2,3,4]}, {'b':[5,6,7,8]}])
|
31
|
-
>>> d.relevant_columns()
|
32
|
-
['a', 'b']
|
33
|
-
|
34
30
|
>>> from edsl.results import Results; Results.example().select('how_feeling', 'how_feeling_yesterday').relevant_columns()
|
35
31
|
['answer.how_feeling', 'answer.how_feeling_yesterday']
|
36
32
|
|
@@ -597,7 +593,7 @@ class DatasetExportMixin:
|
|
597
593
|
return filename
|
598
594
|
|
599
595
|
def tally(
|
600
|
-
self, *fields: Optional[str], top_n: Optional[int] = None, output="
|
596
|
+
self, *fields: Optional[str], top_n: Optional[int] = None, output="dict"
|
601
597
|
) -> Union[dict, "Dataset"]:
|
602
598
|
"""Tally the values of a field or perform a cross-tab of multiple fields.
|
603
599
|
|
@@ -605,11 +601,9 @@ class DatasetExportMixin:
|
|
605
601
|
|
606
602
|
>>> from edsl.results import Results
|
607
603
|
>>> r = Results.example()
|
608
|
-
>>> r.select('how_feeling').tally('answer.how_feeling'
|
604
|
+
>>> r.select('how_feeling').tally('answer.how_feeling')
|
609
605
|
{'OK': 2, 'Great': 1, 'Terrible': 1}
|
610
|
-
>>> r.select('how_feeling').tally('
|
611
|
-
Dataset([{'value': ['OK', 'Great', 'Terrible']}, {'count': [2, 1, 1]}])
|
612
|
-
>>> r.select('how_feeling', 'period').tally('how_feeling', 'period', output = "dict")
|
606
|
+
>>> r.select('how_feeling', 'period').tally('how_feeling', 'period')
|
613
607
|
{('OK', 'morning'): 1, ('Great', 'afternoon'): 1, ('Terrible', 'morning'): 1, ('OK', 'afternoon'): 1}
|
614
608
|
"""
|
615
609
|
from collections import Counter
|
@@ -621,8 +615,6 @@ class DatasetExportMixin:
|
|
621
615
|
column.split(".")[-1] for column in self.relevant_columns()
|
622
616
|
]
|
623
617
|
|
624
|
-
# breakpoint()
|
625
|
-
|
626
618
|
if not all(
|
627
619
|
f in self.relevant_columns() or f in relevant_columns_without_prefix
|
628
620
|
for f in fields
|
@@ -649,7 +641,6 @@ class DatasetExportMixin:
|
|
649
641
|
from edsl.results.Dataset import Dataset
|
650
642
|
|
651
643
|
if output == "dict":
|
652
|
-
# why did I do this?
|
653
644
|
warnings.warn(
|
654
645
|
textwrap.dedent(
|
655
646
|
"""\
|
edsl/scenarios/Scenario.py
CHANGED
@@ -182,19 +182,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
|
|
182
182
|
new_scenario[key] = self[key]
|
183
183
|
return new_scenario
|
184
184
|
|
185
|
-
@classmethod
|
186
|
-
def from_url(cls, url: str, field_name: Optional[str] = "text") -> "Scenario":
|
187
|
-
"""Creates a scenario from a URL.
|
188
|
-
|
189
|
-
:param url: The URL to create the scenario from.
|
190
|
-
:param field_name: The field name to use for the text.
|
191
|
-
|
192
|
-
"""
|
193
|
-
import requests
|
194
|
-
|
195
|
-
text = requests.get(url).text
|
196
|
-
return cls({"url": url, field_name: text})
|
197
|
-
|
198
185
|
@classmethod
|
199
186
|
def from_image(cls, image_path: str) -> str:
|
200
187
|
"""Creates a scenario with a base64 encoding of an image.
|
@@ -220,7 +207,6 @@ class Scenario(Base, UserDict, ScenarioImageMixin, ScenarioHtmlMixin):
|
|
220
207
|
@classmethod
|
221
208
|
def from_pdf(cls, pdf_path):
|
222
209
|
import fitz # PyMuPDF
|
223
|
-
from edsl import Scenario
|
224
210
|
|
225
211
|
# Ensure the file exists
|
226
212
|
if not os.path.exists(pdf_path):
|
edsl/scenarios/ScenarioList.py
CHANGED
@@ -242,16 +242,6 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
|
|
242
242
|
|
243
243
|
return ScenarioList(new_data)
|
244
244
|
|
245
|
-
def from_urls(self, urls: list[str], field_name: Optional[str] = "text") -> ScenarioList:
|
246
|
-
"""Create a ScenarioList from a list of URLs.
|
247
|
-
|
248
|
-
:param urls: A list of URLs.
|
249
|
-
:param field_name: The name of the field to store the text from the URLs.
|
250
|
-
|
251
|
-
|
252
|
-
"""
|
253
|
-
return ScenarioList([Scenario.from_url(url, field_name) for url in urls])
|
254
|
-
|
255
245
|
def select(self, *fields) -> ScenarioList:
|
256
246
|
"""
|
257
247
|
Selects scenarios with only the references fields.
|
@@ -298,15 +288,12 @@ class ScenarioList(Base, UserList, ScenarioListMixin):
|
|
298
288
|
>>> s = ScenarioList.from_list("a", [1,2,3])
|
299
289
|
>>> s.to_dataset()
|
300
290
|
Dataset([{'a': [1, 2, 3]}])
|
301
|
-
>>> s = ScenarioList.from_list("a", [1,2,3]).add_list("b", [4,5,6])
|
302
|
-
>>> s.to_dataset()
|
303
|
-
Dataset([{'a': [1, 2, 3]}, {'b': [4, 5, 6]}])
|
304
291
|
"""
|
305
292
|
from edsl.results.Dataset import Dataset
|
306
293
|
|
307
294
|
keys = self[0].keys()
|
308
|
-
data =
|
309
|
-
return Dataset(data)
|
295
|
+
data = {key: [scenario[key] for scenario in self.data] for key in keys}
|
296
|
+
return Dataset([data])
|
310
297
|
|
311
298
|
def add_list(self, name, values) -> ScenarioList:
|
312
299
|
"""Add a list of values to a ScenarioList.
|
@@ -20,24 +20,13 @@ def to_dataset(func):
|
|
20
20
|
return wrapper
|
21
21
|
|
22
22
|
|
23
|
-
def
|
24
|
-
for attr_name, attr_value in
|
25
|
-
if callable(attr_value)
|
23
|
+
def decorate_all_methods(cls):
|
24
|
+
for attr_name, attr_value in cls.__dict__.items():
|
25
|
+
if callable(attr_value):
|
26
26
|
setattr(cls, attr_name, to_dataset(attr_value))
|
27
27
|
return cls
|
28
28
|
|
29
29
|
|
30
|
-
|
31
|
-
# for attr_name, attr_value in cls.__dict__.items():
|
32
|
-
# if callable(attr_value):
|
33
|
-
# setattr(cls, attr_name, to_dataset(attr_value))
|
34
|
-
# return cls
|
35
|
-
|
36
|
-
|
37
|
-
# @decorate_all_methods
|
30
|
+
@decorate_all_methods
|
38
31
|
class ScenarioListExportMixin(DatasetExportMixin):
|
39
32
|
"""Mixin class for exporting Results objects."""
|
40
|
-
|
41
|
-
def __init_subclass__(cls, **kwargs):
|
42
|
-
super().__init_subclass__(**kwargs)
|
43
|
-
decorate_methods_from_mixin(cls, DatasetExportMixin)
|
@@ -43,9 +43,6 @@ class ScenarioListPdfMixin:
|
|
43
43
|
|
44
44
|
@staticmethod
|
45
45
|
def extract_text_from_pdf(pdf_path):
|
46
|
-
from edsl import Scenario
|
47
|
-
|
48
|
-
# TODO: Add test case
|
49
46
|
# Ensure the file exists
|
50
47
|
if not os.path.exists(pdf_path):
|
51
48
|
raise FileNotFoundError(f"The file {pdf_path} does not exist.")
|