edsl 0.1.50__py3-none-any.whl → 0.1.52__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- edsl/__init__.py +45 -34
- edsl/__version__.py +1 -1
- edsl/base/base_exception.py +2 -2
- edsl/buckets/bucket_collection.py +1 -1
- edsl/buckets/exceptions.py +32 -0
- edsl/buckets/token_bucket_api.py +26 -10
- edsl/caching/cache.py +5 -2
- edsl/caching/remote_cache_sync.py +5 -5
- edsl/caching/sql_dict.py +12 -11
- edsl/config/__init__.py +1 -1
- edsl/config/config_class.py +4 -2
- edsl/conversation/Conversation.py +9 -5
- edsl/conversation/car_buying.py +1 -3
- edsl/conversation/mug_negotiation.py +2 -6
- edsl/coop/__init__.py +11 -8
- edsl/coop/coop.py +15 -13
- edsl/coop/coop_functions.py +1 -1
- edsl/coop/ep_key_handling.py +1 -1
- edsl/coop/price_fetcher.py +2 -2
- edsl/coop/utils.py +2 -2
- edsl/dataset/dataset.py +144 -63
- edsl/dataset/dataset_operations_mixin.py +14 -6
- edsl/dataset/dataset_tree.py +3 -3
- edsl/dataset/display/table_renderers.py +6 -3
- edsl/dataset/file_exports.py +4 -4
- edsl/dataset/r/ggplot.py +3 -3
- edsl/inference_services/available_model_fetcher.py +2 -2
- edsl/inference_services/data_structures.py +5 -5
- edsl/inference_services/inference_service_abc.py +1 -1
- edsl/inference_services/inference_services_collection.py +1 -1
- edsl/inference_services/service_availability.py +3 -3
- edsl/inference_services/services/azure_ai.py +3 -3
- edsl/inference_services/services/google_service.py +1 -1
- edsl/inference_services/services/test_service.py +1 -1
- edsl/instructions/change_instruction.py +5 -4
- edsl/instructions/instruction.py +1 -0
- edsl/instructions/instruction_collection.py +5 -4
- edsl/instructions/instruction_handler.py +10 -8
- edsl/interviews/answering_function.py +20 -21
- edsl/interviews/exception_tracking.py +3 -2
- edsl/interviews/interview.py +1 -1
- edsl/interviews/interview_status_dictionary.py +1 -1
- edsl/interviews/interview_task_manager.py +7 -4
- edsl/interviews/request_token_estimator.py +3 -2
- edsl/interviews/statistics.py +2 -2
- edsl/invigilators/invigilators.py +34 -6
- edsl/jobs/__init__.py +39 -2
- edsl/jobs/async_interview_runner.py +1 -1
- edsl/jobs/check_survey_scenario_compatibility.py +5 -5
- edsl/jobs/data_structures.py +2 -2
- edsl/jobs/html_table_job_logger.py +494 -257
- edsl/jobs/jobs.py +2 -2
- edsl/jobs/jobs_checks.py +5 -5
- edsl/jobs/jobs_component_constructor.py +2 -2
- edsl/jobs/jobs_pricing_estimation.py +1 -1
- edsl/jobs/jobs_runner_asyncio.py +2 -2
- edsl/jobs/jobs_status_enums.py +1 -0
- edsl/jobs/remote_inference.py +47 -13
- edsl/jobs/results_exceptions_handler.py +2 -2
- edsl/language_models/language_model.py +151 -145
- edsl/notebooks/__init__.py +24 -1
- edsl/notebooks/exceptions.py +82 -0
- edsl/notebooks/notebook.py +7 -3
- edsl/notebooks/notebook_to_latex.py +1 -1
- edsl/prompts/__init__.py +23 -2
- edsl/prompts/prompt.py +1 -1
- edsl/questions/__init__.py +4 -4
- edsl/questions/answer_validator_mixin.py +0 -5
- edsl/questions/compose_questions.py +2 -2
- edsl/questions/descriptors.py +1 -1
- edsl/questions/question_base.py +32 -3
- edsl/questions/question_base_prompts_mixin.py +4 -4
- edsl/questions/question_budget.py +503 -102
- edsl/questions/question_check_box.py +658 -156
- edsl/questions/question_dict.py +176 -2
- edsl/questions/question_extract.py +401 -61
- edsl/questions/question_free_text.py +77 -9
- edsl/questions/question_functional.py +118 -9
- edsl/questions/{derived/question_likert_five.py → question_likert_five.py} +2 -2
- edsl/questions/{derived/question_linear_scale.py → question_linear_scale.py} +3 -4
- edsl/questions/question_list.py +246 -26
- edsl/questions/question_matrix.py +586 -73
- edsl/questions/question_multiple_choice.py +213 -47
- edsl/questions/question_numerical.py +360 -29
- edsl/questions/question_rank.py +401 -124
- edsl/questions/question_registry.py +3 -3
- edsl/questions/{derived/question_top_k.py → question_top_k.py} +3 -3
- edsl/questions/{derived/question_yes_no.py → question_yes_no.py} +3 -4
- edsl/questions/register_questions_meta.py +2 -1
- edsl/questions/response_validator_abc.py +6 -2
- edsl/questions/response_validator_factory.py +10 -12
- edsl/results/report.py +1 -1
- edsl/results/result.py +7 -4
- edsl/results/results.py +500 -271
- edsl/results/results_selector.py +2 -2
- edsl/scenarios/construct_download_link.py +3 -3
- edsl/scenarios/scenario.py +1 -2
- edsl/scenarios/scenario_list.py +41 -23
- edsl/surveys/survey_css.py +3 -3
- edsl/surveys/survey_simulator.py +2 -1
- edsl/tasks/__init__.py +22 -2
- edsl/tasks/exceptions.py +72 -0
- edsl/tasks/task_history.py +48 -11
- edsl/templates/error_reporting/base.html +37 -4
- edsl/templates/error_reporting/exceptions_table.html +105 -33
- edsl/templates/error_reporting/interview_details.html +130 -126
- edsl/templates/error_reporting/overview.html +21 -25
- edsl/templates/error_reporting/report.css +215 -46
- edsl/templates/error_reporting/report.js +122 -20
- edsl/tokens/__init__.py +27 -1
- edsl/tokens/exceptions.py +37 -0
- edsl/tokens/interview_token_usage.py +3 -2
- edsl/tokens/token_usage.py +4 -3
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/METADATA +1 -1
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/RECORD +118 -116
- edsl/questions/derived/__init__.py +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/LICENSE +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/WHEEL +0 -0
- {edsl-0.1.50.dist-info → edsl-0.1.52.dist-info}/entry_points.txt +0 -0
@@ -10,7 +10,7 @@ Key concepts and terminology:
|
|
10
10
|
- raw_response: The complete JSON response returned directly from the model API.
|
11
11
|
Contains all model metadata and response information.
|
12
12
|
|
13
|
-
- edsl_augmented_response: The raw model response augmented with EDSL-specific
|
13
|
+
- edsl_augmented_response: The raw model response augmented with EDSL-specific
|
14
14
|
information, such as cache keys, token usage statistics, and cost data.
|
15
15
|
|
16
16
|
- generated_tokens: The actual text output generated by the model in response
|
@@ -62,19 +62,21 @@ from ..key_management import KeyLookupCollection
|
|
62
62
|
from .registry import RegisterLanguageModelsMeta
|
63
63
|
from .raw_response_handler import RawResponseHandler
|
64
64
|
|
65
|
+
|
65
66
|
def handle_key_error(func):
|
66
67
|
"""Decorator to catch and provide user-friendly error messages for KeyError exceptions.
|
67
|
-
|
68
|
+
|
68
69
|
This decorator gracefully handles KeyError exceptions that may occur when parsing
|
69
70
|
model responses with unexpected structures, providing a clear error message to
|
70
71
|
help users understand what went wrong.
|
71
|
-
|
72
|
+
|
72
73
|
Args:
|
73
74
|
func: The function to decorate
|
74
|
-
|
75
|
+
|
75
76
|
Returns:
|
76
77
|
Decorated function that catches KeyError exceptions
|
77
78
|
"""
|
79
|
+
|
78
80
|
@wraps(func)
|
79
81
|
def wrapper(*args, **kwargs):
|
80
82
|
try:
|
@@ -89,20 +91,21 @@ def handle_key_error(func):
|
|
89
91
|
|
90
92
|
class classproperty:
|
91
93
|
"""Descriptor that combines @classmethod and @property behaviors.
|
92
|
-
|
94
|
+
|
93
95
|
This descriptor allows defining properties that work on the class itself
|
94
96
|
rather than on instances, making it possible to have computed attributes
|
95
97
|
at the class level.
|
96
|
-
|
98
|
+
|
97
99
|
Usage:
|
98
100
|
class MyClass:
|
99
101
|
@classproperty
|
100
102
|
def my_prop(cls):
|
101
103
|
return cls.__name__
|
102
104
|
"""
|
105
|
+
|
103
106
|
def __init__(self, method):
|
104
107
|
"""Initialize with the decorated method.
|
105
|
-
|
108
|
+
|
106
109
|
Args:
|
107
110
|
method: The class method to be accessed as a property
|
108
111
|
"""
|
@@ -110,19 +113,17 @@ class classproperty:
|
|
110
113
|
|
111
114
|
def __get__(self, instance, cls):
|
112
115
|
"""Return the result of calling the method on the class.
|
113
|
-
|
116
|
+
|
114
117
|
Args:
|
115
118
|
instance: The instance (if called on an instance)
|
116
119
|
cls: The class (always provided)
|
117
|
-
|
120
|
+
|
118
121
|
Returns:
|
119
122
|
The result of calling the method with the class as argument
|
120
123
|
"""
|
121
124
|
return self.method(cls)
|
122
125
|
|
123
126
|
|
124
|
-
|
125
|
-
|
126
127
|
class LanguageModel(
|
127
128
|
PersistenceMixin,
|
128
129
|
RepresentationMixin,
|
@@ -131,19 +132,19 @@ class LanguageModel(
|
|
131
132
|
metaclass=RegisterLanguageModelsMeta,
|
132
133
|
):
|
133
134
|
"""Abstract base class for all language model implementations in EDSL.
|
134
|
-
|
135
|
+
|
135
136
|
This class defines the common interface and functionality for interacting with
|
136
137
|
various language model providers (OpenAI, Anthropic, etc.). It handles caching,
|
137
138
|
response parsing, token usage tracking, and cost calculation, providing a
|
138
139
|
consistent interface regardless of the underlying model.
|
139
|
-
|
140
|
+
|
140
141
|
Subclasses must implement the async_execute_model_call method to handle the
|
141
142
|
actual API call to the model provider. Other methods may also be overridden
|
142
143
|
to customize behavior for specific models.
|
143
|
-
|
144
|
+
|
144
145
|
The class uses several mixins to provide serialization, pretty printing, and
|
145
146
|
hashing functionality, and a metaclass to automatically register model implementations.
|
146
|
-
|
147
|
+
|
147
148
|
Attributes:
|
148
149
|
_model_: The default model identifier (set by subclasses)
|
149
150
|
key_sequence: Path to extract generated text from model responses
|
@@ -162,11 +163,11 @@ class LanguageModel(
|
|
162
163
|
@classproperty
|
163
164
|
def response_handler(cls):
|
164
165
|
"""Get a handler for processing raw model responses.
|
165
|
-
|
166
|
+
|
166
167
|
This property creates a RawResponseHandler configured for the specific
|
167
168
|
model implementation, using the class's key_sequence and usage_sequence
|
168
169
|
attributes to know how to extract information from the model's response format.
|
169
|
-
|
170
|
+
|
170
171
|
Returns:
|
171
172
|
RawResponseHandler: Handler configured for this model type
|
172
173
|
"""
|
@@ -183,31 +184,31 @@ class LanguageModel(
|
|
183
184
|
**kwargs,
|
184
185
|
):
|
185
186
|
"""Initialize a new language model instance.
|
186
|
-
|
187
|
+
|
187
188
|
Args:
|
188
189
|
tpm: Optional tokens per minute rate limit override
|
189
190
|
rpm: Optional requests per minute rate limit override
|
190
191
|
omit_system_prompt_if_empty_string: Whether to omit the system prompt when empty
|
191
192
|
key_lookup: Optional custom key lookup for API credentials
|
192
193
|
**kwargs: Additional parameters to pass to the model provider
|
193
|
-
|
194
|
+
|
194
195
|
The initialization process:
|
195
196
|
1. Sets up the model identifier from the class attribute
|
196
197
|
2. Configures model parameters by merging defaults with provided values
|
197
198
|
3. Sets up API key lookup and rate limits
|
198
199
|
4. Applies all parameters as instance attributes
|
199
|
-
|
200
|
+
|
200
201
|
For subclasses that define _parameters_ class attribute, these will be
|
201
202
|
used as default parameters that can be overridden by kwargs.
|
202
203
|
"""
|
203
204
|
# Get the model identifier from the class attribute
|
204
205
|
self.model = getattr(self, "_model_", None)
|
205
|
-
|
206
|
+
|
206
207
|
# Set up model parameters by combining defaults with provided values
|
207
208
|
default_parameters = getattr(self, "_parameters_", None)
|
208
209
|
parameters = self._overide_default_parameters(kwargs, default_parameters)
|
209
210
|
self.parameters = parameters
|
210
|
-
|
211
|
+
|
211
212
|
# Initialize basic settings
|
212
213
|
self.remote = False
|
213
214
|
self.omit_system_prompt_if_empty = omit_system_prompt_if_empty_string
|
@@ -237,15 +238,19 @@ class LanguageModel(
|
|
237
238
|
# Skip the API key check. Sometimes this is useful for testing.
|
238
239
|
self._api_token = None
|
239
240
|
|
241
|
+
# Add canned response to parameters
|
242
|
+
if "canned_response" in kwargs:
|
243
|
+
self.parameters["canned_response"] = kwargs["canned_response"]
|
244
|
+
|
240
245
|
def _set_key_lookup(self, key_lookup: "KeyLookup") -> "KeyLookup":
|
241
246
|
"""Set up the API key lookup mechanism.
|
242
|
-
|
247
|
+
|
243
248
|
This method either uses the provided key lookup object or creates a default
|
244
249
|
one that looks for API keys in config files and environment variables.
|
245
|
-
|
250
|
+
|
246
251
|
Args:
|
247
252
|
key_lookup: Optional custom key lookup object
|
248
|
-
|
253
|
+
|
249
254
|
Returns:
|
250
255
|
KeyLookup: The key lookup object to use for API credentials
|
251
256
|
"""
|
@@ -258,10 +263,10 @@ class LanguageModel(
|
|
258
263
|
|
259
264
|
def set_key_lookup(self, key_lookup: "KeyLookup") -> None:
|
260
265
|
"""Update the key lookup mechanism after initialization.
|
261
|
-
|
266
|
+
|
262
267
|
This method allows changing the API key lookup after the model has been
|
263
268
|
created, clearing any cached API tokens.
|
264
|
-
|
269
|
+
|
265
270
|
Args:
|
266
271
|
key_lookup: The new key lookup object to use
|
267
272
|
"""
|
@@ -271,13 +276,13 @@ class LanguageModel(
|
|
271
276
|
|
272
277
|
def ask_question(self, question: "QuestionBase") -> str:
|
273
278
|
"""Ask a question using this language model and return the response.
|
274
|
-
|
279
|
+
|
275
280
|
This is a convenience method that extracts the necessary prompts from a
|
276
281
|
question object and makes a model call.
|
277
|
-
|
282
|
+
|
278
283
|
Args:
|
279
284
|
question: The EDSL question object to ask
|
280
|
-
|
285
|
+
|
281
286
|
Returns:
|
282
287
|
str: The model's response to the question
|
283
288
|
"""
|
@@ -288,10 +293,10 @@ class LanguageModel(
|
|
288
293
|
@property
|
289
294
|
def rpm(self):
|
290
295
|
"""Get the requests per minute rate limit for this model.
|
291
|
-
|
296
|
+
|
292
297
|
This property provides the rate limit either from an explicitly set value,
|
293
298
|
from the model info in the key lookup, or from the default value.
|
294
|
-
|
299
|
+
|
295
300
|
Returns:
|
296
301
|
float: The requests per minute rate limit
|
297
302
|
"""
|
@@ -305,10 +310,10 @@ class LanguageModel(
|
|
305
310
|
@property
|
306
311
|
def tpm(self):
|
307
312
|
"""Get the tokens per minute rate limit for this model.
|
308
|
-
|
313
|
+
|
309
314
|
This property provides the rate limit either from an explicitly set value,
|
310
315
|
from the model info in the key lookup, or from the default value.
|
311
|
-
|
316
|
+
|
312
317
|
Returns:
|
313
318
|
float: The tokens per minute rate limit
|
314
319
|
"""
|
@@ -323,7 +328,7 @@ class LanguageModel(
|
|
323
328
|
@tpm.setter
|
324
329
|
def tpm(self, value):
|
325
330
|
"""Set the tokens per minute rate limit.
|
326
|
-
|
331
|
+
|
327
332
|
Args:
|
328
333
|
value: The new tokens per minute limit
|
329
334
|
"""
|
@@ -332,7 +337,7 @@ class LanguageModel(
|
|
332
337
|
@rpm.setter
|
333
338
|
def rpm(self, value):
|
334
339
|
"""Set the requests per minute rate limit.
|
335
|
-
|
340
|
+
|
336
341
|
Args:
|
337
342
|
value: The new requests per minute limit
|
338
343
|
"""
|
@@ -341,13 +346,13 @@ class LanguageModel(
|
|
341
346
|
@property
|
342
347
|
def api_token(self) -> str:
|
343
348
|
"""Get the API token for this model's service.
|
344
|
-
|
349
|
+
|
345
350
|
This property lazily fetches the API token from the key lookup
|
346
351
|
mechanism when first accessed, caching it for subsequent uses.
|
347
|
-
|
352
|
+
|
348
353
|
Returns:
|
349
354
|
str: The API token for authenticating with the model provider
|
350
|
-
|
355
|
+
|
351
356
|
Raises:
|
352
357
|
ValueError: If no API key is found for this model's service
|
353
358
|
"""
|
@@ -362,10 +367,10 @@ class LanguageModel(
|
|
362
367
|
|
363
368
|
def __getitem__(self, key):
|
364
369
|
"""Allow dictionary-style access to model attributes.
|
365
|
-
|
370
|
+
|
366
371
|
Args:
|
367
372
|
key: The attribute name to access
|
368
|
-
|
373
|
+
|
369
374
|
Returns:
|
370
375
|
The value of the specified attribute
|
371
376
|
"""
|
@@ -373,13 +378,13 @@ class LanguageModel(
|
|
373
378
|
|
374
379
|
def hello(self, verbose=False):
|
375
380
|
"""Run a simple test to verify the model connection is working.
|
376
|
-
|
381
|
+
|
377
382
|
This method makes a basic model call to check if the API credentials
|
378
383
|
are valid and the model is responsive.
|
379
|
-
|
384
|
+
|
380
385
|
Args:
|
381
386
|
verbose: If True, prints the masked API token
|
382
|
-
|
387
|
+
|
383
388
|
Returns:
|
384
389
|
str: The model's response to a simple greeting
|
385
390
|
"""
|
@@ -393,14 +398,14 @@ class LanguageModel(
|
|
393
398
|
|
394
399
|
def has_valid_api_key(self) -> bool:
|
395
400
|
"""Check if the model has a valid API key available.
|
396
|
-
|
401
|
+
|
397
402
|
This method verifies if the necessary API key is available in
|
398
403
|
environment variables or configuration for this model's service.
|
399
404
|
Test models always return True.
|
400
|
-
|
405
|
+
|
401
406
|
Returns:
|
402
407
|
bool: True if a valid API key is available, False otherwise
|
403
|
-
|
408
|
+
|
404
409
|
Examples:
|
405
410
|
>>> LanguageModel.example().has_valid_api_key() : # doctest: +SKIP
|
406
411
|
True
|
@@ -416,14 +421,14 @@ class LanguageModel(
|
|
416
421
|
|
417
422
|
def __hash__(self) -> int:
|
418
423
|
"""Generate a hash value based on model identity and parameters.
|
419
|
-
|
424
|
+
|
420
425
|
This method allows language model instances to be used as dictionary
|
421
426
|
keys or in sets by providing a stable hash value based on the
|
422
427
|
model's essential characteristics.
|
423
|
-
|
428
|
+
|
424
429
|
Returns:
|
425
430
|
int: A hash value for the model instance
|
426
|
-
|
431
|
+
|
427
432
|
Examples:
|
428
433
|
>>> m = LanguageModel.example()
|
429
434
|
>>> hash(m) # Actual value may vary
|
@@ -433,17 +438,17 @@ class LanguageModel(
|
|
433
438
|
|
434
439
|
def __eq__(self, other) -> bool:
|
435
440
|
"""Check if two language model instances are functionally equivalent.
|
436
|
-
|
441
|
+
|
437
442
|
Two models are considered equal if they have the same model identifier
|
438
443
|
and the same parameter settings, meaning they would produce the same
|
439
444
|
outputs given the same inputs.
|
440
|
-
|
445
|
+
|
441
446
|
Args:
|
442
447
|
other: Another model to compare with
|
443
|
-
|
448
|
+
|
444
449
|
Returns:
|
445
450
|
bool: True if the models are functionally equivalent
|
446
|
-
|
451
|
+
|
447
452
|
Examples:
|
448
453
|
>>> m1 = LanguageModel.example()
|
449
454
|
>>> m2 = LanguageModel.example()
|
@@ -455,33 +460,33 @@ class LanguageModel(
|
|
455
460
|
@staticmethod
|
456
461
|
def _overide_default_parameters(passed_parameter_dict, default_parameter_dict):
|
457
462
|
"""Merge default parameters with user-specified parameters.
|
458
|
-
|
463
|
+
|
459
464
|
This method creates a parameter dictionary where explicitly passed
|
460
465
|
parameters take precedence over default values, while ensuring all
|
461
466
|
required parameters have a value.
|
462
|
-
|
467
|
+
|
463
468
|
Args:
|
464
469
|
passed_parameter_dict: Dictionary of user-specified parameters
|
465
470
|
default_parameter_dict: Dictionary of default parameter values
|
466
|
-
|
471
|
+
|
467
472
|
Returns:
|
468
473
|
dict: Combined parameter dictionary with defaults and overrides
|
469
|
-
|
474
|
+
|
470
475
|
Examples:
|
471
476
|
>>> LanguageModel._overide_default_parameters(
|
472
|
-
... passed_parameter_dict={"temperature": 0.5},
|
477
|
+
... passed_parameter_dict={"temperature": 0.5},
|
473
478
|
... default_parameter_dict={"temperature": 0.9})
|
474
479
|
{'temperature': 0.5}
|
475
|
-
|
480
|
+
|
476
481
|
>>> LanguageModel._overide_default_parameters(
|
477
|
-
... passed_parameter_dict={"temperature": 0.5},
|
482
|
+
... passed_parameter_dict={"temperature": 0.5},
|
478
483
|
... default_parameter_dict={"temperature": 0.9, "max_tokens": 1000})
|
479
484
|
{'temperature': 0.5, 'max_tokens': 1000}
|
480
485
|
"""
|
481
486
|
# Handle the case when data is loaded from a dict after serialization
|
482
487
|
if "parameters" in passed_parameter_dict:
|
483
488
|
passed_parameter_dict = passed_parameter_dict["parameters"]
|
484
|
-
|
489
|
+
|
485
490
|
# Create new dict with defaults, overridden by passed parameters
|
486
491
|
return {
|
487
492
|
parameter_name: passed_parameter_dict.get(parameter_name, default_value)
|
@@ -490,14 +495,14 @@ class LanguageModel(
|
|
490
495
|
|
491
496
|
def __call__(self, user_prompt: str, system_prompt: str):
|
492
497
|
"""Allow the model to be called directly as a function.
|
493
|
-
|
498
|
+
|
494
499
|
This method provides a convenient way to use the model by calling
|
495
500
|
it directly with prompts, like `response = model(user_prompt, system_prompt)`.
|
496
|
-
|
501
|
+
|
497
502
|
Args:
|
498
503
|
user_prompt: The user message or input prompt
|
499
504
|
system_prompt: The system message or context
|
500
|
-
|
505
|
+
|
501
506
|
Returns:
|
502
507
|
The response from the model
|
503
508
|
"""
|
@@ -506,17 +511,17 @@ class LanguageModel(
|
|
506
511
|
@abstractmethod
|
507
512
|
async def async_execute_model_call(self, user_prompt: str, system_prompt: str):
|
508
513
|
"""Execute the model call asynchronously.
|
509
|
-
|
514
|
+
|
510
515
|
This abstract method must be implemented by all model subclasses
|
511
516
|
to handle the actual API call to the language model provider.
|
512
|
-
|
517
|
+
|
513
518
|
Args:
|
514
519
|
user_prompt: The user message or input prompt
|
515
520
|
system_prompt: The system message or context
|
516
|
-
|
521
|
+
|
517
522
|
Returns:
|
518
523
|
Coroutine that resolves to the model response
|
519
|
-
|
524
|
+
|
520
525
|
Note:
|
521
526
|
Implementations should handle the actual API communication,
|
522
527
|
including authentication, request formatting, and response parsing.
|
@@ -527,15 +532,15 @@ class LanguageModel(
|
|
527
532
|
self, user_prompt: str, system_prompt: str
|
528
533
|
):
|
529
534
|
"""Execute the model call remotely through the EDSL Coop service.
|
530
|
-
|
535
|
+
|
531
536
|
This method allows offloading the model call to a remote server,
|
532
537
|
which can be useful for models not available in the local environment
|
533
538
|
or to avoid rate limits.
|
534
|
-
|
539
|
+
|
535
540
|
Args:
|
536
541
|
user_prompt: The user message or input prompt
|
537
542
|
system_prompt: The system message or context
|
538
|
-
|
543
|
+
|
539
544
|
Returns:
|
540
545
|
Coroutine that resolves to the model response from the remote service
|
541
546
|
"""
|
@@ -550,18 +555,19 @@ class LanguageModel(
|
|
550
555
|
@jupyter_nb_handler
|
551
556
|
def execute_model_call(self, *args, **kwargs):
|
552
557
|
"""Execute a model call synchronously.
|
553
|
-
|
558
|
+
|
554
559
|
This method is a synchronous wrapper around the asynchronous execution,
|
555
560
|
making it easier to use the model in non-async contexts. It's decorated
|
556
561
|
with jupyter_nb_handler to ensure proper handling in notebook environments.
|
557
|
-
|
562
|
+
|
558
563
|
Args:
|
559
564
|
*args: Positional arguments to pass to async_execute_model_call
|
560
565
|
**kwargs: Keyword arguments to pass to async_execute_model_call
|
561
|
-
|
566
|
+
|
562
567
|
Returns:
|
563
568
|
The model response
|
564
569
|
"""
|
570
|
+
|
565
571
|
async def main():
|
566
572
|
results = await asyncio.gather(
|
567
573
|
self.async_execute_model_call(*args, **kwargs)
|
@@ -573,16 +579,16 @@ class LanguageModel(
|
|
573
579
|
@classmethod
|
574
580
|
def get_generated_token_string(cls, raw_response: dict[str, Any]) -> str:
|
575
581
|
"""Extract the generated text from a raw model response.
|
576
|
-
|
582
|
+
|
577
583
|
This method navigates the response structure using the model's key_sequence
|
578
584
|
to find and return just the generated text, without metadata.
|
579
|
-
|
585
|
+
|
580
586
|
Args:
|
581
587
|
raw_response: The complete response dictionary from the model API
|
582
|
-
|
588
|
+
|
583
589
|
Returns:
|
584
590
|
str: The generated text string
|
585
|
-
|
591
|
+
|
586
592
|
Examples:
|
587
593
|
>>> m = LanguageModel.example(test_model=True)
|
588
594
|
>>> raw_response = m.execute_model_call("Hello, model!", "You are a helpful agent.")
|
@@ -594,14 +600,14 @@ class LanguageModel(
|
|
594
600
|
@classmethod
|
595
601
|
def get_usage_dict(cls, raw_response: dict[str, Any]) -> dict[str, Any]:
|
596
602
|
"""Extract token usage statistics from a raw model response.
|
597
|
-
|
603
|
+
|
598
604
|
This method navigates the response structure to find and return
|
599
605
|
information about token usage, which is used for cost calculation
|
600
606
|
and monitoring.
|
601
|
-
|
607
|
+
|
602
608
|
Args:
|
603
609
|
raw_response: The complete response dictionary from the model API
|
604
|
-
|
610
|
+
|
605
611
|
Returns:
|
606
612
|
dict: Dictionary of token usage statistics (input tokens, output tokens, etc.)
|
607
613
|
"""
|
@@ -610,14 +616,14 @@ class LanguageModel(
|
|
610
616
|
@classmethod
|
611
617
|
def parse_response(cls, raw_response: dict[str, Any]) -> EDSLOutput:
|
612
618
|
"""Parse the raw API response into a standardized EDSL output format.
|
613
|
-
|
619
|
+
|
614
620
|
This method processes the model's response to extract the generated content
|
615
621
|
and format it according to EDSL's expected structure, making it consistent
|
616
622
|
across different model providers.
|
617
|
-
|
623
|
+
|
618
624
|
Args:
|
619
625
|
raw_response: The complete response dictionary from the model API
|
620
|
-
|
626
|
+
|
621
627
|
Returns:
|
622
628
|
EDSLOutput: Standardized output structure with answer and optional comment
|
623
629
|
"""
|
@@ -633,7 +639,7 @@ class LanguageModel(
|
|
633
639
|
invigilator=None,
|
634
640
|
) -> ModelResponse:
|
635
641
|
"""Handle model calls with caching for efficiency.
|
636
|
-
|
642
|
+
|
637
643
|
This method implements the caching logic for model calls, checking if a
|
638
644
|
response is already cached before making an actual API call. It handles
|
639
645
|
the complete workflow of:
|
@@ -642,7 +648,7 @@ class LanguageModel(
|
|
642
648
|
3. Making the API call if needed
|
643
649
|
4. Storing new responses in the cache
|
644
650
|
5. Adding metadata like cost and cache status
|
645
|
-
|
651
|
+
|
646
652
|
Args:
|
647
653
|
user_prompt: The user's message or input prompt
|
648
654
|
system_prompt: The system's message or context
|
@@ -650,10 +656,10 @@ class LanguageModel(
|
|
650
656
|
iteration: The iteration number, used for the cache key
|
651
657
|
files_list: Optional list of files to include in the prompt
|
652
658
|
invigilator: Optional invigilator object, not used in caching
|
653
|
-
|
659
|
+
|
654
660
|
Returns:
|
655
661
|
ModelResponse: Response object with the model output and metadata
|
656
|
-
|
662
|
+
|
657
663
|
Examples:
|
658
664
|
>>> from edsl import Cache
|
659
665
|
>>> m = LanguageModel.example(test_model=True)
|
@@ -675,10 +681,9 @@ class LanguageModel(
|
|
675
681
|
"user_prompt": user_prompt_with_hashes,
|
676
682
|
"iteration": iteration,
|
677
683
|
}
|
678
|
-
|
684
|
+
|
679
685
|
# Try to fetch from cache
|
680
686
|
cached_response, cache_key = cache.fetch(**cache_call_params)
|
681
|
-
|
682
687
|
if cache_used := cached_response is not None:
|
683
688
|
# Cache hit - use the cached response
|
684
689
|
response = json.loads(cached_response)
|
@@ -690,21 +695,22 @@ class LanguageModel(
|
|
690
695
|
if hasattr(self, "remote") and self.remote
|
691
696
|
else self.async_execute_model_call
|
692
697
|
)
|
693
|
-
|
698
|
+
|
694
699
|
# Prepare parameters for the model call
|
695
700
|
params = {
|
696
701
|
"user_prompt": user_prompt,
|
697
702
|
"system_prompt": system_prompt,
|
698
703
|
"files_list": files_list,
|
699
704
|
}
|
700
|
-
|
705
|
+
|
701
706
|
# Get timeout from configuration
|
702
707
|
from ..config import CONFIG
|
708
|
+
|
703
709
|
TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
|
704
|
-
|
710
|
+
|
705
711
|
# Execute the model call with timeout
|
706
712
|
response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
|
707
|
-
|
713
|
+
|
708
714
|
# Store the response in the cache
|
709
715
|
new_cache_key = cache.store(
|
710
716
|
**cache_call_params, response=response, service=self._inference_service_
|
@@ -713,7 +719,6 @@ class LanguageModel(
|
|
713
719
|
|
714
720
|
# Calculate cost for the response
|
715
721
|
cost = self.cost(response)
|
716
|
-
|
717
722
|
# Return a structured response with metadata
|
718
723
|
return ModelResponse(
|
719
724
|
response=response,
|
@@ -734,15 +739,15 @@ class LanguageModel(
|
|
734
739
|
top_logprobs=2,
|
735
740
|
):
|
736
741
|
"""Ask a simple question with log probability tracking.
|
737
|
-
|
742
|
+
|
738
743
|
This is a convenience method for getting responses with log probabilities,
|
739
744
|
which can be useful for analyzing model confidence and alternatives.
|
740
|
-
|
745
|
+
|
741
746
|
Args:
|
742
747
|
question: The EDSL question object to ask
|
743
748
|
system_prompt: System message to use (default is human agent instruction)
|
744
749
|
top_logprobs: Number of top alternative tokens to return probabilities for
|
745
|
-
|
750
|
+
|
746
751
|
Returns:
|
747
752
|
The model response, including log probabilities if supported
|
748
753
|
"""
|
@@ -762,14 +767,14 @@ class LanguageModel(
|
|
762
767
|
**kwargs,
|
763
768
|
) -> AgentResponseDict:
|
764
769
|
"""Get a complete response with all metadata and parsed format.
|
765
|
-
|
770
|
+
|
766
771
|
This method handles the complete pipeline for:
|
767
772
|
1. Making a model call (with caching)
|
768
773
|
2. Parsing the response
|
769
774
|
3. Constructing a full response object with inputs, outputs, and parsed data
|
770
|
-
|
775
|
+
|
771
776
|
It's the primary method used by higher-level components to interact with models.
|
772
|
-
|
777
|
+
|
773
778
|
Args:
|
774
779
|
user_prompt: The user's message or input prompt
|
775
780
|
system_prompt: The system's message or context
|
@@ -777,7 +782,7 @@ class LanguageModel(
|
|
777
782
|
iteration: The iteration number (default: 1)
|
778
783
|
files_list: Optional list of files to include in the prompt
|
779
784
|
**kwargs: Additional parameters (invigilator can be provided here)
|
780
|
-
|
785
|
+
|
781
786
|
Returns:
|
782
787
|
AgentResponseDict: Complete response object with inputs, raw outputs, and parsed data
|
783
788
|
"""
|
@@ -789,19 +794,19 @@ class LanguageModel(
|
|
789
794
|
"cache": cache,
|
790
795
|
"files_list": files_list,
|
791
796
|
}
|
792
|
-
|
797
|
+
|
793
798
|
# Add invigilator if provided
|
794
799
|
if "invigilator" in kwargs:
|
795
800
|
params.update({"invigilator": kwargs["invigilator"]})
|
796
801
|
|
797
802
|
# Create structured input record
|
798
803
|
model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
|
799
|
-
|
804
|
+
|
800
805
|
# Get model response (using cache if available)
|
801
806
|
model_outputs: ModelResponse = (
|
802
807
|
await self._async_get_intended_model_call_outcome(**params)
|
803
808
|
)
|
804
|
-
|
809
|
+
|
805
810
|
# Parse the response into EDSL's standard format
|
806
811
|
edsl_dict: EDSLOutput = self.parse_response(model_outputs.response)
|
807
812
|
|
@@ -811,31 +816,31 @@ class LanguageModel(
|
|
811
816
|
model_outputs=model_outputs,
|
812
817
|
edsl_dict=edsl_dict,
|
813
818
|
)
|
814
|
-
|
815
819
|
return agent_response_dict
|
816
820
|
|
817
821
|
get_response = sync_wrapper(async_get_response)
|
818
822
|
|
819
823
|
def cost(self, raw_response: dict[str, Any]) -> Union[float, str]:
|
820
824
|
"""Calculate the monetary cost of a model API call.
|
821
|
-
|
825
|
+
|
822
826
|
This method extracts token usage information from the response and
|
823
827
|
uses the price manager to calculate the actual cost in dollars based
|
824
828
|
on the model's pricing structure and token counts.
|
825
|
-
|
829
|
+
|
826
830
|
Args:
|
827
831
|
raw_response: The complete response dictionary from the model API
|
828
|
-
|
832
|
+
|
829
833
|
Returns:
|
830
834
|
Union[float, str]: The calculated cost in dollars, or an error message
|
831
835
|
"""
|
832
836
|
# Extract token usage data from the response
|
833
837
|
usage = self.get_usage_dict(raw_response)
|
834
|
-
|
838
|
+
|
835
839
|
# Use the price manager to calculate the actual cost
|
836
840
|
from .price_manager import PriceManager
|
841
|
+
|
837
842
|
price_manager = PriceManager()
|
838
|
-
|
843
|
+
|
839
844
|
return price_manager.calculate_cost(
|
840
845
|
inference_service=self._inference_service_,
|
841
846
|
model=self.model,
|
@@ -846,17 +851,17 @@ class LanguageModel(
|
|
846
851
|
|
847
852
|
def to_dict(self, add_edsl_version: bool = True) -> dict[str, Any]:
|
848
853
|
"""Serialize the model instance to a dictionary representation.
|
849
|
-
|
854
|
+
|
850
855
|
This method creates a dictionary containing all the information needed
|
851
856
|
to recreate this model, including its identifier, parameters, and service.
|
852
857
|
Optionally includes EDSL version information for compatibility checking.
|
853
|
-
|
858
|
+
|
854
859
|
Args:
|
855
860
|
add_edsl_version: Whether to include EDSL version and class name (default: True)
|
856
|
-
|
861
|
+
|
857
862
|
Returns:
|
858
863
|
dict: Dictionary representation of this model instance
|
859
|
-
|
864
|
+
|
860
865
|
Examples:
|
861
866
|
>>> m = LanguageModel.example()
|
862
867
|
>>> m.to_dict()
|
@@ -868,30 +873,30 @@ class LanguageModel(
|
|
868
873
|
"parameters": self.parameters,
|
869
874
|
"inference_service": self._inference_service_,
|
870
875
|
}
|
871
|
-
|
876
|
+
|
872
877
|
# Add EDSL version and class information if requested
|
873
878
|
if add_edsl_version:
|
874
879
|
from edsl import __version__
|
875
880
|
|
876
881
|
d["edsl_version"] = __version__
|
877
882
|
d["edsl_class_name"] = self.__class__.__name__
|
878
|
-
|
883
|
+
|
879
884
|
return d
|
880
885
|
|
881
886
|
@classmethod
|
882
887
|
@remove_edsl_version
|
883
888
|
def from_dict(cls, data: dict) -> "LanguageModel":
|
884
889
|
"""Create a language model instance from a dictionary representation.
|
885
|
-
|
890
|
+
|
886
891
|
This class method deserializes a model from its dictionary representation,
|
887
892
|
finding the correct model class based on the model identifier and service.
|
888
|
-
|
893
|
+
|
889
894
|
Args:
|
890
895
|
data: Dictionary containing the model configuration
|
891
|
-
|
896
|
+
|
892
897
|
Returns:
|
893
898
|
LanguageModel: A new model instance of the appropriate type
|
894
|
-
|
899
|
+
|
895
900
|
Note:
|
896
901
|
This method does not use the stored inference_service directly but
|
897
902
|
fetches the model class based on the model name and service name.
|
@@ -902,24 +907,25 @@ class LanguageModel(
|
|
902
907
|
model_class = get_model_class(
|
903
908
|
data["model"], service_name=data.get("inference_service", None)
|
904
909
|
)
|
905
|
-
|
910
|
+
|
906
911
|
# Create and return a new instance
|
907
912
|
return model_class(**data)
|
908
913
|
|
909
914
|
def __repr__(self) -> str:
|
910
915
|
"""Generate a string representation of the model.
|
911
|
-
|
916
|
+
|
912
917
|
This representation includes the model identifier and all parameters,
|
913
918
|
providing a clear picture of how the model is configured.
|
914
|
-
|
919
|
+
|
915
920
|
Returns:
|
916
921
|
str: A string representation of the model
|
917
922
|
"""
|
918
923
|
# Format the parameters as a string
|
919
924
|
param_string = ", ".join(
|
920
|
-
f
|
925
|
+
f'{key} = """{value}"""' if key == "canned_response" else f"{key} = {value}"
|
926
|
+
for key, value in self.parameters.items()
|
921
927
|
)
|
922
|
-
|
928
|
+
|
923
929
|
# Combine model name and parameters
|
924
930
|
return (
|
925
931
|
f"Model(model_name = '{self.model}'"
|
@@ -929,16 +935,16 @@ class LanguageModel(
|
|
929
935
|
|
930
936
|
def __add__(self, other_model: "LanguageModel") -> "LanguageModel":
|
931
937
|
"""Define behavior when models are combined with the + operator.
|
932
|
-
|
933
|
-
This operator is used in survey builder contexts, but note that it
|
938
|
+
|
939
|
+
This operator is used in survey builder contexts, but note that it
|
934
940
|
replaces the left model with the right model rather than combining them.
|
935
|
-
|
941
|
+
|
936
942
|
Args:
|
937
943
|
other_model: Another model to combine with this one
|
938
|
-
|
944
|
+
|
939
945
|
Returns:
|
940
946
|
LanguageModel: The other model if provided, otherwise this model
|
941
|
-
|
947
|
+
|
942
948
|
Warning:
|
943
949
|
This doesn't truly combine models - it replaces one with the other.
|
944
950
|
For running multiple models, use a single 'by' call with multiple models.
|
@@ -957,37 +963,37 @@ class LanguageModel(
|
|
957
963
|
throw_exception: bool = False,
|
958
964
|
) -> "LanguageModel":
|
959
965
|
"""Create an example language model instance for testing and demonstration.
|
960
|
-
|
966
|
+
|
961
967
|
This method provides a convenient way to create a model instance for
|
962
968
|
examples, tests, and documentation. It can create either a real model
|
963
969
|
(with API key checking disabled) or a test model that returns predefined
|
964
970
|
responses.
|
965
|
-
|
971
|
+
|
966
972
|
Args:
|
967
973
|
test_model: If True, creates a test model that doesn't make real API calls
|
968
974
|
canned_response: For test models, the predefined response to return
|
969
975
|
throw_exception: For test models, whether to throw an exception instead of responding
|
970
|
-
|
976
|
+
|
971
977
|
Returns:
|
972
978
|
LanguageModel: An example model instance
|
973
|
-
|
979
|
+
|
974
980
|
Examples:
|
975
981
|
Create a test model with a custom response:
|
976
|
-
|
982
|
+
|
977
983
|
>>> from edsl.language_models import LanguageModel
|
978
984
|
>>> m = LanguageModel.example(test_model=True, canned_response="WOWZA!")
|
979
985
|
>>> isinstance(m, LanguageModel)
|
980
986
|
True
|
981
|
-
|
987
|
+
|
982
988
|
Use the test model to answer a question:
|
983
|
-
|
989
|
+
|
984
990
|
>>> from edsl import QuestionFreeText
|
985
991
|
>>> q = QuestionFreeText(question_text="What is your name?", question_name='example')
|
986
992
|
>>> q.by(m).run(cache=False, disable_remote_cache=True, disable_remote_inference=True).select('example').first()
|
987
993
|
'WOWZA!'
|
988
|
-
|
994
|
+
|
989
995
|
Create a test model that throws exceptions:
|
990
|
-
|
996
|
+
|
991
997
|
>>> m = LanguageModel.example(test_model=True, canned_response="WOWZA!", throw_exception=True)
|
992
998
|
>>> r = q.by(m).run(cache=False, disable_remote_cache=True, disable_remote_inference=True, print_exceptions=True)
|
993
999
|
Exception report saved to ...
|
@@ -1006,14 +1012,14 @@ class LanguageModel(
|
|
1006
1012
|
|
1007
1013
|
def from_cache(self, cache: "Cache") -> "LanguageModel":
|
1008
1014
|
"""Create a new model that only returns responses from the cache.
|
1009
|
-
|
1015
|
+
|
1010
1016
|
This method creates a modified copy of the model that will only use
|
1011
1017
|
cached responses, never making new API calls. This is useful for
|
1012
1018
|
offline operation or repeatable experiments.
|
1013
|
-
|
1019
|
+
|
1014
1020
|
Args:
|
1015
1021
|
cache: The cache object containing previously cached responses
|
1016
|
-
|
1022
|
+
|
1017
1023
|
Returns:
|
1018
1024
|
LanguageModel: A new model instance that only reads from cache
|
1019
1025
|
"""
|
@@ -1024,7 +1030,7 @@ class LanguageModel(
|
|
1024
1030
|
# Create a deep copy of this model instance
|
1025
1031
|
new_instance = deepcopy(self)
|
1026
1032
|
print("Cache entries", len(cache))
|
1027
|
-
|
1033
|
+
|
1028
1034
|
# Filter the cache to only include entries for this model
|
1029
1035
|
new_instance.cache = Cache(
|
1030
1036
|
data={k: v for k, v in cache.items() if v.model == self.model}
|