edsl 0.1.36.dev5__py3-none-any.whl → 0.1.37__py3-none-any.whl

edsl/jobs/interviews/InterviewExceptionEntry.py

@@ -67,7 +67,11 @@ class InterviewExceptionEntry:
         m = LanguageModel.example(test_model=True)
         q = QuestionFreeText.example(exception_to_throw=ValueError)
         results = q.by(m).run(
-            skip_retry=True, print_exceptions=False, raise_validation_errors=True
+            skip_retry=True,
+            print_exceptions=False,
+            raise_validation_errors=True,
+            disable_remote_cache=True,
+            disable_remote_inference=True,
         )
         return results.task_history.exceptions[0]["how_are_you"][0]
 
@@ -132,18 +136,25 @@ class InterviewExceptionEntry:
         )
         console.print(tb)
         return html_output.getvalue()
-    
+
     @staticmethod
     def serialize_exception(exception: Exception) -> dict:
         return {
             "type": type(exception).__name__,
             "message": str(exception),
-            "traceback": "".join(traceback.format_exception(type(exception), exception, exception.__traceback__)),
+            "traceback": "".join(
+                traceback.format_exception(
+                    type(exception), exception, exception.__traceback__
+                )
+            ),
         }
-    
+
     @staticmethod
     def deserialize_exception(data: dict) -> Exception:
-        exception_class = globals()[data["type"]]
+        try:
+            exception_class = globals()[data["type"]]
+        except KeyError:
+            exception_class = Exception
         return exception_class(data["message"])
 
     def to_dict(self) -> dict:
@@ -158,7 +169,7 @@ class InterviewExceptionEntry:
             "traceback": self.traceback,
             "invigilator": self.invigilator.to_dict(),
         }
-    
+
     @classmethod
     def from_dict(cls, data: dict) -> "InterviewExceptionEntry":
         """Create an InterviewExceptionEntry from a dictionary."""
@@ -168,13 +179,6 @@ class InterviewExceptionEntry:
         invigilator = InvigilatorAI.from_dict(data["invigilator"])
         return cls(exception=exception, invigilator=invigilator)
 
-    def push(self):
-        from edsl import Coop
-
-        coop = Coop()
-        results = coop.error_create(self.to_dict())
-        return results
-
 
 if __name__ == "__main__":
     import doctest

edsl/jobs/interviews/ReportErrors.py

@@ -36,8 +36,8 @@ class ReportErrors:
             print("No input received within the timeout period.")
 
     def upload(self):
-        coop = Coop()
-        coop.error_create(error_data=self.data)
+        # The previous implementation was removed because it relied on the old Coop ErrorModel
+        pass
 
 
 def main():

edsl/jobs/runners/JobsRunnerAsyncio.py

@@ -19,6 +19,7 @@ from edsl.results.Results import Results
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.data.Cache import Cache
 
+
 class StatusTracker(UserList):
     def __init__(self, total_tasks: int):
         self.total_tasks = total_tasks
@@ -164,20 +165,20 @@ class JobsRunnerAsyncio:
 
         prompt_dictionary = {}
         for answer_key_name in answer_key_names:
-            prompt_dictionary[answer_key_name + "_user_prompt"] = (
-                question_name_to_prompts[answer_key_name]["user_prompt"]
-            )
-            prompt_dictionary[answer_key_name + "_system_prompt"] = (
-                question_name_to_prompts[answer_key_name]["system_prompt"]
-            )
+            prompt_dictionary[
+                answer_key_name + "_user_prompt"
+            ] = question_name_to_prompts[answer_key_name]["user_prompt"]
+            prompt_dictionary[
+                answer_key_name + "_system_prompt"
+            ] = question_name_to_prompts[answer_key_name]["system_prompt"]
 
         raw_model_results_dictionary = {}
         cache_used_dictionary = {}
         for result in valid_results:
             question_name = result.question_name
-            raw_model_results_dictionary[question_name + "_raw_model_response"] = (
-                result.raw_model_response
-            )
+            raw_model_results_dictionary[
+                question_name + "_raw_model_response"
+            ] = result.raw_model_response
             raw_model_results_dictionary[question_name + "_cost"] = result.cost
             one_use_buys = (
                 "NA"

edsl/jobs/tasks/TaskHistory.py

@@ -39,6 +39,7 @@ class TaskHistory:
             skip_retry=True,
             cache=False,
             raise_validation_errors=True,
+            disable_remote_cache=True,
             disable_remote_inference=True,
         )
 

edsl/language_models/KeyLookup.py

@@ -0,0 +1,30 @@
+import os
+from collections import UserDict
+
+from edsl.enums import service_to_api_keyname
+from edsl.exceptions import MissingAPIKeyError
+
+
+class KeyLookup(UserDict):
+    @classmethod
+    def from_os_environ(cls):
+        """Create an instance of KeyLookupAPI with keys from os.environ"""
+        return cls({key: value for key, value in os.environ.items()})
+
+    def get_api_token(self, service: str, remote: bool = False):
+        key_name = service_to_api_keyname.get(service, "NOT FOUND")
+
+        if service == "bedrock":
+            api_token = [self.get(key_name[0]), self.get(key_name[1])]
+            missing_token = any(token is None for token in api_token)
+        else:
+            api_token = self.get(key_name)
+            missing_token = api_token is None
+
+        if missing_token and service != "test" and not remote:
+            raise MissingAPIKeyError(
+                f"""The key for service: `{service}` is not set.
+                    Need a key with name {key_name} in your .env file."""
+            )
+
+        return api_token

edsl/language_models/LanguageModel.py

@@ -17,9 +17,7 @@ import warnings
 from functools import wraps
 import asyncio
 import json
-import time
 import os
-import hashlib
 from typing import (
     Coroutine,
     Any,
@@ -30,6 +28,7 @@ from typing import (
     get_type_hints,
     TypedDict,
     Optional,
+    TYPE_CHECKING,
 )
 from abc import ABC, abstractmethod
 
@@ -49,34 +48,16 @@ from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 from edsl.language_models.repair import repair
 from edsl.enums import InferenceServiceType
 from edsl.Base import RichPrintingMixin, PersistenceMixin
-from edsl.enums import service_to_api_keyname
-from edsl.exceptions import MissingAPIKeyError
 from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
 from edsl.exceptions.language_models import LanguageModelBadResponseError
 
-TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
-
-
-def convert_answer(response_part):
-    import json
-
-    response_part = response_part.strip()
-
-    if response_part == "None":
-        return None
-
-    repaired = repair_json(response_part)
-    if repaired == '""':
-        # it was a literal string
-        return response_part
+from edsl.language_models.KeyLookup import KeyLookup
 
-    try:
-        return json.loads(repaired)
-    except json.JSONDecodeError as j:
-        # last resort
-        return response_part
+TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
+# you might be tempated to move this to be a static method of LanguageModel, but this doesn't work
+# for reasons I don't understand. So leave it here.
 def extract_item_from_raw_response(data, key_sequence):
     if isinstance(data, str):
         try:
@@ -167,7 +148,12 @@ class LanguageModel(
     _safety_factor = 0.8
 
     def __init__(
-        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
+        self,
+        tpm: float = None,
+        rpm: float = None,
+        omit_system_prompt_if_empty_string: bool = True,
+        key_lookup: Optional[KeyLookup] = None,
+        **kwargs,
     ):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
@@ -200,29 +186,26 @@ class LanguageModel(
             # Skip the API key check. Sometimes this is useful for testing.
             self._api_token = None
 
+        if key_lookup is not None:
+            self.key_lookup = key_lookup
+        else:
+            self.key_lookup = KeyLookup.from_os_environ()
+
     def ask_question(self, question):
         user_prompt = question.get_instructions().render(question.data).text
         system_prompt = "You are a helpful agent pretending to be a human."
         return self.execute_model_call(user_prompt, system_prompt)
 
+    def set_key_lookup(self, key_lookup: KeyLookup):
+        del self._api_token
+        self.key_lookup = key_lookup
+
     @property
     def api_token(self) -> str:
         if not hasattr(self, "_api_token"):
-            key_name = service_to_api_keyname.get(self._inference_service_, "NOT FOUND")
-            if self._inference_service_ == "bedrock":
-                self._api_token = [os.getenv(key_name[0]), os.getenv(key_name[1])]
-                # Check if any of the tokens are None
-                missing_token = any(token is None for token in self._api_token)
-            else:
-                self._api_token = os.getenv(key_name)
-                missing_token = self._api_token is None
-            if missing_token and self._inference_service_ != "test" and not self.remote:
-                print("raising error")
-                raise MissingAPIKeyError(
-                    f"""The key for service: `{self._inference_service_}` is not set.
-                        Need a key with name {key_name} in your .env file."""
-                )
-
+            self._api_token = self.key_lookup.get_api_token(
+                self._inference_service_, self.remote
+            )
         return self._api_token
 
     def __getitem__(self, key):
@@ -291,21 +274,6 @@ class LanguageModel(
         if tpm is not None:
             self._tpm = tpm
         return None
-        # self._set_rate_limits(rpm=rpm, tpm=tpm)
-
-    # def _set_rate_limits(self, rpm=None, tpm=None) -> None:
-    #     """Set the rate limits for the model.
-
-    #     If the model does not have rate limits, use the default rate limits."""
-    #     if rpm is not None and tpm is not None:
-    #         self.__rate_limits = {"rpm": rpm, "tpm": tpm}
-    #         return
-
-    #     if self.__rate_limits is None:
-    #         if hasattr(self, "get_rate_limits"):
-    #             self.__rate_limits = self.get_rate_limits()
-    #         else:
-    #             self.__rate_limits = self.__default_rate_limits
 
     @property
     def RPM(self):
@@ -416,6 +384,26 @@ class LanguageModel(
             )
         return extract_item_from_raw_response(raw_response, cls.usage_sequence)
 
+    @staticmethod
+    def convert_answer(response_part):
+        import json
+
+        response_part = response_part.strip()
+
+        if response_part == "None":
+            return None
+
+        repaired = repair_json(response_part)
+        if repaired == '""':
+            # it was a literal string
+            return response_part
+
+        try:
+            return json.loads(repaired)
+        except json.JSONDecodeError as j:
+            # last resort
+            return response_part
+
     @classmethod
     def parse_response(cls, raw_response: dict[str, Any]) -> EDSLOutput:
         """Parses the API response and returns the response text."""
@@ -425,13 +413,13 @@ class LanguageModel(
         if last_newline == -1:
             # There is no comment
             edsl_dict = {
-                "answer": convert_answer(generated_token_string),
+                "answer": cls.convert_answer(generated_token_string),
                 "generated_tokens": generated_token_string,
                 "comment": None,
             }
         else:
             edsl_dict = {
-                "answer": convert_answer(generated_token_string[:last_newline]),
+                "answer": cls.convert_answer(generated_token_string[:last_newline]),
                 "comment": generated_token_string[last_newline + 1 :].strip(),
                 "generated_tokens": generated_token_string,
             }
@@ -492,7 +480,7 @@ class LanguageModel(
             params = {
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
-                "files_list": files_list
+                "files_list": files_list,
                 # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
@@ -699,7 +687,7 @@ class LanguageModel(
         True
         >>> from edsl import QuestionFreeText
         >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
-        >>> q.by(m).run(cache = False).select('example').first()
+        >>> q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True).select('example').first()
         'WOWZA!'
         """
         from edsl import Model

edsl/language_models/__init__.py

@@ -1,2 +1,3 @@
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.language_models.registry import Model
+from edsl.language_models.KeyLookup import KeyLookup

edsl/prompts/Prompt.py

@@ -17,14 +17,6 @@ class PreserveUndefined(Undefined):
 
 
 from edsl.exceptions.prompts import TemplateRenderError
-
-# from edsl.prompts.prompt_config import (
-#     C2A,
-#     names_to_component_types,
-#     ComponentTypes,
-#     NEGATIVE_INFINITY,
-# )
-# from edsl.prompts.registry import RegisterPromptsMeta
 from edsl.Base import PersistenceMixin, RichPrintingMixin
 
 MAX_NESTING = 100
@@ -60,6 +52,9 @@ class Prompt(PersistenceMixin, RichPrintingMixin):
                 text = self.default_instructions
             else:
                 text = ""
+        if isinstance(text, Prompt):
+            # make it idempotent w/ a prompt
+            text = text.text
         self._text = text
 
     @classmethod
@@ -245,10 +240,14 @@ class Prompt(PersistenceMixin, RichPrintingMixin):
         >>> p.render({"person": "Mr. {{last_name}}"})
         Prompt(text=\"""Hello, Mr. {{ last_name }}\""")
         """
-        new_text = self._render(
-            self.text, primary_replacement, **additional_replacements
-        )
-        return self.__class__(text=new_text)
+        try:
+            new_text = self._render(
+                self.text, primary_replacement, **additional_replacements
+            )
+            return self.__class__(text=new_text)
+        except Exception as e:
+            print(f"Error rendering prompt: {e}")
+            return self
 
     @staticmethod
     def _render(

edsl/questions/QuestionBase.py

@@ -150,14 +150,21 @@ class QuestionBase(
             "_include_comment",
             "_fake_data_factory",
             "_use_code",
-            "_answering_instructions",
-            "_question_presentation",
             "_model_instructions",
         ]
+        only_if_not_na_list = ["_answering_instructions", "_question_presentation"]
+
+        def ok(key, value):
+            if not key.startswith("_"):
+                return False
+            if key in exclude_list:
+                return False
+            if key in only_if_not_na_list and value is None:
+                return False
+            return True
+
         candidate_data = {
-            k.replace("_", "", 1): v
-            for k, v in self.__dict__.items()
-            if k.startswith("_") and k not in exclude_list
+            k.replace("_", "", 1): v for k, v in self.__dict__.items() if ok(k, v)
         }
 
         if "func" in candidate_data:
@@ -176,7 +183,9 @@ class QuestionBase(
         """
         candidate_data = self.data.copy()
         candidate_data["question_type"] = self.question_type
-        return candidate_data
+        return {
+            key: value for key, value in candidate_data.items() if value is not None
+        }
 
     @add_edsl_version
     def to_dict(self) -> dict[str, Any]:
@@ -239,6 +248,8 @@ class QuestionBase(
         show_answer: bool = True,
         model: Optional["LanguageModel"] = None,
         cache=False,
+        disable_remote_cache: bool = False,
+        disable_remote_inference: bool = False,
         **kwargs,
     ):
         """Run an example of the question.
@@ -247,7 +258,7 @@ class QuestionBase(
         >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
         >>> m.execute_model_call("", "")
         {'message': [{'text': "Yo, what's up?"}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
-        >>> Q.run_example(show_answer = True, model = m)
+        >>> Q.run_example(show_answer = True, model = m, disable_remote_cache = True, disable_remote_inference = True)
         ┏━━━━━━━━━━━━━━━━┓
         ┃ answer         ┃
         ┃ .how_are_you   ┃
@@ -259,25 +270,48 @@ class QuestionBase(
             from edsl import Model
 
             model = Model()
-        results = cls.example(**kwargs).by(model).run(cache=cache)
+        results = (
+            cls.example(**kwargs)
+            .by(model)
+            .run(
+                cache=cache,
+                disable_remote_cache=disable_remote_cache,
+                disable_remote_inference=disable_remote_inference,
+            )
+        )
         if show_answer:
             results.select("answer.*").print()
         else:
             return results
 
-    def __call__(self, just_answer=True, model=None, agent=None, **kwargs):
+    def __call__(
+        self,
+        just_answer=True,
+        model=None,
+        agent=None,
+        disable_remote_cache: bool = False,
+        disable_remote_inference: bool = False,
+        **kwargs,
+    ):
         """Call the question.
 
 
         >>> from edsl import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> q(model = m)
+        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True)
         "Yo, what's up?"
 
         """
         survey = self.to_survey()
-        results = survey(model=model, agent=agent, **kwargs, cache=False)
+        results = survey(
+            model=model,
+            agent=agent,
+            **kwargs,
+            cache=False,
+            disable_remote_cache=disable_remote_cache,
+            disable_remote_inference=disable_remote_inference,
+        )
         if just_answer:
             return results.select(f"answer.{self.question_name}").first()
         else:
@@ -295,6 +329,7 @@ class QuestionBase(
         just_answer: bool = True,
         model: Optional["Model"] = None,
         agent: Optional["Agent"] = None,
+        disable_remote_inference: bool = False,
         **kwargs,
     ) -> Union[Any, "Results"]:
         """Call the question asynchronously.
@@ -303,12 +338,17 @@ class QuestionBase(
         >>> from edsl import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Blue")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> async def test_run_async(): result = await q.run_async(model=m); print(result)
+        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True); print(result)
         >>> asyncio.run(test_run_async())
         Blue
         """
         survey = self.to_survey()
-        results = await survey.run_async(model=model, agent=agent, **kwargs)
+        results = await survey.run_async(
+            model=model,
+            agent=agent,
+            disable_remote_inference=disable_remote_inference,
+            **kwargs,
+        )
         if just_answer:
             return results.select(f"answer.{self.question_name}").first()
         else:

edsl/questions/QuestionBasePromptsMixin.py

@@ -30,38 +30,6 @@ template_manager = TemplateManager()
 
 
 class QuestionBasePromptsMixin:
-    # @classmethod
-    # @lru_cache(maxsize=1)
-    # def _read_template(cls, template_name):
-    #     with resources.open_text(
-    #         f"edsl.questions.templates.{cls.question_type}", template_name
-    #     ) as file:
-    #         return file.read()
-
-    # @classmethod
-    # def applicable_prompts(
-    #     cls, model: Optional[str] = None
-    # ) -> list[type["PromptBase"]]:
-    #     """Get the prompts that are applicable to the question type.
-
-    #     :param model: The language model to use.
-
-    #     >>> from edsl.questions import QuestionFreeText
-    #     >>> QuestionFreeText.applicable_prompts()
-    #     [<class 'edsl.prompts.library.question_freetext.FreeText'>]
-
-    #     :param model: The language model to use. If None, assumes does not matter.
-
-    #     """
-    #     from edsl.prompts.registry import get_classes as prompt_lookup
-
-    #     applicable_prompts = prompt_lookup(
-    #         component_type="question_instructions",
-    #         question_type=cls.question_type,
-    #         model=model,
-    #     )
-    #     return applicable_prompts
-
     @property
     def model_instructions(self) -> dict:
         """Get the model-specific instructions for the question."""
@@ -231,7 +199,7 @@ class QuestionBasePromptsMixin:
     @property
     def new_default_instructions(self) -> "Prompt":
         "This is set up as a property because there are mutable question values that determine how it is rendered."
-        return self.question_presentation + self.answering_instructions
+        return Prompt(self.question_presentation) + Prompt(self.answering_instructions)
 
     @property
     def parameters(self) -> set[str]:

edsl/questions/QuestionFreeText.py

@@ -14,6 +14,7 @@ from pydantic import BaseModel
 from typing import Optional, Any, List
 
 from edsl.exceptions import QuestionAnswerValidationError
+from edsl.prompts.Prompt import Prompt
 
 
 class FreeTextResponse(BaseModel):

edsl/questions/QuestionFunctional.py

@@ -19,7 +19,7 @@ class QuestionFunctional(QuestionBase):
     >>> question.activate()
     >>> scenario = Scenario({"numbers": [1, 2, 3, 4, 5]})
     >>> agent = Agent(traits={"multiplier": 10})
-    >>> results = question.by(scenario).by(agent).run()
+    >>> results = question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
     >>> results.select("answer.*").to_list()[0] == 150
     True
 
@@ -27,7 +27,7 @@ class QuestionFunctional(QuestionBase):
 
     >>> from edsl.questions.QuestionBase import QuestionBase
     >>> new_question = QuestionBase.from_dict(question.to_dict())
-    >>> results = new_question.by(scenario).by(agent).run()
+    >>> results = new_question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
     >>> results.select("answer.*").to_list()[0] == 150
     True