edsl 0.1.37.dev4__py3-none-any.whl → 0.1.37.dev6__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -17,9 +17,7 @@ import warnings
 from functools import wraps
 import asyncio
 import json
-import time
 import os
-import hashlib
 from typing import (
     Coroutine,
     Any,
@@ -30,6 +28,7 @@ from typing import (
     get_type_hints,
     TypedDict,
     Optional,
+    TYPE_CHECKING,
 )
 from abc import ABC, abstractmethod
 
@@ -49,34 +48,16 @@ from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
 from edsl.language_models.repair import repair
 from edsl.enums import InferenceServiceType
 from edsl.Base import RichPrintingMixin, PersistenceMixin
-from edsl.enums import service_to_api_keyname
-from edsl.exceptions import MissingAPIKeyError
 from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
 from edsl.exceptions.language_models import LanguageModelBadResponseError
 
-TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
-
-
-def convert_answer(response_part):
-    import json
-
-    response_part = response_part.strip()
-
-    if response_part == "None":
-        return None
-
-    repaired = repair_json(response_part)
-    if repaired == '""':
-        # it was a literal string
-        return response_part
+from edsl.language_models.KeyLookup import KeyLookup
 
-    try:
-        return json.loads(repaired)
-    except json.JSONDecodeError as j:
-        # last resort
-        return response_part
+TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
+# you might be tempated to move this to be a static method of LanguageModel, but this doesn't work
+# for reasons I don't understand. So leave it here.
 def extract_item_from_raw_response(data, key_sequence):
     if isinstance(data, str):
         try:
@@ -167,7 +148,12 @@ class LanguageModel(
     _safety_factor = 0.8
 
     def __init__(
-        self, tpm=None, rpm=None, omit_system_prompt_if_empty_string=True, **kwargs
+        self,
+        tpm: float = None,
+        rpm: float = None,
+        omit_system_prompt_if_empty_string: bool = True,
+        key_lookup: Optional[KeyLookup] = None,
+        **kwargs,
     ):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
@@ -200,29 +186,26 @@ class LanguageModel(
             # Skip the API key check. Sometimes this is useful for testing.
             self._api_token = None
 
+        if key_lookup is not None:
+            self.key_lookup = key_lookup
+        else:
+            self.key_lookup = KeyLookup.from_os_environ()
+
     def ask_question(self, question):
         user_prompt = question.get_instructions().render(question.data).text
         system_prompt = "You are a helpful agent pretending to be a human."
         return self.execute_model_call(user_prompt, system_prompt)
 
+    def set_key_lookup(self, key_lookup: KeyLookup):
+        del self._api_token
+        self.key_lookup = key_lookup
+
     @property
     def api_token(self) -> str:
         if not hasattr(self, "_api_token"):
-            key_name = service_to_api_keyname.get(self._inference_service_, "NOT FOUND")
-            if self._inference_service_ == "bedrock":
-                self._api_token = [os.getenv(key_name[0]), os.getenv(key_name[1])]
-                # Check if any of the tokens are None
-                missing_token = any(token is None for token in self._api_token)
-            else:
-                self._api_token = os.getenv(key_name)
-                missing_token = self._api_token is None
-            if missing_token and self._inference_service_ != "test" and not self.remote:
-                print("raising error")
-                raise MissingAPIKeyError(
-                    f"""The key for service: `{self._inference_service_}` is not set.
-                        Need a key with name {key_name} in your .env file."""
-                )
-
+            self._api_token = self.key_lookup.get_api_token(
+                self._inference_service_, self.remote
+            )
         return self._api_token
 
     def __getitem__(self, key):
@@ -291,21 +274,6 @@ class LanguageModel(
         if tpm is not None:
             self._tpm = tpm
         return None
-        # self._set_rate_limits(rpm=rpm, tpm=tpm)
-
-    # def _set_rate_limits(self, rpm=None, tpm=None) -> None:
-    #     """Set the rate limits for the model.
-
-    #     If the model does not have rate limits, use the default rate limits."""
-    #     if rpm is not None and tpm is not None:
-    #         self.__rate_limits = {"rpm": rpm, "tpm": tpm}
-    #         return
-
-    #     if self.__rate_limits is None:
-    #         if hasattr(self, "get_rate_limits"):
-    #             self.__rate_limits = self.get_rate_limits()
-    #         else:
-    #             self.__rate_limits = self.__default_rate_limits
 
     @property
     def RPM(self):
@@ -416,6 +384,26 @@ class LanguageModel(
             )
         return extract_item_from_raw_response(raw_response, cls.usage_sequence)
 
+    @staticmethod
+    def convert_answer(response_part):
+        import json
+
+        response_part = response_part.strip()
+
+        if response_part == "None":
+            return None
+
+        repaired = repair_json(response_part)
+        if repaired == '""':
+            # it was a literal string
+            return response_part
+
+        try:
+            return json.loads(repaired)
+        except json.JSONDecodeError as j:
+            # last resort
+            return response_part
+
     @classmethod
     def parse_response(cls, raw_response: dict[str, Any]) -> EDSLOutput:
         """Parses the API response and returns the response text."""
@@ -425,13 +413,13 @@ class LanguageModel(
         if last_newline == -1:
             # There is no comment
             edsl_dict = {
-                "answer": convert_answer(generated_token_string),
+                "answer": cls.convert_answer(generated_token_string),
                 "generated_tokens": generated_token_string,
                 "comment": None,
             }
         else:
             edsl_dict = {
-                "answer": convert_answer(generated_token_string[:last_newline]),
+                "answer": cls.convert_answer(generated_token_string[:last_newline]),
                 "comment": generated_token_string[last_newline + 1 :].strip(),
                 "generated_tokens": generated_token_string,
             }
@@ -492,7 +480,7 @@ class LanguageModel(
             params = {
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
-                "files_list": files_list
+                "files_list": files_list,
                 # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
             # response = await f(**params)
@@ -699,7 +687,7 @@ class LanguageModel(
         True
         >>> from edsl import QuestionFreeText
         >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
-        >>> q.by(m).run(cache = False).select('example').first()
+        >>> q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True).select('example').first()
         'WOWZA!'
         """
         from edsl import Model

edsl/language_models/__init__.py

@@ -1,2 +1,3 @@
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.language_models.registry import Model
+from edsl.language_models.KeyLookup import KeyLookup

edsl/prompts/Prompt.py

@@ -240,10 +240,14 @@ class Prompt(PersistenceMixin, RichPrintingMixin):
         >>> p.render({"person": "Mr. {{last_name}}"})
         Prompt(text=\"""Hello, Mr. {{ last_name }}\""")
         """
-        new_text = self._render(
-            self.text, primary_replacement, **additional_replacements
-        )
-        return self.__class__(text=new_text)
+        try:
+            new_text = self._render(
+                self.text, primary_replacement, **additional_replacements
+            )
+            return self.__class__(text=new_text)
+        except Exception as e:
+            print(f"Error rendering prompt: {e}")
+            return self
 
     @staticmethod
     def _render(

edsl/questions/QuestionBase.py

@@ -150,14 +150,21 @@ class QuestionBase(
             "_include_comment",
             "_fake_data_factory",
             "_use_code",
-            "_answering_instructions",
-            "_question_presentation",
             "_model_instructions",
         ]
+        only_if_not_na_list = ["_answering_instructions", "_question_presentation"]
+
+        def ok(key, value):
+            if not key.startswith("_"):
+                return False
+            if key in exclude_list:
+                return False
+            if key in only_if_not_na_list and value is None:
+                return False
+            return True
+
         candidate_data = {
-            k.replace("_", "", 1): v
-            for k, v in self.__dict__.items()
-            if k.startswith("_") and k not in exclude_list
+            k.replace("_", "", 1): v for k, v in self.__dict__.items() if ok(k, v)
         }
 
         if "func" in candidate_data:
@@ -176,7 +183,9 @@ class QuestionBase(
         """
         candidate_data = self.data.copy()
         candidate_data["question_type"] = self.question_type
-        return candidate_data
+        return {
+            key: value for key, value in candidate_data.items() if value is not None
+        }
 
     @add_edsl_version
     def to_dict(self) -> dict[str, Any]:
@@ -239,6 +248,8 @@ class QuestionBase(
         show_answer: bool = True,
         model: Optional["LanguageModel"] = None,
         cache=False,
+        disable_remote_cache: bool = False,
+        disable_remote_inference: bool = False,
         **kwargs,
     ):
         """Run an example of the question.
@@ -247,7 +258,7 @@ class QuestionBase(
         >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
         >>> m.execute_model_call("", "")
         {'message': [{'text': "Yo, what's up?"}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
-        >>> Q.run_example(show_answer = True, model = m)
+        >>> Q.run_example(show_answer = True, model = m, disable_remote_cache = True, disable_remote_inference = True)
         ┏━━━━━━━━━━━━━━━━┓
         ┃ answer         ┃
         ┃ .how_are_you   ┃
@@ -259,25 +270,48 @@ class QuestionBase(
             from edsl import Model
 
             model = Model()
-        results = cls.example(**kwargs).by(model).run(cache=cache)
+        results = (
+            cls.example(**kwargs)
+            .by(model)
+            .run(
+                cache=cache,
+                disable_remote_cache=disable_remote_cache,
+                disable_remote_inference=disable_remote_inference,
+            )
+        )
         if show_answer:
             results.select("answer.*").print()
         else:
             return results
 
-    def __call__(self, just_answer=True, model=None, agent=None, **kwargs):
+    def __call__(
+        self,
+        just_answer=True,
+        model=None,
+        agent=None,
+        disable_remote_cache: bool = False,
+        disable_remote_inference: bool = False,
+        **kwargs,
+    ):
         """Call the question.
 
 
         >>> from edsl import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> q(model = m)
+        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True)
         "Yo, what's up?"
 
         """
         survey = self.to_survey()
-        results = survey(model=model, agent=agent, **kwargs, cache=False)
+        results = survey(
+            model=model,
+            agent=agent,
+            **kwargs,
+            cache=False,
+            disable_remote_cache=disable_remote_cache,
+            disable_remote_inference=disable_remote_inference,
+        )
         if just_answer:
             return results.select(f"answer.{self.question_name}").first()
         else:
@@ -295,6 +329,7 @@ class QuestionBase(
         just_answer: bool = True,
         model: Optional["Model"] = None,
         agent: Optional["Agent"] = None,
+        disable_remote_inference: bool = False,
         **kwargs,
     ) -> Union[Any, "Results"]:
         """Call the question asynchronously.
@@ -303,12 +338,17 @@ class QuestionBase(
         >>> from edsl import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Blue")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> async def test_run_async(): result = await q.run_async(model=m); print(result)
+        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True); print(result)
         >>> asyncio.run(test_run_async())
         Blue
         """
         survey = self.to_survey()
-        results = await survey.run_async(model=model, agent=agent, **kwargs)
+        results = await survey.run_async(
+            model=model,
+            agent=agent,
+            disable_remote_inference=disable_remote_inference,
+            **kwargs,
+        )
         if just_answer:
             return results.select(f"answer.{self.question_name}").first()
         else:

edsl/questions/QuestionBasePromptsMixin.py

@@ -30,38 +30,6 @@ template_manager = TemplateManager()
 
 
 class QuestionBasePromptsMixin:
-    # @classmethod
-    # @lru_cache(maxsize=1)
-    # def _read_template(cls, template_name):
-    #     with resources.open_text(
-    #         f"edsl.questions.templates.{cls.question_type}", template_name
-    #     ) as file:
-    #         return file.read()
-
-    # @classmethod
-    # def applicable_prompts(
-    #     cls, model: Optional[str] = None
-    # ) -> list[type["PromptBase"]]:
-    #     """Get the prompts that are applicable to the question type.
-
-    #     :param model: The language model to use.
-
-    #     >>> from edsl.questions import QuestionFreeText
-    #     >>> QuestionFreeText.applicable_prompts()
-    #     [<class 'edsl.prompts.library.question_freetext.FreeText'>]
-
-    #     :param model: The language model to use. If None, assumes does not matter.
-
-    #     """
-    #     from edsl.prompts.registry import get_classes as prompt_lookup
-
-    #     applicable_prompts = prompt_lookup(
-    #         component_type="question_instructions",
-    #         question_type=cls.question_type,
-    #         model=model,
-    #     )
-    #     return applicable_prompts
-
     @property
     def model_instructions(self) -> dict:
         """Get the model-specific instructions for the question."""
@@ -231,7 +199,7 @@ class QuestionBasePromptsMixin:
     @property
     def new_default_instructions(self) -> "Prompt":
         "This is set up as a property because there are mutable question values that determine how it is rendered."
-        return self.question_presentation + self.answering_instructions
+        return Prompt(self.question_presentation) + Prompt(self.answering_instructions)
 
     @property
     def parameters(self) -> set[str]:

edsl/questions/QuestionFunctional.py

@@ -19,7 +19,7 @@ class QuestionFunctional(QuestionBase):
     >>> question.activate()
     >>> scenario = Scenario({"numbers": [1, 2, 3, 4, 5]})
     >>> agent = Agent(traits={"multiplier": 10})
-    >>> results = question.by(scenario).by(agent).run()
+    >>> results = question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
     >>> results.select("answer.*").to_list()[0] == 150
     True
 
@@ -27,7 +27,7 @@ class QuestionFunctional(QuestionBase):
 
     >>> from edsl.questions.QuestionBase import QuestionBase
     >>> new_question = QuestionBase.from_dict(question.to_dict())
-    >>> results = new_question.by(scenario).by(agent).run()
+    >>> results = new_question.by(scenario).by(agent).run(disable_remote_cache = True, disable_remote_inference = True)
     >>> results.select("answer.*").to_list()[0] == 150
     True
 

edsl/questions/descriptors.py

@@ -53,33 +53,12 @@ class BaseDescriptor(ABC):
 
     def __set__(self, instance, value: Any) -> None:
         """Set the value of the attribute."""
-        self.validate(value, instance)
-        # from edsl.prompts.registry import get_classes
-
-        instance.__dict__[self.name] = value
-        # if self.name == "_instructions":
-        #     instructions = value
-        #     if value is not None:
-        #         instance.__dict__[self.name] = instructions
-        #         instance.set_instructions = True
-        #     else:
-        #         potential_prompt_classes = get_classes(
-        #             question_type=instance.question_type
-        #         )
-        #         if len(potential_prompt_classes) > 0:
-        #             instructions = potential_prompt_classes[0]().text
-        #             instance.__dict__[self.name] = instructions
-        #             instance.set_instructions = False
-        #         else:
-        #             if not hasattr(instance, "default_instructions"):
-        #                 raise Exception(
-        #                     "No default instructions found and no matching prompts!"
-        #                 )
-        #             instructions = instance.default_instructions
-        #             instance.__dict__[self.name] = instructions
-        #             instance.set_instructions = False
-
-        # instance.set_instructions = value != instance.default_instructions
+        new_value = self.validate(value, instance)
+
+        if new_value is not None:
+            instance.__dict__[self.name] = new_value
+        else:
+            instance.__dict__[self.name] = value
 
     def __set_name__(self, owner, name: str) -> None:
         """Set the name of the attribute."""
@@ -400,10 +379,24 @@ class QuestionTextDescriptor(BaseDescriptor):
         if contains_single_braced_substring(value):
             import warnings
 
+            # # warnings.warn(
+            # #     f"WARNING: Question text contains a single-braced substring: If you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html",
+            # #     UserWarning,
+            # # )
             warnings.warn(
-                f"WARNING: Question text contains a single-braced substring: If you intended to parameterize the question with a Scenario this should be changed to a double-braced substring, e.g. {{variable}}.\nSee details on constructing Scenarios in the docs: https://docs.expectedparrot.com/en/latest/scenarios.html",
+                "WARNING: Question text contains a single-braced substring. "
+                "If you intended to parameterize the question with a Scenario, this will "
+                "be changed to a double-braced substring, e.g. {{variable}}.\n"
+                "See details on constructing Scenarios in the docs: "
+                "https://docs.expectedparrot.com/en/latest/scenarios.html",
                 UserWarning,
             )
+            # Automatically replace single braces with double braces
+            # This is here because if the user is using an f-string, the double brace will get converted to a single brace.
+            # This undoes that.
+            value = re.sub(r"\{([^\{\}]+)\}", r"{{\1}}", value)
+            return value
+
         # iterate through all doubles braces and check if they are valid python identifiers
         for match in re.finditer(r"\{\{([^\{\}]+)\}\}", value):
             if " " in match.group(1).strip():
@@ -411,6 +404,8 @@ class QuestionTextDescriptor(BaseDescriptor):
                     f"Question text contains an invalid identifier: '{match.group(1)}'"
                 )
 
+        return None
+
 
 if __name__ == "__main__":
     import doctest

edsl/results/DatasetExportMixin.py

@@ -437,7 +437,30 @@ class DatasetExportMixin:
         b64 = base64.b64encode(csv_string.encode()).decode()
         return f'<a href="data:file/csv;base64,{b64}" download="my_data.csv">Download CSV file</a>'
 
-    def to_pandas(self, remove_prefix: bool = False) -> "pd.DataFrame":
+    def to_pandas(
+        self, remove_prefix: bool = False, lists_as_strings=False
+    ) -> "DataFrame":
+        """Convert the results to a pandas DataFrame, ensuring that lists remain as lists.
+
+        :param remove_prefix: Whether to remove the prefix from the column names.
+
+        """
+        return self._to_pandas_strings(remove_prefix)
+        # if lists_as_strings:
+        #     return self._to_pandas_strings(remove_prefix=remove_prefix)
+
+        # import pandas as pd
+
+        # df = pd.DataFrame(self.data)
+
+        # if remove_prefix:
+        #     # Optionally remove prefixes from column names
+        #     df.columns = [col.split(".")[-1] for col in df.columns]
+
+        # df_sorted = df.sort_index(axis=1)  # Sort columns alphabetically
+        # return df_sorted
+
+    def _to_pandas_strings(self, remove_prefix: bool = False) -> "pd.DataFrame":
         """Convert the results to a pandas DataFrame.
 
         :param remove_prefix: Whether to remove the prefix from the column names.
@@ -451,6 +474,7 @@ class DatasetExportMixin:
         2           Terrible
         3                 OK
         """
+
         import pandas as pd
 
         csv_string = self.to_csv(remove_prefix=remove_prefix)

edsl/results/Result.py

@@ -257,10 +257,25 @@ class Result(Base, UserDict):
 
         """
         d = {}
-        data_types = self.sub_dicts.keys()
+        problem_keys = []
+        data_types = sorted(self.sub_dicts.keys())
         for data_type in data_types:
             for key in self.sub_dicts[data_type]:
+                if key in d:
+                    import warnings
+
+                    warnings.warn(
+                        f"Key '{key}' of data type '{data_type}' is already in use. Renaming to {key}_{data_type}"
+                    )
+                    problem_keys.append((key, data_type))
+                    key = f"{key}_{data_type}"
+                    # raise ValueError(f"Key '{key}' is already in the dictionary")
                 d[key] = data_type
+
+        for key, data_type in problem_keys:
+            self.sub_dicts[data_type][f"{key}_{data_type}"] = self.sub_dicts[
+                data_type
+            ].pop(key)
         return d
 
     def rows(self, index) -> tuple[int, str, str, str]: