edsl 0.1.27.dev2__py3-none-any.whl → 0.1.29__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -7,26 +7,18 @@ import asyncio
 import json
 import time
 import os
-
 from typing import Coroutine, Any, Callable, Type, List, get_type_hints
-
-from abc import ABC, abstractmethod, ABCMeta
-
-from rich.table import Table
+from abc import ABC, abstractmethod
 
 from edsl.config import CONFIG
 
-from edsl.utilities.utilities import clean_json
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+
 from edsl.language_models.repair import repair
-from edsl.exceptions.language_models import LanguageModelAttributeTypeError
 from edsl.enums import InferenceServiceType
 from edsl.Base import RichPrintingMixin, PersistenceMixin
-from edsl.data.Cache import Cache
 from edsl.enums import service_to_api_keyname
-
-
 from edsl.exceptions import MissingAPIKeyError
 from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
 
@@ -142,7 +134,7 @@ class LanguageModel(
     def has_valid_api_key(self) -> bool:
         """Check if the model has a valid API key.
 
-        >>> LanguageModel.example().has_valid_api_key()
+        >>> LanguageModel.example().has_valid_api_key() : # doctest: +SKIP
         True
 
         This method is used to check if the model has a valid API key.
@@ -159,7 +151,9 @@ class LanguageModel(
 
     def __hash__(self):
         """Allow the model to be used as a key in a dictionary."""
-        return hash(self.model + str(self.parameters))
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict())
 
     def __eq__(self, other):
         """Check is two models are the same.
@@ -207,8 +201,8 @@ class LanguageModel(
         """Model's tokens-per-minute limit.
 
         >>> m = LanguageModel.example()
-        >>> m.TPM
-        1600000.0
+        >>> m.TPM > 0
+        True
         """
         self._set_rate_limits()
         return self._safety_factor * self.__rate_limits["tpm"]
@@ -285,36 +279,14 @@ class LanguageModel(
         """
         raise NotImplementedError
 
-    def _update_response_with_tracking(
-        self, response: dict, start_time: int, cached_response=False, cache_key=None
-    ):
-        """Update the response with tracking information.
-
-        >>> m = LanguageModel.example()
-        >>> m._update_response_with_tracking(response={"response": "Hello"}, start_time=0, cached_response=False, cache_key=None)
-        {'response': 'Hello', 'elapsed_time': ..., 'timestamp': ..., 'cached_response': False, 'cache_key': None}
-
-
-        """
-        end_time = time.time()
-        response.update(
-            {
-                "elapsed_time": end_time - start_time,
-                "timestamp": end_time,
-                "cached_response": cached_response,
-                "cache_key": cache_key,
-            }
-        )
-        return response
-
     async def async_get_raw_response(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache,
+        cache: "Cache",
         iteration: int = 0,
         encoded_image=None,
-    ) -> dict[str, Any]:
+    ) -> tuple[dict, bool, str]:
         """Handle caching of responses.
 
         :param user_prompt: The user's prompt.
@@ -322,8 +294,7 @@ class LanguageModel(
         :param iteration: The iteration number.
         :param cache: The cache to use.
 
-        If the cache isn't being used, it just returns a 'fresh' call to the LLM,
-        but appends some tracking information to the response (using the _update_response_with_tracking method).
+        If the cache isn't being used, it just returns a 'fresh' call to the LLM.
         But if cache is being used, it first checks the database to see if the response is already there.
         If it is, it returns the cached response, but again appends some tracking information.
         If it isn't, it calls the LLM, saves the response to the database, and returns the response with tracking information.
@@ -334,7 +305,7 @@ class LanguageModel(
         >>> from edsl import Cache
         >>> m = LanguageModel.example(test_model = True)
         >>> m.get_raw_response(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
-        {'message': '{"answer": "Hello world"}', 'elapsed_time': ..., 'timestamp': ..., 'cached_response': False, 'cache_key': '24ff6ac2bc2f1729f817f261e0792577'}
+        ({'message': '{"answer": "Hello world"}'}, False, '24ff6ac2bc2f1729f817f261e0792577')
         """
         start_time = time.time()
 
@@ -379,12 +350,7 @@ class LanguageModel(
             )
             cache_used = False
 
-        return self._update_response_with_tracking(
-            response=response,
-            start_time=start_time,
-            cached_response=cache_used,
-            cache_key=cache_key,
-        )
+        return response, cache_used, cache_key
 
     get_raw_response = sync_wrapper(async_get_raw_response)
 
@@ -427,14 +393,18 @@ class LanguageModel(
         if encoded_image:
             params["encoded_image"] = encoded_image
 
-        raw_response = await self.async_get_raw_response(**params)
+        raw_response, cache_used, cache_key = await self.async_get_raw_response(
+            **params
+        )
         response = self.parse_response(raw_response)
 
         try:
             dict_response = json.loads(response)
         except json.JSONDecodeError as e:
             # TODO: Turn into logs to generate issues
-            dict_response, success = await repair(response, str(e))
+            dict_response, success = await repair(
+                bad_json=response, error_message=str(e), cache=cache
+            )
             if not success:
                 raise Exception(
                     f"""Even the repair failed. The error was: {e}. The response was: {response}."""
@@ -442,7 +412,8 @@ class LanguageModel(
 
         dict_response.update(
             {
-                "cached_response": raw_response["cached_response"],
+                "cached_used": cache_used,
+                "cache_key": cache_key,
                 "usage": raw_response.get("usage", {}),
                 "raw_model_response": raw_response,
             }
@@ -458,15 +429,18 @@ class LanguageModel(
     #######################
     # SERIALIZATION METHODS
     #######################
+    def _to_dict(self) -> dict[str, Any]:
+        return {"model": self.model, "parameters": self.parameters}
+
     @add_edsl_version
     def to_dict(self) -> dict[str, Any]:
         """Convert instance to a dictionary.
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
-        {'model': 'gpt-4-1106-preview', 'parameters': {'temperature': 0.5, 'max_tokens': 1000, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'logprobs': False, 'top_logprobs': 3}}
+        {'model': 'gpt-4-1106-preview', 'parameters': {'temperature': 0.5, 'max_tokens': 1000, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'logprobs': False, 'top_logprobs': 3}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
-        return {"model": self.model, "parameters": self.parameters}
+        return self._to_dict()
 
     @classmethod
     @remove_edsl_version
@@ -508,6 +482,8 @@ class LanguageModel(
 
     def rich_print(self):
         """Display an object as a table."""
+        from rich.table import Table
+
         table = Table(title="Language Model")
         table.add_column("Attribute", style="bold")
         table.add_column("Value")
@@ -519,8 +495,18 @@ class LanguageModel(
         return table
 
     @classmethod
-    def example(cls, test_model=False):
-        """Return a default instance of the class."""
+    def example(cls, test_model: bool = False, canned_response: str = "Hello world"):
+        """Return a default instance of the class.
+
+        >>> from edsl.language_models import LanguageModel
+        >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!")
+        >>> isinstance(m, LanguageModel)
+        True
+        >>> from edsl import QuestionFreeText
+        >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
+        >>> q.by(m).run(cache = False).select('example').first()
+        'WOWZA!'
+        """
         from edsl import Model
 
         class TestLanguageModelGood(LanguageModel):
@@ -533,7 +519,8 @@ class LanguageModel(
                 self, user_prompt: str, system_prompt: str
             ) -> dict[str, Any]:
                 await asyncio.sleep(0.1)
-                return {"message": """{"answer": "Hello world"}"""}
+                # return {"message": """{"answer": "Hello, world"}"""}
+                return {"message": f'{{"answer": "{canned_response}"}}'}
 
             def parse_response(self, raw_response: dict[str, Any]) -> str:
                 return raw_response["message"]

edsl/language_models/ModelList.py

@@ -0,0 +1,96 @@
+from typing import Optional
+from collections import UserList
+from edsl import Model
+
+from edsl.language_models import LanguageModel
+from edsl.Base import Base
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.utilities.utilities import is_valid_variable_name
+from edsl.utilities.utilities import dict_hash
+
+
+class ModelList(Base, UserList):
+    def __init__(self, data: Optional[list] = None):
+        """Initialize the ScenarioList class.
+
+        >>> from edsl import Model
+        >>> m = ModelList(Model.available())
+
+        """
+        if data is not None:
+            super().__init__(data)
+        else:
+            super().__init__([])
+
+    @property
+    def names(self):
+        """
+
+        >>> ModelList.example().names
+        {'...'}
+        """
+        return set([model.model for model in self])
+
+    def rich_print(self):
+        pass
+
+    def __repr__(self):
+        return f"ModelList({super().__repr__()})"
+
+    def __hash__(self):
+        """Return a hash of the ModelList. This is used for comparison of ModelLists.
+
+        >>> hash(ModelList.example())
+        1423518243781418961
+
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self._to_dict(sort=True))
+
+    def _to_dict(self, sort=False):
+        if sort:
+            model_list = sorted([model for model in self], key=lambda x: hash(x))
+            return {"models": [model._to_dict() for model in model_list]}
+        else:
+            return {"models": [model._to_dict() for model in self]}
+
+    @classmethod
+    def from_names(self, *args, **kwargs):
+        """A a model list from a list of names"""
+        if len(args) == 1 and isinstance(args[0], list):
+            args = args[0]
+        return ModelList([Model(model_name, **kwargs) for model_name in args])
+
+    @add_edsl_version
+    def to_dict(self):
+        """
+        Convert the ModelList to a dictionary.
+        >>> ModelList.example().to_dict()
+        {'models': [...], 'edsl_version': '...', 'edsl_class_name': 'ModelList'}
+        """
+        return self._to_dict()
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        """
+        Create a ModelList from a dictionary.
+
+        >>> newm = ModelList.from_dict(ModelList.example().to_dict())
+        >>> assert ModelList.example() == newm
+        """
+        return cls(data=[LanguageModel.from_dict(model) for model in data["models"]])
+
+    def code(self):
+        pass
+
+    @classmethod
+    def example(cl):
+        return ModelList([LanguageModel.example() for _ in range(3)])
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/registry.py

@@ -38,6 +38,20 @@ class Model(metaclass=Meta):
         factory = registry.create_model_factory(model_name)
         return factory(*args, **kwargs)
 
+    @classmethod
+    def add_model(cls, service_name, model_name):
+        from edsl.inference_services.registry import default
+
+        registry = default
+        registry.add_model(service_name, model_name)
+
+    @classmethod
+    def services(cls, registry=None):
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+        return [r._inference_service_ for r in registry.services]
+
     @classmethod
     def available(cls, search_term=None, name_only=False, registry=None):
         from edsl.inference_services.registry import default

edsl/language_models/repair.py

@@ -1,14 +1,14 @@
 import json
 import asyncio
+import warnings
 
-from edsl.utilities.utilities import clean_json
 
+async def async_repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    from edsl.utilities.utilities import clean_json
 
-async def async_repair(bad_json, error_message=""):
     s = clean_json(bad_json)
-    from edsl import Model
-
-    m = Model()
 
     try:
         # this is the OpenAI version, but that's fine
@@ -17,56 +17,128 @@ async def async_repair(bad_json, error_message=""):
     except json.JSONDecodeError:
         valid_dict = {}
         success = False
-        # print("Replacing control characters didn't work. Trying with the model.")
+        # print("Replacing control characters didn't work. Trying extracting the sub-string.")
+    else:
+        return valid_dict, success
+
+    try:
+        from edsl.utilities.repair_functions import extract_json_from_string
+
+        valid_dict = extract_json_from_string(s)
+        success = True
+    except ValueError:
+        valid_dict = {}
+        success = False
     else:
         return valid_dict, success
 
-    prompt = f"""This is the output from a less capable language model.  
-    It was supposed to respond with just a JSON object with an answer to a question and some commentary, 
-    in a field called "comment" next to "answer".
-    Please repair this bad JSON: {bad_json}."""
+    from edsl import Model
 
-    if error_message:
-        prompt += f" Parsing error message: {error_message}"
+    m = Model()
 
-    try:
-        results = await m.async_execute_model_call(
-            prompt,
-            system_prompt="You are a helpful agent. Only return the repaired JSON, nothing else.",
+    from edsl import QuestionExtract
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+
+        q = QuestionExtract(
+            question_text="""
+        A language model was supposed to respond to a question. 
+        The response should have been JSON object with an answer to a question and some commentary.
+        
+        It should have retured a string like this: 
+        
+        '{'answer': 'The answer to the question.', 'comment': 'Some commentary.'}'
+        
+        or:
+
+        '{'answer': 'The answer to the question.'}'
+
+        The answer field is very like an integer number. The comment field is always string.
+
+        You job is to return just the repaired JSON object that the model should have returned, properly formatted.
+
+            - It might have included some preliminary comments.
+            - It might have included some control characters.
+            - It might have included some extraneous text.
+
+        DO NOT include any extraneous text in your response. Just return the repaired JSON object.
+        Do not preface the JSON object with any text. Just return the JSON object.
+
+        Bad answer: """
+            + str(bad_json)
+            + "The model received a user prompt of: '"
+            + str(user_prompt)
+            + """'
+        The model received a system prompt of: ' """
+            + str(system_prompt)
+            + """
+        '
+        Please return the repaired JSON object, following the instructions the original model should have followed, though 
+        using 'new_answer' a nd 'new_comment' as the keys.""",
+            answer_template={
+                "new_answer": "<number, string, list, etc.>",
+                "new_comment": "Model's comments",
+            },
+            question_name="model_repair",
         )
-    except Exception as e:
-        return {}, False
+
+    results = await q.run_async(cache=cache)
 
     try:
         # this is the OpenAI version, but that's fine
-        valid_dict = json.loads(results["choices"][0]["message"]["content"])
+        valid_dict = json.loads(json.dumps(results))
         success = True
+        # this is to deal with the fact that the model returns the answer and comment as new_answer and new_comment
+        valid_dict["answer"] = valid_dict.pop("new_answer")
+        valid_dict["comment"] = valid_dict.pop("new_comment")
     except json.JSONDecodeError:
         valid_dict = {}
         success = False
+        from rich import print
+        from rich.console import Console
+        from rich.syntax import Syntax
+
+        console = Console()
+        error_message = (
+            f"All repairs. failed. LLM Model given [red]{str(bad_json)}[/red]"
+        )
+        console.print("    " + error_message)
+        model_returned = results["choices"][0]["message"]["content"]
+        console.print(f"LLM Model returned: [blue]{model_returned}[/blue]")
 
     return valid_dict, success
 
 
-def repair_wrapper(bad_json, error_message=""):
+def repair_wrapper(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
     try:
         loop = asyncio.get_event_loop()
         if loop.is_running():
             # Add repair as a task to the running loop
-            task = loop.create_task(async_repair(bad_json, error_message))
+            task = loop.create_task(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
             return task
         else:
             # Run a new event loop for repair
-            return loop.run_until_complete(async_repair(bad_json, error_message))
+            return loop.run_until_complete(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
     except RuntimeError:
         # Create a new event loop if one is not already available
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        return loop.run_until_complete(async_repair(bad_json, error_message))
+        return loop.run_until_complete(
+            async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+        )
 
 
-def repair(bad_json, error_message=""):
-    return repair_wrapper(bad_json, error_message)
+def repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
 
 
 # Example usage: