edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/language_models/model.py

@@ -0,0 +1,256 @@
+import textwrap
+from random import random
+from typing import Optional, TYPE_CHECKING, List
+
+from edsl.utilities.PrettyList import PrettyList
+from edsl.config import CONFIG
+
+from edsl.inference_services.InferenceServicesCollection import (
+    InferenceServicesCollection,
+)
+from edsl.inference_services.data_structures import AvailableModels
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.enums import InferenceServiceLiteral
+
+if TYPE_CHECKING:
+    from edsl.results.Dataset import Dataset
+
+
+def get_model_class(model_name, registry: Optional[InferenceServicesCollection] = None):
+    from edsl.inference_services.registry import default
+
+    registry = registry or default
+    factory = registry.create_model_factory(model_name)
+    return factory
+
+
+class Meta(type):
+    def __repr__(cls):
+        return textwrap.dedent(
+            f"""\
+        Available models: {cls.available()}
+        
+        To create an instance, you can do: 
+        >>> m = Model('gpt-4-1106-preview', temperature=0.5, ...)
+        
+        To get the default model, you can leave out the model name. 
+        To see the available models, you can do:
+        >>> Model.available()
+        """
+        )
+
+
+class Model(metaclass=Meta):
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+    _registry: InferenceServicesCollection = None  # Class-level registry storage
+
+    @classmethod
+    def get_registry(cls) -> InferenceServicesCollection:
+        """Get the current registry or initialize with default if None"""
+        if cls._registry is None:
+            from edsl.inference_services.registry import default
+
+            cls._registry = default
+        return cls._registry
+
+    @classmethod
+    def set_registry(cls, registry: InferenceServicesCollection) -> None:
+        """Set a new registry"""
+        cls._registry = registry
+
+    def __new__(
+        cls,
+        model_name: Optional[str] = None,
+        service_name: Optional[InferenceServiceLiteral] = None,
+        registry: Optional[InferenceServicesCollection] = None,
+        *args,
+        **kwargs,
+    ):
+        "Instantiate a new language model."
+        # Map index to the respective subclass
+        if model_name is None:
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
+
+        if registry is not None:
+            cls.set_registry(registry)
+
+        if isinstance(model_name, int):  # can refer to a model by index
+            model_name = cls.available(name_only=True)[model_name]
+
+        factory = cls.get_registry().create_model_factory(
+            model_name, service_name=service_name
+        )
+        return factory(*args, **kwargs)
+
+    @classmethod
+    def add_model(cls, service_name, model_name) -> None:
+        cls.get_registry().add_model(service_name, model_name)
+
+    @classmethod
+    def service_classes(cls) -> List["InferenceServiceABC"]:
+        """Returns a list of service classes.
+
+        >>> Model.service_classes()
+        [...]
+        """
+        return [r for r in cls.services(name_only=True)]
+
+    @classmethod
+    def services(cls, name_only: bool = False) -> List[str]:
+        """Returns a list of services, annotated with whether the user has local keys for them."""
+        services_with_local_keys = set(cls.key_info().select("service").to_list())
+        f = lambda service_name: (
+            "yes" if service_name in services_with_local_keys else " "
+        )
+        if name_only:
+            return PrettyList(
+                [r._inference_service_ for r in cls.get_registry().services],
+                columns=["Service Name"],
+            )
+        else:
+            return PrettyList(
+                [
+                    (r._inference_service_, f(r._inference_service_))
+                    for r in cls.get_registry().services
+                ],
+                columns=["Service Name", "Local key?"],
+            )
+
+    @classmethod
+    def services_with_local_keys(cls) -> set:
+        """Returns a list of services for which the user has local keys."""
+        return set(cls.key_info().select("service").to_list())
+
+    @classmethod
+    def key_info(cls, obscure_api_key: bool = True) -> "Dataset":
+        """Returns a dataset of local key information."""
+        from edsl.language_models.key_management.KeyLookupCollection import (
+            KeyLookupCollection,
+        )
+        from edsl.scenarios import Scenario, ScenarioList
+
+        klc = KeyLookupCollection()
+        klc.add_key_lookup(fetch_order=None)
+        sl = ScenarioList()
+        for service, entry in list(klc.data.values())[0].items():
+            sl.append(Scenario({"service": service} | entry.to_dict()))
+        if obscure_api_key:
+            for service in sl:
+                service["api_token"] = (
+                    service["api_token"][:4] + "..." + service["api_token"][-4:]
+                )
+        return sl.to_dataset()
+
+    @classmethod
+    def search_models(cls, search_term: str):
+        return cls.available(search_term=search_term)
+
+    @classmethod
+    def all_known_models(cls) -> "AvailableModels":
+        return cls.get_registry().available()
+
+    @classmethod
+    def available_with_local_keys(cls):
+        services_with_local_keys = set(cls.key_info().select("service").to_list())
+        return [
+            m
+            for m in cls.get_registry().available()
+            if m.service_name in services_with_local_keys
+        ]
+
+    @classmethod
+    def available(
+        cls,
+        search_term: str = None,
+        name_only: bool = False,
+        service: Optional[str] = None,
+    ):
+        # if search_term is None and service is None:
+        #     print("Getting available models...")
+        #     print("You have local keys for the following services:")
+        #     print(cls.services_with_local_keys())
+        #     print("\n")
+        #     print("To see models by service, use the 'service' parameter.")
+        #     print("E.g., Model.available(service='openai')")
+        #     return None
+
+        if service is not None:
+            if service not in cls.services(name_only=True):
+                raise ValueError(
+                    f"Service {service} not found in available services.",
+                    f"Available services are: {cls.services()}",
+                )
+
+        full_list = cls.get_registry().available(service=service)
+
+        if search_term is None:
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in full_list],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+        else:
+            filtered_results = [
+                m
+                for m in full_list
+                if search_term in m.model_name or search_term in m.service_name
+            ]
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in filtered_results],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+
+    @classmethod
+    def check_models(cls, verbose=False):
+        print("Checking all available models...\n")
+        for model in cls.available(name_only=True):
+            print(f"Now checking: {model}")
+            try:
+                m = cls(model)
+            except Exception as e:
+                print(f"Error creating instance of {model}: {e}")
+                continue
+            try:
+                results = m.hello(verbose)
+                if verbose:
+                    print(f"Results from model call: {results}")
+            except Exception as e:
+                print(f"Error calling 'hello' on {model}: {e}")
+                continue
+            print("OK!")
+            print("\n")
+
+    @classmethod
+    def example(cls, randomize: bool = False) -> "Model":
+        """
+        Returns an example Model instance.
+
+        :param randomize: If True, the temperature is set to a random decimal between 0 and 1.
+        """
+        temperature = 0.5 if not randomize else round(random(), 2)
+        model_name = cls.default_model
+        return cls(model_name, temperature=temperature)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    available = Model.available()
+    m = Model("gpt-4-1106-preview")
+    results = m.execute_model_call("Hello world")
+    print(results)

edsl/language_models/repair.py

@@ -1,76 +1,144 @@
 import json
 import asyncio
+import warnings
 
 
-async def async_repair(bad_json, error_message=""):
-    from edsl.language_models import LanguageModelOpenAIFour
+async def async_repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    from edsl.utilities.utilities import clean_json
 
-    m = LanguageModelOpenAIFour()
+    s = clean_json(bad_json)
 
-    prompt = f"""This is the output from a less capable language model.  
-    It was supposed to respond with just a JSON object with an answer to a question and some commentary, 
-    in a field called "comment" next to "answer".
-    Please repair this bad JSON: {bad_json}."""
-
-    if error_message:
-        prompt += f" Parsing error message: {error_message}"
+    try:
+        # this is the OpenAI version, but that's fine
+        valid_dict = json.loads(s)
+        success = True
+    except json.JSONDecodeError:
+        valid_dict = {}
+        success = False
+        # print("Replacing control characters didn't work. Trying extracting the sub-string.")
+    else:
+        return valid_dict, success
 
     try:
-        results = await m.async_execute_model_call(
-            prompt,
-            system_prompt="You are a helpful agent. Only return the repaired JSON, nothing else.",
+        from edsl.utilities.repair_functions import extract_json_from_string
+
+        valid_dict = extract_json_from_string(s)
+        success = True
+    except ValueError:
+        valid_dict = {}
+        success = False
+    else:
+        return valid_dict, success
+
+    from edsl.language_models.model import Model
+
+    m = Model()
+
+    from edsl.questions.QuestionExtract import QuestionExtract
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+
+        q = QuestionExtract(
+            question_text="""
+        A language model was supposed to respond to a question. 
+        The response should have been JSON object with an answer to a question and some commentary.
+        
+        It should have retured a string like this: 
+        
+        '{'answer': 'The answer to the question.', 'comment': 'Some commentary.'}'
+        
+        or:
+
+        '{'answer': 'The answer to the question.'}'
+
+        The answer field is very like an integer number. The comment field is always string.
+
+        You job is to return just the repaired JSON object that the model should have returned, properly formatted.
+
+            - It might have included some preliminary comments.
+            - It might have included some control characters.
+            - It might have included some extraneous text.
+
+        DO NOT include any extraneous text in your response. Just return the repaired JSON object.
+        Do not preface the JSON object with any text. Just return the JSON object.
+
+        Bad answer: """
+            + str(bad_json)
+            + "The model received a user prompt of: '"
+            + str(user_prompt)
+            + """'
+        The model received a system prompt of: ' """
+            + str(system_prompt)
+            + """
+        '
+        Please return the repaired JSON object, following the instructions the original model should have followed, though 
+        using 'new_answer' a nd 'new_comment' as the keys.""",
+            answer_template={
+                "new_answer": "<number, string, list, etc.>",
+                "new_comment": "Model's comments",
+            },
+            question_name="model_repair",
         )
-    except Exception as e:
-        return {}, False
+
+    results = await q.run_async(cache=cache)
 
     try:
-        valid_dict = json.loads(results["choices"][0]["message"]["content"])
+        # this is the OpenAI version, but that's fine
+        valid_dict = json.loads(json.dumps(results))
         success = True
+        # this is to deal with the fact that the model returns the answer and comment as new_answer and new_comment
+        valid_dict["answer"] = valid_dict.pop("new_answer")
+        valid_dict["comment"] = valid_dict.pop("new_comment")
     except json.JSONDecodeError:
         valid_dict = {}
         success = False
+        from rich import print
+        from rich.console import Console
+        from rich.syntax import Syntax
+
+        console = Console()
+        error_message = (
+            f"All repairs. failed. LLM Model given [red]{str(bad_json)}[/red]"
+        )
+        console.print("    " + error_message)
+        model_returned = results["choices"][0]["message"]["content"]
+        console.print(f"LLM Model returned: [blue]{model_returned}[/blue]")
 
     return valid_dict, success
 
 
-def repair_wrapper(bad_json, error_message=""):
+def repair_wrapper(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
     try:
         loop = asyncio.get_event_loop()
         if loop.is_running():
             # Add repair as a task to the running loop
-            task = loop.create_task(async_repair(bad_json, error_message))
+            task = loop.create_task(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
             return task
         else:
             # Run a new event loop for repair
-            return loop.run_until_complete(async_repair(bad_json, error_message))
+            return loop.run_until_complete(
+                async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+            )
     except RuntimeError:
         # Create a new event loop if one is not already available
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        return loop.run_until_complete(async_repair(bad_json, error_message))
-
-
-def repair(bad_json, error_message=""):
-    return repair_wrapper(bad_json, error_message)
-
-
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
-
-
-# def repair_wrapper(bad_json, error_message=""):
-#     loop = asyncio.get_event_loop()
-#     if loop.is_running():
-#         # Add repair as a task to the running loop
-#         task = loop.create_task(repair(bad_json, error_message))
-#         return task
-#     else:
-#         # Run a new event loop for repair
-#         return loop.run_until_complete(repair(bad_json, error_message))
+        return loop.run_until_complete(
+            async_repair(bad_json, error_message, user_prompt, system_prompt, cache)
+        )
 
 
-# Example usage:
-# result, success = repair_wrapper('{"name": "John Doe", "age": 30,}')  # example bad JSON
+def repair(
+    bad_json, error_message="", user_prompt=None, system_prompt=None, cache=None
+):
+    return repair_wrapper(bad_json, error_message, user_prompt, system_prompt, cache)
 
 
 if __name__ == "__main__":

edsl/language_models/utilities.py

@@ -0,0 +1,65 @@
+import asyncio
+from typing import Any, Optional, List
+from edsl.enums import InferenceServiceType
+
+
+def create_survey(num_questions: int, chained: bool = True, take_scenario=False):
+    from edsl.surveys.Survey import Survey
+    from edsl.questions.QuestionFreeText import QuestionFreeText
+
+    survey = Survey()
+    for i in range(num_questions):
+        if take_scenario:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX and {{scenario_value }}",
+                question_name=f"question_{i}",
+            )
+        else:
+            q = QuestionFreeText(
+                question_text=f"XX{i}XX", question_name=f"question_{i}"
+            )
+        survey.add_question(q)
+        if i > 0 and chained:
+            survey.add_targeted_memory(f"question_{i}", f"question_{i-1}")
+    return survey
+
+
+def create_language_model(
+    exception: Exception, fail_at_number: int, never_ending=False
+):
+    from edsl.language_models.LanguageModel import LanguageModel
+
+    class LanguageModelFromUtilities(LanguageModel):
+        _model_ = "test"
+        _parameters_ = {"temperature": 0.5}
+        _inference_service_ = InferenceServiceType.TEST.value
+        key_sequence = ["message", 0, "text"]
+        usage_sequence = ["usage"]
+        input_token_name = "prompt_tokens"
+        output_token_name = "completion_tokens"
+        _rpm = 1000000000000
+        _tpm = 1000000000000
+
+        async def async_execute_model_call(
+            self,
+            user_prompt: str,
+            system_prompt: str,
+            files_list: Optional[List[Any]] = None,
+        ) -> dict[str, Any]:
+            question_number = int(
+                user_prompt.split("XX")[1]
+            )  ## grabs the question number from the prompt
+            await asyncio.sleep(0.1)
+            if never_ending:  ## you're not going anywhere buddy
+                await asyncio.sleep(float("inf"))
+            if question_number == fail_at_number:
+                if asyncio.iscoroutinefunction(exception):
+                    await exception()
+                else:
+                    raise exception
+            return {
+                "message": [{"text": "SPAM!"}],
+                "usage": {"prompt_tokens": 1, "completion_tokens": 1},
+            }
+
+    return LanguageModelFromUtilities