edsl 0.1.33.dev1__py3-none-any.whl → 0.1.33.dev2__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -1,4 +1,16 @@
-"""This module contains the LanguageModel class, which is an abstract base class for all language models."""
+"""This module contains the LanguageModel class, which is an abstract base class for all language models.
+
+Terminology:
+
+raw_response: The JSON response from the model. This has all the model meta-data about the call. 
+
+edsl_augmented_response: The JSON response from model, but augmented with EDSL-specific information, 
+such as the cache key, token usage, etc. 
+
+generated_tokens: The actual tokens generated by the model. This is the output that is used by the user.
+edsl_answer_dict: The parsed JSON response from the model either {'answer': ...} or {'answer': ..., 'comment': ...}
+
+"""
 
 from __future__ import annotations
 import warnings
@@ -8,47 +20,103 @@ import json
 import time
 import os
 import hashlib
-from typing import Coroutine, Any, Callable, Type, List, get_type_hints
+from typing import (
+    Coroutine,
+    Any,
+    Callable,
+    Type,
+    Union,
+    List,
+    get_type_hints,
+    TypedDict,
+    Optional,
+)
 from abc import ABC, abstractmethod
 
+from json_repair import repair_json
 
-class IntendedModelCallOutcome:
-    "This is a tuple-like class that holds the response, cache_used, and cache_key."
-
-    def __init__(self, response: dict, cache_used: bool, cache_key: str):
-        self.response = response
-        self.cache_used = cache_used
-        self.cache_key = cache_key
-
-    def __iter__(self):
-        """Iterate over the class attributes.
-
-        >>> a, b, c = IntendedModelCallOutcome({'answer': "yes"}, True, 'x1289')
-        >>> a
-        {'answer': 'yes'}
-        """
-        yield self.response
-        yield self.cache_used
-        yield self.cache_key
-
-    def __len__(self):
-        return 3
-
-    def __repr__(self):
-        return f"IntendedModelCallOutcome(response = {self.response}, cache_used = {self.cache_used}, cache_key = '{self.cache_key}')"
+from edsl.data_transfer_models import (
+    ModelResponse,
+    ModelInputs,
+    EDSLOutput,
+    AgentResponseDict,
+)
 
 
 from edsl.config import CONFIG
-
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
 from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-
 from edsl.language_models.repair import repair
 from edsl.enums import InferenceServiceType
 from edsl.Base import RichPrintingMixin, PersistenceMixin
 from edsl.enums import service_to_api_keyname
 from edsl.exceptions import MissingAPIKeyError
 from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
+from edsl.exceptions.language_models import LanguageModelBadResponseError
+
+TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
+
+
+def convert_answer(response_part):
+    import json
+
+    response_part = response_part.strip()
+
+    if response_part == "None":
+        return None
+
+    repaired = repair_json(response_part)
+    if repaired == '""':
+        # it was a literal string
+        return response_part
+
+    try:
+        return json.loads(repaired)
+    except json.JSONDecodeError as j:
+        # last resort
+        return response_part
+
+
+def extract_item_from_raw_response(data, key_sequence):
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except json.JSONDecodeError as e:
+            return data
+    current_data = data
+    for i, key in enumerate(key_sequence):
+        try:
+            if isinstance(current_data, (list, tuple)):
+                if not isinstance(key, int):
+                    raise TypeError(
+                        f"Expected integer index for sequence at position {i}, got {type(key).__name__}"
+                    )
+                if key < 0 or key >= len(current_data):
+                    raise IndexError(
+                        f"Index {key} out of range for sequence of length {len(current_data)} at position {i}"
+                    )
+            elif isinstance(current_data, dict):
+                if key not in current_data:
+                    raise KeyError(
+                        f"Key '{key}' not found in dictionary at position {i}"
+                    )
+            else:
+                raise TypeError(
+                    f"Cannot index into {type(current_data).__name__} at position {i}. Full response is: {data} of type {type(data)}. Key sequence is: {key_sequence}"
+                )
+
+            current_data = current_data[key]
+        except Exception as e:
+            path = " -> ".join(map(str, key_sequence[: i + 1]))
+            if "error" in data:
+                msg = data["error"]
+            else:
+                msg = f"Error accessing path: {path}. {str(e)}. Full response is: '{data}'"
+            raise LanguageModelBadResponseError(message=msg, response_json=data)
+    if isinstance(current_data, str):
+        return current_data.strip()
+    else:
+        return current_data
 
 
 def handle_key_error(func):
@@ -92,7 +160,9 @@ class LanguageModel(
     """
 
     _model_ = None
-
+    key_sequence = (
+        None  # This should be something like ["choices", 0, "message", "content"]
+    )
     __rate_limits = None
     __default_rate_limits = {
         "rpm": 10_000,
@@ -100,7 +170,7 @@ class LanguageModel(
     }  # TODO: Use the OpenAI Teir 1 rate limits
     _safety_factor = 0.8
 
-    def __init__(self, **kwargs):
+    def __init__(self, tpm=None, rpm=None, **kwargs):
         """Initialize the LanguageModel."""
         self.model = getattr(self, "_model_", None)
         default_parameters = getattr(self, "_parameters_", None)
@@ -108,6 +178,12 @@ class LanguageModel(
         self.parameters = parameters
         self.remote = False
 
+        if rpm is not None:
+            self._rpm = rpm
+
+        if tpm is not None:
+            self._tpm = tpm
+
         for key, value in parameters.items():
             setattr(self, key, value)
 
@@ -133,7 +209,6 @@ class LanguageModel(
     def api_token(self) -> str:
         if not hasattr(self, "_api_token"):
             key_name = service_to_api_keyname.get(self._inference_service_, "NOT FOUND")
-
             if self._inference_service_ == "bedrock":
                 self._api_token = [os.getenv(key_name[0]), os.getenv(key_name[1])]
                 # Check if any of the tokens are None
@@ -142,13 +217,13 @@ class LanguageModel(
                 self._api_token = os.getenv(key_name)
                 missing_token = self._api_token is None
             if missing_token and self._inference_service_ != "test" and not self.remote:
-                print("rainsing error")
+                print("raising error")
                 raise MissingAPIKeyError(
                     f"""The key for service: `{self._inference_service_}` is not set.
                         Need a key with name {key_name} in your .env file."""
                 )
 
-            return self._api_token
+        return self._api_token
 
     def __getitem__(self, key):
         return getattr(self, key)
@@ -209,7 +284,7 @@ class LanguageModel(
         >>> m = LanguageModel.example()
         >>> m.set_rate_limits(rpm=100, tpm=1000)
         >>> m.RPM
-        80.0
+        100
         """
         self._set_rate_limits(rpm=rpm, tpm=tpm)
 
@@ -230,8 +305,32 @@ class LanguageModel(
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
-        self._set_rate_limits()
-        return self._safety_factor * self.__rate_limits["rpm"]
+        # self._set_rate_limits()
+        # return self._safety_factor * self.__rate_limits["rpm"]
+        return self.rpm
+
+    @property
+    def TPM(self):
+        """Model's tokens-per-minute limit."""
+        # self._set_rate_limits()
+        # return self._safety_factor * self.__rate_limits["tpm"]
+        return self.tpm
+
+    @property
+    def rpm(self):
+        return self._rpm
+
+    @rpm.setter
+    def rpm(self, value):
+        self._rpm = value
+
+    @property
+    def tpm(self):
+        return self._tpm
+
+    @tpm.setter
+    def tpm(self, value):
+        self._tpm = value
 
     @property
     def TPM(self):
@@ -270,11 +369,10 @@ class LanguageModel(
         >>> m = LanguageModel.example(test_model = True)
         >>> async def test(): return await m.async_execute_model_call("Hello, model!", "You are a helpful agent.")
         >>> asyncio.run(test())
-        {'message': '{"answer": "Hello world"}'}
+        {'message': [{'text': 'Hello world'}], ...}
 
         >>> m.execute_model_call("Hello, model!", "You are a helpful agent.")
-        {'message': '{"answer": "Hello world"}'}
-
+        {'message': [{'text': 'Hello world'}], ...}
         """
         pass
 
@@ -307,68 +405,40 @@ class LanguageModel(
 
         return main()
 
-    @abstractmethod
-    def parse_response(raw_response: dict[str, Any]) -> str:
-        """Parse the response and returns the response text.
-
-        >>> m = LanguageModel.example(test_model = True)
-        >>> m
-        Model(model_name = 'test', temperature = 0.5)
-
-        What is returned by the API is model-specific and often includes meta-data that we do not need.
-        For example, here is the results from a call to GPT-4:
-        To actually track the response, we need to grab
-        data["choices[0]"]["message"]["content"].
-        """
-        raise NotImplementedError
-
-    async def _async_prepare_response(
-        self, model_call_outcome: IntendedModelCallOutcome, cache: "Cache"
-    ) -> dict:
-        """Prepare the response for return."""
-
-        model_response = {
-            "cache_used": model_call_outcome.cache_used,
-            "cache_key": model_call_outcome.cache_key,
-            "usage": model_call_outcome.response.get("usage", {}),
-            "raw_model_response": model_call_outcome.response,
-        }
+    @classmethod
+    def get_generated_token_string(cls, raw_response: dict[str, Any]) -> str:
+        """Return the generated token string from the raw response."""
+        return extract_item_from_raw_response(raw_response, cls.key_sequence)
 
-        answer_portion = self.parse_response(model_call_outcome.response)
-        try:
-            answer_dict = json.loads(answer_portion)
-        except json.JSONDecodeError as e:
-            # TODO: Turn into logs to generate issues
-            answer_dict, success = await repair(
-                bad_json=answer_portion, error_message=str(e), cache=cache
+    @classmethod
+    def get_usage_dict(cls, raw_response: dict[str, Any]) -> dict[str, Any]:
+        """Return the usage dictionary from the raw response."""
+        if not hasattr(cls, "usage_sequence"):
+            raise NotImplementedError(
+                "This inference service does not have a usage_sequence."
             )
-            if not success:
-                raise Exception(
-                    f"""Even the repair failed. The error was: {e}. The response was: {answer_portion}."""
-                )
-
-        return {**model_response, **answer_dict}
+        return extract_item_from_raw_response(raw_response, cls.usage_sequence)
 
-    async def async_get_raw_response(
-        self,
-        user_prompt: str,
-        system_prompt: str,
-        cache: "Cache",
-        iteration: int = 0,
-        encoded_image=None,
-    ) -> IntendedModelCallOutcome:
-        import warnings
-
-        warnings.warn(
-            "This method is deprecated. Use async_get_intended_model_call_outcome."
-        )
-        return await self._async_get_intended_model_call_outcome(
-            user_prompt=user_prompt,
-            system_prompt=system_prompt,
-            cache=cache,
-            iteration=iteration,
-            encoded_image=encoded_image,
-        )
+    @classmethod
+    def parse_response(cls, raw_response: dict[str, Any]) -> EDSLOutput:
+        """Parses the API response and returns the response text."""
+        generated_token_string = cls.get_generated_token_string(raw_response)
+        last_newline = generated_token_string.rfind("\n")
+
+        if last_newline == -1:
+            # There is no comment
+            edsl_dict = {
+                "answer": convert_answer(generated_token_string),
+                "generated_tokens": generated_token_string,
+                "comment": None,
+            }
+        else:
+            edsl_dict = {
+                "answer": convert_answer(generated_token_string[:last_newline]),
+                "comment": generated_token_string[last_newline + 1 :].strip(),
+                "generated_tokens": generated_token_string,
+            }
+        return EDSLOutput(**edsl_dict)
 
     async def _async_get_intended_model_call_outcome(
         self,
@@ -377,7 +447,7 @@ class LanguageModel(
         cache: "Cache",
         iteration: int = 0,
         encoded_image=None,
-    ) -> IntendedModelCallOutcome:
+    ) -> ModelResponse:
         """Handle caching of responses.
 
         :param user_prompt: The user's prompt.
@@ -396,8 +466,7 @@ class LanguageModel(
         >>> from edsl import Cache
         >>> m = LanguageModel.example(test_model = True)
         >>> m._get_intended_model_call_outcome(user_prompt = "Hello", system_prompt = "hello", cache = Cache())
-        IntendedModelCallOutcome(response = {'message': '{"answer": "Hello world"}'}, cache_used = False, cache_key = '24ff6ac2bc2f1729f817f261e0792577')
-        """
+        ModelResponse(...)"""
 
         if encoded_image:
             # the image has is appended to the user_prompt for hash-lookup purposes
@@ -425,21 +494,28 @@ class LanguageModel(
                 "system_prompt": system_prompt,
                 **({"encoded_image": encoded_image} if encoded_image else {}),
             }
-            response = await f(**params)
+            # response = await f(**params)
+            response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
             new_cache_key = cache.store(
                 **cache_call_params, response=response
             )  # store the response in the cache
             assert new_cache_key == cache_key  # should be the same
 
-        return IntendedModelCallOutcome(
-            response=response, cache_used=cache_used, cache_key=cache_key
+        cost = self.cost(response)
+
+        return ModelResponse(
+            response=response,
+            cache_used=cache_used,
+            cache_key=cache_key,
+            cached_response=cached_response,
+            cost=cost,
         )
 
     _get_intended_model_call_outcome = sync_wrapper(
         _async_get_intended_model_call_outcome
     )
 
-    get_raw_response = sync_wrapper(async_get_raw_response)
+    # get_raw_response = sync_wrapper(async_get_raw_response)
 
     def simple_ask(
         self,
@@ -478,14 +554,66 @@ class LanguageModel(
             "cache": cache,
             **({"encoded_image": encoded_image} if encoded_image else {}),
         }
-        model_call_outcome = await self._async_get_intended_model_call_outcome(**params)
-        return await self._async_prepare_response(model_call_outcome, cache=cache)
+        model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
+        model_outputs = await self._async_get_intended_model_call_outcome(**params)
+        edsl_dict = self.parse_response(model_outputs.response)
+        agent_response_dict = AgentResponseDict(
+            model_inputs=model_inputs,
+            model_outputs=model_outputs,
+            edsl_dict=edsl_dict,
+        )
+        return agent_response_dict
+
+        # return await self._async_prepare_response(model_call_outcome, cache=cache)
 
     get_response = sync_wrapper(async_get_response)
 
-    def cost(self, raw_response: dict[str, Any]) -> float:
+    def cost(self, raw_response: dict[str, Any]) -> Union[float, str]:
         """Return the dollar cost of a raw response."""
-        raise NotImplementedError
+
+        usage = self.get_usage_dict(raw_response)
+        from edsl.coop import Coop
+
+        c = Coop()
+        price_lookup = c.fetch_prices()
+        key = (self._inference_service_, self.model)
+        if key not in price_lookup:
+            return f"Could not find price for model {self.model} in the price lookup."
+
+        relevant_prices = price_lookup[key]
+        try:
+            input_tokens = int(usage[self.input_token_name])
+            output_tokens = int(usage[self.output_token_name])
+        except Exception as e:
+            return f"Could not fetch tokens from model response: {e}"
+
+        try:
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+        except Exception as e:
+            if "output" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'output' key."
+            if "input" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'input' key."
+            return f"Could not fetch prices from {relevant_prices} - {e}"
+
+        if inverse_input_price == "infinity":
+            input_cost = 0
+        else:
+            try:
+                input_cost = input_tokens / float(inverse_input_price)
+            except Exception as e:
+                return f"Could not compute input price - {e}."
+
+        if inverse_output_price == "infinity":
+            output_cost = 0
+        else:
+            try:
+                output_cost = output_tokens / float(inverse_output_price)
+            except Exception as e:
+                return f"Could not compute output price - {e}"
+
+        return input_cost + output_cost
 
     #######################
     # SERIALIZATION METHODS
@@ -499,7 +627,7 @@ class LanguageModel(
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
-        {'model': 'gpt-4-1106-preview', 'parameters': {'temperature': 0.5, 'max_tokens': 1000, 'top_p': 1, 'frequency_penalty': 0, 'presence_penalty': 0, 'logprobs': False, 'top_logprobs': 3}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
+        {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
         return self._to_dict()
 
@@ -575,26 +703,8 @@ class LanguageModel(
         """
         from edsl import Model
 
-        class TestLanguageModelGood(LanguageModel):
-            use_cache = False
-            _model_ = "test"
-            _parameters_ = {"temperature": 0.5}
-            _inference_service_ = InferenceServiceType.TEST.value
-
-            async def async_execute_model_call(
-                self, user_prompt: str, system_prompt: str
-            ) -> dict[str, Any]:
-                await asyncio.sleep(0.1)
-                # return {"message": """{"answer": "Hello, world"}"""}
-                if throw_exception:
-                    raise Exception("This is a test error")
-                return {"message": f'{{"answer": "{canned_response}"}}'}
-
-            def parse_response(self, raw_response: dict[str, Any]) -> str:
-                return raw_response["message"]
-
         if test_model:
-            m = TestLanguageModelGood()
+            m = Model("test", canned_response=canned_response)
             return m
         else:
             return Model(skip_api_key_check=True)

edsl/language_models/ModelList.py

@@ -40,8 +40,8 @@ class ModelList(Base, UserList):
     def __hash__(self):
         """Return a hash of the ModelList. This is used for comparison of ModelLists.
 
-        >>> hash(ModelList.example())
-        1423518243781418961
+        >>> isinstance(hash(Model()), int)
+        True
 
         """
         from edsl.utilities.utilities import dict_hash

edsl/language_models/RegisterLanguageModelsMeta.py

@@ -47,13 +47,6 @@ class RegisterLanguageModelsMeta(ABCMeta):
                 must_be_async=True,
             )
             # LanguageModel children have to implement the parse_response method
-            RegisterLanguageModelsMeta.verify_method(
-                candidate_class=cls,
-                method_name="parse_response",
-                expected_return_type=str,
-                required_parameters=[("raw_response", dict[str, Any])],
-                must_be_async=False,
-            )
             RegisterLanguageModelsMeta._registry[model_name] = cls
 
     @classmethod
@@ -98,7 +91,7 @@ class RegisterLanguageModelsMeta(ABCMeta):
 
         required_parameters = required_parameters or []
         method = getattr(candidate_class, method_name)
-        signature = inspect.signature(method)
+        # signature = inspect.signature(method)
 
         RegisterLanguageModelsMeta._check_return_type(method, expected_return_type)
 
@@ -106,11 +99,11 @@ class RegisterLanguageModelsMeta(ABCMeta):
             RegisterLanguageModelsMeta._check_is_coroutine(method)
 
         # Check the parameters
-        params = signature.parameters
-        for param_name, param_type in required_parameters:
-            RegisterLanguageModelsMeta._verify_parameter(
-                params, param_name, param_type, method_name
-            )
+        # params = signature.parameters
+        # for param_name, param_type in required_parameters:
+        #     RegisterLanguageModelsMeta._verify_parameter(
+        #         params, param_name, param_type, method_name
+        #     )
 
     @staticmethod
     def _check_method_defined(cls, method_name):
@@ -167,23 +160,15 @@ class RegisterLanguageModelsMeta(ABCMeta):
         Check if the return type of a method is as expected.
 
         Example:
-        >>> class M:
-        ...     async def f(self) -> str: pass
-        >>> RegisterLanguageModelsMeta._check_return_type(M.f, str)
-        >>> class N:
-        ...     async def f(self) -> int: pass
-        >>> RegisterLanguageModelsMeta._check_return_type(N.f, str)
-        Traceback (most recent call last):
-        ...
-        TypeError: Return type of f must be <class 'str'>. Got <class 'int'>.
         """
-        if inspect.isroutine(method):
-            # return_type = inspect.signature(method).return_annotation
-            return_type = get_type_hints(method)["return"]
-            if return_type != expected_return_type:
-                raise TypeError(
-                    f"Return type of {method.__name__} must be {expected_return_type}. Got {return_type}."
-                )
+        pass
+        # if inspect.isroutine(method):
+        #     # return_type = inspect.signature(method).return_annotation
+        #     return_type = get_type_hints(method)["return"]
+        #     if return_type != expected_return_type:
+        #         raise TypeError(
+        #             f"Return type of {method.__name__} must be {expected_return_type}. Got {return_type}."
+        #         )
 
     @classmethod
     def model_names_to_classes(cls):

edsl/language_models/fake_openai_call.py

@@ -0,0 +1,15 @@
+from openai import AsyncOpenAI
+import asyncio
+
+client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
+
+
+async def main():
+    response = await client.chat.completions.create(
+        model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Question XX42"}]
+    )
+    print(response)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

edsl/language_models/fake_openai_service.py

@@ -0,0 +1,61 @@
+import threading
+import asyncio
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+import uvicorn
+import json
+from typing import Any
+
+app = FastAPI()
+
+
+async def generate_response(question_number: int) -> dict:
+    # Simulate some asynchronous work
+    await asyncio.sleep(1)
+    return {
+        "id": "chatcmpl-123",
+        "object": "chat.completion",
+        "created": 1677652288,
+        "model": "gpt-3.5-turbo-0613",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": json.dumps(
+                        {"answer": f"SPAM for question {question_number}!"}
+                    ),
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21},
+    }
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    body = await request.json()
+    user_prompt = body["messages"][-1]["content"]
+    question_number = int(user_prompt.split("XX")[1])
+
+    response = await generate_response(question_number)
+    return JSONResponse(content=response)
+
+
+def run_server():
+    uvicorn.run(app, host="127.0.0.1", port=8000)
+
+
+if __name__ == "__main__":
+    # Start the server in a separate thread
+    server_thread = threading.Thread(target=run_server)
+    server_thread.start()
+
+# Your main code here
+# ...
+
+# To use this with the OpenAI SDK:
+# from openai import AsyncOpenAI
+# client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
+# response = await client.chat.completions.create(model="gpt-3.5-turbo", messages=[...])