edsl 0.1.15__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/language_models/ModelList.py

@@ -0,0 +1,164 @@
+from typing import Optional, List, TYPE_CHECKING
+from collections import UserList
+
+from edsl.Base import Base
+from edsl.language_models.model import Model
+
+#
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+from edsl.utilities.is_valid_variable_name import is_valid_variable_name
+
+if TYPE_CHECKING:
+    from edsl.inference_services.data_structures import AvailableModels
+    from edsl.language_models import LanguageModel
+
+
+class ModelList(Base, UserList):
+    __documentation__ = """https://docs.expectedparrot.com/en/latest/language_models.html#module-edsl.language_models.ModelList"""
+
+    def __init__(self, data: Optional["LanguageModel"] = None):
+        """Initialize the ScenarioList class.
+
+        >>> from edsl import Model
+        >>> m = ModelList(Model.available())
+
+        """
+        if data is not None:
+            super().__init__(data)
+        else:
+            super().__init__([])
+
+    @property
+    def names(self):
+        """
+
+        >>> ModelList.example().names
+        {'...'}
+        """
+        return set([model.model for model in self])
+
+    def __repr__(self):
+        return f"ModelList({super().__repr__()})"
+
+    def _summary(self):
+        return {"models": len(self)}
+
+    def __hash__(self):
+        """Return a hash of the ModelList. This is used for comparison of ModelLists.
+
+        >>> isinstance(hash(Model()), int)
+        True
+
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
+
+    def to_scenario_list(self):
+        from edsl.scenarios.ScenarioList import ScenarioList
+        from edsl.scenarios.Scenario import Scenario
+
+        sl = ScenarioList()
+        for model in self:
+            d = {"model": model.model}
+            d.update(model.parameters)
+            sl.append(Scenario(d))
+        return sl
+
+    def tree(self, node_list: Optional[List[str]] = None):
+        return self.to_scenario_list().tree(node_list)
+
+    def table(
+        self,
+        *fields,
+        tablefmt: Optional[str] = None,
+        pretty_labels: Optional[dict] = None,
+    ):
+        """
+        >>> ModelList.example().table('model')
+        model
+        -------
+        gpt-4o
+        gpt-4o
+        gpt-4o
+        """
+        return (
+            self.to_scenario_list()
+            .to_dataset()
+            .table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
+        )
+
+    def to_list(self) -> list:
+        return self.to_scenario_list().to_list()
+
+    def to_dict(self, sort=False, add_edsl_version=True):
+        if sort:
+            model_list = sorted([model for model in self], key=lambda x: hash(x))
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version)
+                    for model in model_list
+                ]
+            }
+        else:
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version) for model in self
+                ]
+            }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "ModelList"
+
+        return d
+
+    @classmethod
+    def from_names(self, *args, **kwargs):
+        """A a model list from a list of names"""
+        if len(args) == 1 and isinstance(args[0], list):
+            args = args[0]
+        return ModelList([Model(model_name, **kwargs) for model_name in args])
+
+    @classmethod
+    def from_available_models(self, available_models_list: "AvailableModels"):
+        """Create a ModelList from an AvailableModels object"""
+        return ModelList(
+            [
+                Model(model.model_name, service_name=model.service_name)
+                for model in available_models_list
+            ]
+        )
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        """
+        Create a ModelList from a dictionary.
+
+        >>> newm = ModelList.from_dict(ModelList.example().to_dict())
+        >>> assert ModelList.example() == newm
+        """
+        from edsl.language_models.LanguageModel import LanguageModel
+
+        return cls(data=[LanguageModel.from_dict(model) for model in data["models"]])
+
+    def code(self):
+        pass
+
+    @classmethod
+    def example(cls, randomize: bool = False) -> "ModelList":
+        """
+        Returns an example ModelList instance.
+
+        :param randomize: If True, uses Model's randomize method.
+        """
+
+        return cls([Model.example(randomize) for _ in range(3)])
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/PriceManager.py

@@ -0,0 +1,127 @@
+from typing import Dict, Tuple, Optional, Union
+
+
+class PriceManager:
+    _instance = None
+    _price_lookup: Dict[Tuple[str, str], Dict] = {}
+    _is_initialized = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(PriceManager, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        # Only initialize once, even if __init__ is called multiple times
+        if not self._is_initialized:
+            self._is_initialized = True
+            self.refresh_prices()
+
+    def refresh_prices(self) -> None:
+        """
+        Fetch fresh prices from the Coop service and update the internal price lookup.
+
+        """
+        from edsl.coop import Coop
+
+        c = Coop()
+        try:
+            self._price_lookup = c.fetch_prices()
+        except Exception as e:
+            print(f"Error fetching prices: {str(e)}")
+
+    def get_price(self, inference_service: str, model: str) -> Optional[Dict]:
+        """
+        Get the price information for a specific service and model combination.
+
+        Args:
+            inference_service (str): The name of the inference service
+            model (str): The model identifier
+
+        Returns:
+            Optional[Dict]: Price information if found, None otherwise
+        """
+        key = (inference_service, model)
+        return self._price_lookup.get(key)
+
+    def get_all_prices(self) -> Dict[Tuple[str, str], Dict]:
+        """
+        Get the complete price lookup dictionary.
+
+        Returns:
+            Dict[Tuple[str, str], Dict]: The complete price lookup dictionary
+        """
+        return self._price_lookup.copy()
+
+    def calculate_cost(
+        self,
+        inference_service: str,
+        model: str,
+        usage: Dict[str, Union[str, int]],
+        input_token_name: str,
+        output_token_name: str,
+    ) -> Union[float, str]:
+        """
+        Calculate the total cost for a model usage based on input and output tokens.
+
+        Args:
+            inference_service (str): The inference service identifier
+            model (str): The model identifier
+            usage (Dict[str, Union[str, int]]): Dictionary containing token usage information
+            input_token_name (str): Key name for input tokens in the usage dict
+            output_token_name (str): Key name for output tokens in the usage dict
+
+        Returns:
+            Union[float, str]: Total cost if calculation successful, error message string if not
+        """
+        relevant_prices = self.get_price(inference_service, model)
+        if relevant_prices is None:
+            return f"Could not find price for model {model} in the price lookup."
+
+        # Extract token counts
+        try:
+            input_tokens = int(usage[input_token_name])
+            output_tokens = int(usage[output_token_name])
+        except Exception as e:
+            return f"Could not fetch tokens from model response: {e}"
+
+        # Extract price information
+        try:
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+        except Exception as e:
+            if "output" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'output' key."
+            if "input" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'input' key."
+            return f"Could not fetch prices from {relevant_prices} - {e}"
+
+        # Calculate input cost
+        if inverse_input_price == "infinity":
+            input_cost = 0
+        else:
+            try:
+                input_cost = input_tokens / float(inverse_input_price)
+            except Exception as e:
+                return f"Could not compute input price - {e}."
+
+        # Calculate output cost
+        if inverse_output_price == "infinity":
+            output_cost = 0
+        else:
+            try:
+                output_cost = output_tokens / float(inverse_output_price)
+            except Exception as e:
+                return f"Could not compute output price - {e}"
+
+        return input_cost + output_cost
+
+    @property
+    def is_initialized(self) -> bool:
+        """
+        Check if the PriceManager has been initialized.
+
+        Returns:
+            bool: True if initialized, False otherwise
+        """
+        return self._is_initialized

edsl/language_models/RawResponseHandler.py

@@ -0,0 +1,106 @@
+import json
+from typing import Optional, Any, List
+from edsl.exceptions.language_models import LanguageModelBadResponseError
+
+from json_repair import repair_json
+
+
+def _extract_item_from_raw_response(data, sequence):
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except json.JSONDecodeError as e:
+            return data
+    current_data = data
+    for i, key in enumerate(sequence):
+        try:
+            if isinstance(current_data, (list, tuple)):
+                if not isinstance(key, int):
+                    raise TypeError(
+                        f"Expected integer index for sequence at position {i}, got {type(key).__name__}"
+                    )
+                if key < 0 or key >= len(current_data):
+                    raise IndexError(
+                        f"Index {key} out of range for sequence of length {len(current_data)} at position {i}"
+                    )
+            elif isinstance(current_data, dict):
+                if key not in current_data:
+                    raise KeyError(
+                        f"Key '{key}' not found in dictionary at position {i}"
+                    )
+            else:
+                raise TypeError(
+                    f"Cannot index into {type(current_data).__name__} at position {i}. Full response is: {data} of type {type(data)}. Key sequence is: {sequence}"
+                )
+
+            current_data = current_data[key]
+        except Exception as e:
+            path = " -> ".join(map(str, sequence[: i + 1]))
+            if "error" in data:
+                msg = data["error"]
+            else:
+                msg = f"Error accessing path: {path}. {str(e)}. Full response is: '{data}'"
+            raise LanguageModelBadResponseError(message=msg, response_json=data)
+    if isinstance(current_data, str):
+        return current_data.strip()
+    else:
+        return current_data
+
+
+class RawResponseHandler:
+    """Class to handle raw responses from language models."""
+
+    def __init__(self, key_sequence: list, usage_sequence: Optional[list] = None):
+        self.key_sequence = key_sequence
+        self.usage_sequence = usage_sequence
+
+    def get_generated_token_string(self, raw_response):
+        return _extract_item_from_raw_response(raw_response, self.key_sequence)
+
+    def get_usage_dict(self, raw_response):
+        if self.usage_sequence is None:
+            return {}
+        return _extract_item_from_raw_response(raw_response, self.usage_sequence)
+
+    def parse_response(self, raw_response: dict[str, Any]) -> "EDSLOutput":
+        """Parses the API response and returns the response text."""
+
+        from edsl.data_transfer_models import EDSLOutput
+
+        generated_token_string = self.get_generated_token_string(raw_response)
+        last_newline = generated_token_string.rfind("\n")
+
+        if last_newline == -1:
+            # There is no comment
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string),
+                "generated_tokens": generated_token_string,
+                "comment": None,
+            }
+        else:
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string[:last_newline]),
+                "comment": generated_token_string[last_newline + 1 :].strip(),
+                "generated_tokens": generated_token_string,
+            }
+        return EDSLOutput(**edsl_dict)
+
+    @staticmethod
+    def convert_answer(response_part):
+        import json
+
+        response_part = response_part.strip()
+
+        if response_part == "None":
+            return None
+
+        repaired = repair_json(response_part)
+        if repaired == '""':
+            # it was a literal string
+            return response_part
+
+        try:
+            return json.loads(repaired)
+        except json.JSONDecodeError as j:
+            # last resort
+            return response_part

edsl/language_models/RegisterLanguageModelsMeta.py

@@ -0,0 +1,184 @@
+from abc import ABC, ABCMeta
+from typing import Any, List, Callable
+import inspect
+from typing import get_type_hints
+from edsl.exceptions.language_models import LanguageModelAttributeTypeError
+from edsl.enums import InferenceServiceType
+
+
+class RegisterLanguageModelsMeta(ABCMeta):
+    """Metaclass to register output elements in a registry i.e., those that have a parent."""
+
+    _registry = {}  # Initialize the registry as a dictionary
+    REQUIRED_CLASS_ATTRIBUTES = ["_model_", "_parameters_", "_inference_service_"]
+
+    def __init__(cls, name, bases, dct):
+        """Register the class in the registry if it has a _model_ attribute."""
+        super(RegisterLanguageModelsMeta, cls).__init__(name, bases, dct)
+        # if name != "LanguageModel":
+        if (model_name := getattr(cls, "_model_", None)) is not None:
+            RegisterLanguageModelsMeta.check_required_class_variables(
+                cls, RegisterLanguageModelsMeta.REQUIRED_CLASS_ATTRIBUTES
+            )
+
+            ## Check that model name is valid
+            # if not LanguageModelType.is_value_valid(model_name):
+            #     acceptable_values = [item.value for item in LanguageModelType]
+            #     raise LanguageModelAttributeTypeError(
+            #         f"""A LanguageModel's model must be one of {LanguageModelType} values, which are
+            #         {acceptable_values}. You passed {model_name}."""
+            #     )
+
+            if not InferenceServiceType.is_value_valid(
+                inference_service := getattr(cls, "_inference_service_", None)
+            ):
+                acceptable_values = [item.value for item in InferenceServiceType]
+                raise LanguageModelAttributeTypeError(
+                    f"""A LanguageModel's model must have an _inference_service_ value from 
+                    {acceptable_values}. You passed {inference_service}."""
+                )
+
+            # LanguageModel children have to implement the async_execute_model_call method
+            RegisterLanguageModelsMeta.verify_method(
+                candidate_class=cls,
+                method_name="async_execute_model_call",
+                expected_return_type=dict[str, Any],
+                required_parameters=[("user_prompt", str), ("system_prompt", str)],
+                must_be_async=True,
+            )
+            # LanguageModel children have to implement the parse_response method
+            RegisterLanguageModelsMeta._registry[model_name] = cls
+
+    @classmethod
+    def get_registered_classes(cls):
+        """Return the registry."""
+        return cls._registry
+
+    @staticmethod
+    def check_required_class_variables(
+        candidate_class: "LanguageModel", required_attributes: List[str] = None
+    ):
+        """Check if a class has the required attributes.
+
+        >>> class M:
+        ...     _model_ = "m"
+        ...     _parameters_ = {}
+        >>> RegisterLanguageModelsMeta.check_required_class_variables(M, ["_model_", "_parameters_"])
+        >>> class M2:
+        ...     _model_ = "m"
+        >>> RegisterLanguageModelsMeta.check_required_class_variables(M2, ["_model_", "_parameters_"])
+        Traceback (most recent call last):
+        ...
+        Exception: Class M2 does not have required attribute _parameters_
+        """
+        required_attributes = required_attributes or []
+        for attribute in required_attributes:
+            if not hasattr(candidate_class, attribute):
+                raise Exception(
+                    f"Class {candidate_class.__name__} does not have required attribute {attribute}"
+                )
+
+    @staticmethod
+    def verify_method(
+        candidate_class: "LanguageModel",
+        method_name: str,
+        expected_return_type: Any,
+        required_parameters: List[tuple[str, Any]] = None,
+        must_be_async: bool = False,
+    ):
+        """Verify that a method is defined in a class, has the correct return type, and has the correct parameters."""
+        RegisterLanguageModelsMeta._check_method_defined(candidate_class, method_name)
+
+        required_parameters = required_parameters or []
+        method = getattr(candidate_class, method_name)
+        # signature = inspect.signature(method)
+
+        RegisterLanguageModelsMeta._check_return_type(method, expected_return_type)
+
+        if must_be_async:
+            RegisterLanguageModelsMeta._check_is_coroutine(method)
+
+        # Check the parameters
+        # params = signature.parameters
+        # for param_name, param_type in required_parameters:
+        #     RegisterLanguageModelsMeta._verify_parameter(
+        #         params, param_name, param_type, method_name
+        #     )
+
+    @staticmethod
+    def _check_method_defined(cls, method_name):
+        """Check if a method is defined in a class.
+
+        Example:
+        >>> class M:
+        ...     def f(self): pass
+        >>> RegisterLanguageModelsMeta._check_method_defined(M, "f")
+        >>> RegisterLanguageModelsMeta._check_method_defined(M, "g")
+        Traceback (most recent call last):
+        ...
+        NotImplementedError: g method must be implemented.
+        """
+        if not hasattr(cls, method_name):
+            raise NotImplementedError(f"{method_name} method must be implemented.")
+
+    @staticmethod
+    def _check_is_coroutine(func: Callable):
+        """Check to make sure it's a coroutine function.
+
+        Example:
+
+        >>> def f(): pass
+        >>> RegisterLanguageModelsMeta._check_is_coroutine(f)
+        Traceback (most recent call last):
+        ...
+        TypeError: A LangugeModel class with method f must be an asynchronous method.
+        """
+        if not inspect.iscoroutinefunction(func):
+            raise TypeError(
+                f"A LangugeModel class with method {func.__name__} must be an asynchronous method."
+            )
+
+    @staticmethod
+    def _verify_parameter(params, param_name, param_type, method_name):
+        """Verify that a parameter is defined in a method and has the correct type."""
+        pass
+        # if param_name not in params:
+        #     raise TypeError(
+        #         f"""Parameter "{param_name}" of method "{method_name}" must be defined.
+        #         """
+        #     )
+        # if params[param_name].annotation != param_type:
+        #     raise TypeError(
+        #         f"""Parameter "{param_name}" of method "{method_name}" must be of type {param_type.__name__}.
+        #         Got {params[param_name].annotation} instead.
+        #         """
+        #     )
+
+    @staticmethod
+    def _check_return_type(method, expected_return_type):
+        """
+        Check if the return type of a method is as expected.
+
+        Example:
+        """
+        pass
+        # if inspect.isroutine(method):
+        #     # return_type = inspect.signature(method).return_annotation
+        #     return_type = get_type_hints(method)["return"]
+        #     if return_type != expected_return_type:
+        #         raise TypeError(
+        #             f"Return type of {method.__name__} must be {expected_return_type}. Got {return_type}."
+        #         )
+
+    @classmethod
+    def model_names_to_classes(cls):
+        """Return a dictionary of model names to classes."""
+        d = {}
+        for classname, cls in cls._registry.items():
+            if hasattr(cls, "_model_"):
+                d[cls._model_] = cls
+            else:
+                raise Exception(
+                    f"Class {classname} does not have a _model_ class attribute."
+                )
+        return d

edsl/language_models/__init__.py

@@ -1,9 +1,2 @@
-from edsl.language_models.schemas import model_prices
 from edsl.language_models.LanguageModel import LanguageModel
-from edsl.language_models.model_interfaces.LanguageModelOpenAIThreeFiveTurbo import (
-    LanguageModelOpenAIThreeFiveTurbo,
-)
-from edsl.language_models.model_interfaces.LanguageModelOpenAIFour import (
-    LanguageModelOpenAIFour,
-)
-from edsl.language_models.model_interfaces.GeminiPro import GeminiPro
+from edsl.language_models.model import Model

edsl/language_models/fake_openai_call.py

@@ -0,0 +1,15 @@
+from openai import AsyncOpenAI
+import asyncio
+
+client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
+
+
+async def main():
+    response = await client.chat.completions.create(
+        model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Question XX42"}]
+    )
+    print(response)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

edsl/language_models/fake_openai_service.py

@@ -0,0 +1,61 @@
+import threading
+import asyncio
+from fastapi import FastAPI, Request
+from fastapi.responses import JSONResponse
+import uvicorn
+import json
+from typing import Any
+
+app = FastAPI()
+
+
+async def generate_response(question_number: int) -> dict:
+    # Simulate some asynchronous work
+    await asyncio.sleep(1)
+    return {
+        "id": "chatcmpl-123",
+        "object": "chat.completion",
+        "created": 1677652288,
+        "model": "gpt-3.5-turbo-0613",
+        "choices": [
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": json.dumps(
+                        {"answer": f"SPAM for question {question_number}!"}
+                    ),
+                },
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 9, "completion_tokens": 12, "total_tokens": 21},
+    }
+
+
+@app.post("/v1/chat/completions")
+async def chat_completions(request: Request):
+    body = await request.json()
+    user_prompt = body["messages"][-1]["content"]
+    question_number = int(user_prompt.split("XX")[1])
+
+    response = await generate_response(question_number)
+    return JSONResponse(content=response)
+
+
+def run_server():
+    uvicorn.run(app, host="127.0.0.1", port=8000)
+
+
+if __name__ == "__main__":
+    # Start the server in a separate thread
+    server_thread = threading.Thread(target=run_server)
+    server_thread.start()
+
+# Your main code here
+# ...
+
+# To use this with the OpenAI SDK:
+# from openai import AsyncOpenAI
+# client = AsyncOpenAI(base_url="http://127.0.0.1:8000/v1", api_key="fake_key")
+# response = await client.chat.completions.create(model="gpt-3.5-turbo", messages=[...])