edsl 0.1.39.dev3__py3-none-any.whl → 0.1.39.dev5__py3-none-any.whl

edsl/language_models/ModelList.py

@@ -1,155 +1,164 @@
-from typing import Optional, List
-from collections import UserList
-from edsl import Model
-
-from edsl.language_models import LanguageModel
-from edsl.Base import Base
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-from edsl.utilities.utilities import is_valid_variable_name
-from edsl.utilities.utilities import dict_hash
-
-
-class ModelList(Base, UserList):
-    __documentation__ = """https://docs.expectedparrot.com/en/latest/language_models.html#module-edsl.language_models.ModelList"""
-
-    def __init__(self, data: Optional[list] = None):
-        """Initialize the ScenarioList class.
-
-        >>> from edsl import Model
-        >>> m = ModelList(Model.available())
-
-        """
-        if data is not None:
-            super().__init__(data)
-        else:
-            super().__init__([])
-
-    @property
-    def names(self):
-        """
-
-        >>> ModelList.example().names
-        {'...'}
-        """
-        return set([model.model for model in self])
-
-    def rich_print(self):
-        pass
-
-    def __repr__(self):
-        return f"ModelList({super().__repr__()})"
-
-    def _summary(self):
-        return {"EDSL Class": "ModelList", "Number of Models": len(self)}
-
-    def __hash__(self):
-        """Return a hash of the ModelList. This is used for comparison of ModelLists.
-
-        >>> isinstance(hash(Model()), int)
-        True
-
-        """
-        from edsl.utilities.utilities import dict_hash
-
-        return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
-
-    def to_scenario_list(self):
-        from edsl import ScenarioList, Scenario
-
-        sl = ScenarioList()
-        for model in self:
-            d = {"model": model.model}
-            d.update(model.parameters)
-            sl.append(Scenario(d))
-        return sl
-
-    def tree(self, node_list: Optional[List[str]] = None):
-        return self.to_scenario_list().tree(node_list)
-
-    def table(
-        self,
-        *fields,
-        tablefmt: Optional[str] = None,
-        pretty_labels: Optional[dict] = None,
-    ):
-        """
-        >>> ModelList.example().table("model")
-        model
-        -------
-        gpt-4o
-        gpt-4o
-        gpt-4o
-        """
-        return (
-            self.to_scenario_list()
-            .to_dataset()
-            .table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
-        )
-
-    def to_list(self):
-        return self.to_scenario_list().to_list()
-
-    def to_dict(self, sort=False, add_edsl_version=True):
-        if sort:
-            model_list = sorted([model for model in self], key=lambda x: hash(x))
-            d = {
-                "models": [
-                    model.to_dict(add_edsl_version=add_edsl_version)
-                    for model in model_list
-                ]
-            }
-        else:
-            d = {
-                "models": [
-                    model.to_dict(add_edsl_version=add_edsl_version) for model in self
-                ]
-            }
-        if add_edsl_version:
-            from edsl import __version__
-
-            d["edsl_version"] = __version__
-            d["edsl_class_name"] = "ModelList"
-
-        return d
-
-    def _repr_html_(self):
-        """Return an HTML representation of the ModelList."""
-        footer = f"<a href={self.__documentation__}>(docs)</a>"
-        return str(self.summary(format="html")) + footer
-
-    @classmethod
-    def from_names(self, *args, **kwargs):
-        """A a model list from a list of names"""
-        if len(args) == 1 and isinstance(args[0], list):
-            args = args[0]
-        return ModelList([Model(model_name, **kwargs) for model_name in args])
-
-    @classmethod
-    @remove_edsl_version
-    def from_dict(cls, data):
-        """
-        Create a ModelList from a dictionary.
-
-        >>> newm = ModelList.from_dict(ModelList.example().to_dict())
-        >>> assert ModelList.example() == newm
-        """
-        return cls(data=[LanguageModel.from_dict(model) for model in data["models"]])
-
-    def code(self):
-        pass
-
-    @classmethod
-    def example(cls, randomize: bool = False) -> "ModelList":
-        """
-        Returns an example ModelList instance.
-
-        :param randomize: If True, uses Model's randomize method.
-        """
-
-        return cls([Model.example(randomize) for _ in range(3)])
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
+from typing import Optional, List, TYPE_CHECKING
+from collections import UserList
+
+from edsl.Base import Base
+from edsl.language_models.model import Model
+
+#
+from edsl.utilities.remove_edsl_version import remove_edsl_version
+from edsl.utilities.is_valid_variable_name import is_valid_variable_name
+
+if TYPE_CHECKING:
+    from edsl.inference_services.data_structures import AvailableModels
+    from edsl.language_models import LanguageModel
+
+
+class ModelList(Base, UserList):
+    __documentation__ = """https://docs.expectedparrot.com/en/latest/language_models.html#module-edsl.language_models.ModelList"""
+
+    def __init__(self, data: Optional["LanguageModel"] = None):
+        """Initialize the ScenarioList class.
+
+        >>> from edsl import Model
+        >>> m = ModelList(Model.available())
+
+        """
+        if data is not None:
+            super().__init__(data)
+        else:
+            super().__init__([])
+
+    @property
+    def names(self):
+        """
+
+        >>> ModelList.example().names
+        {'...'}
+        """
+        return set([model.model for model in self])
+
+    def __repr__(self):
+        return f"ModelList({super().__repr__()})"
+
+    def _summary(self):
+        return {"models": len(self)}
+
+    def __hash__(self):
+        """Return a hash of the ModelList. This is used for comparison of ModelLists.
+
+        >>> isinstance(hash(Model()), int)
+        True
+
+        """
+        from edsl.utilities.utilities import dict_hash
+
+        return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
+
+    def to_scenario_list(self):
+        from edsl.scenarios.ScenarioList import ScenarioList
+        from edsl.scenarios.Scenario import Scenario
+
+        sl = ScenarioList()
+        for model in self:
+            d = {"model": model.model}
+            d.update(model.parameters)
+            sl.append(Scenario(d))
+        return sl
+
+    def tree(self, node_list: Optional[List[str]] = None):
+        return self.to_scenario_list().tree(node_list)
+
+    def table(
+        self,
+        *fields,
+        tablefmt: Optional[str] = None,
+        pretty_labels: Optional[dict] = None,
+    ):
+        """
+        >>> ModelList.example().table('model')
+        model
+        -------
+        gpt-4o
+        gpt-4o
+        gpt-4o
+        """
+        return (
+            self.to_scenario_list()
+            .to_dataset()
+            .table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
+        )
+
+    def to_list(self) -> list:
+        return self.to_scenario_list().to_list()
+
+    def to_dict(self, sort=False, add_edsl_version=True):
+        if sort:
+            model_list = sorted([model for model in self], key=lambda x: hash(x))
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version)
+                    for model in model_list
+                ]
+            }
+        else:
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version) for model in self
+                ]
+            }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "ModelList"
+
+        return d
+
+    @classmethod
+    def from_names(self, *args, **kwargs):
+        """A a model list from a list of names"""
+        if len(args) == 1 and isinstance(args[0], list):
+            args = args[0]
+        return ModelList([Model(model_name, **kwargs) for model_name in args])
+
+    @classmethod
+    def from_available_models(self, available_models_list: "AvailableModels"):
+        """Create a ModelList from an AvailableModels object"""
+        return ModelList(
+            [
+                Model(model.model_name, service_name=model.service_name)
+                for model in available_models_list
+            ]
+        )
+
+    @classmethod
+    @remove_edsl_version
+    def from_dict(cls, data):
+        """
+        Create a ModelList from a dictionary.
+
+        >>> newm = ModelList.from_dict(ModelList.example().to_dict())
+        >>> assert ModelList.example() == newm
+        """
+        from edsl.language_models.LanguageModel import LanguageModel
+
+        return cls(data=[LanguageModel.from_dict(model) for model in data["models"]])
+
+    def code(self):
+        pass
+
+    @classmethod
+    def example(cls, randomize: bool = False) -> "ModelList":
+        """
+        Returns an example ModelList instance.
+
+        :param randomize: If True, uses Model's randomize method.
+        """
+
+        return cls([Model.example(randomize) for _ in range(3)])
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)

edsl/language_models/PriceManager.py

@@ -0,0 +1,127 @@
+from typing import Dict, Tuple, Optional, Union
+
+
+class PriceManager:
+    _instance = None
+    _price_lookup: Dict[Tuple[str, str], Dict] = {}
+    _is_initialized = False
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super(PriceManager, cls).__new__(cls)
+        return cls._instance
+
+    def __init__(self):
+        # Only initialize once, even if __init__ is called multiple times
+        if not self._is_initialized:
+            self._is_initialized = True
+            self.refresh_prices()
+
+    def refresh_prices(self) -> None:
+        """
+        Fetch fresh prices from the Coop service and update the internal price lookup.
+
+        """
+        from edsl.coop import Coop
+
+        c = Coop()
+        try:
+            self._price_lookup = c.fetch_prices()
+        except Exception as e:
+            print(f"Error fetching prices: {str(e)}")
+
+    def get_price(self, inference_service: str, model: str) -> Optional[Dict]:
+        """
+        Get the price information for a specific service and model combination.
+
+        Args:
+            inference_service (str): The name of the inference service
+            model (str): The model identifier
+
+        Returns:
+            Optional[Dict]: Price information if found, None otherwise
+        """
+        key = (inference_service, model)
+        return self._price_lookup.get(key)
+
+    def get_all_prices(self) -> Dict[Tuple[str, str], Dict]:
+        """
+        Get the complete price lookup dictionary.
+
+        Returns:
+            Dict[Tuple[str, str], Dict]: The complete price lookup dictionary
+        """
+        return self._price_lookup.copy()
+
+    def calculate_cost(
+        self,
+        inference_service: str,
+        model: str,
+        usage: Dict[str, Union[str, int]],
+        input_token_name: str,
+        output_token_name: str,
+    ) -> Union[float, str]:
+        """
+        Calculate the total cost for a model usage based on input and output tokens.
+
+        Args:
+            inference_service (str): The inference service identifier
+            model (str): The model identifier
+            usage (Dict[str, Union[str, int]]): Dictionary containing token usage information
+            input_token_name (str): Key name for input tokens in the usage dict
+            output_token_name (str): Key name for output tokens in the usage dict
+
+        Returns:
+            Union[float, str]: Total cost if calculation successful, error message string if not
+        """
+        relevant_prices = self.get_price(inference_service, model)
+        if relevant_prices is None:
+            return f"Could not find price for model {model} in the price lookup."
+
+        # Extract token counts
+        try:
+            input_tokens = int(usage[input_token_name])
+            output_tokens = int(usage[output_token_name])
+        except Exception as e:
+            return f"Could not fetch tokens from model response: {e}"
+
+        # Extract price information
+        try:
+            inverse_output_price = relevant_prices["output"]["one_usd_buys"]
+            inverse_input_price = relevant_prices["input"]["one_usd_buys"]
+        except Exception as e:
+            if "output" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'output' key."
+            if "input" not in relevant_prices:
+                return f"Could not fetch prices from {relevant_prices} - {e}; Missing 'input' key."
+            return f"Could not fetch prices from {relevant_prices} - {e}"
+
+        # Calculate input cost
+        if inverse_input_price == "infinity":
+            input_cost = 0
+        else:
+            try:
+                input_cost = input_tokens / float(inverse_input_price)
+            except Exception as e:
+                return f"Could not compute input price - {e}."
+
+        # Calculate output cost
+        if inverse_output_price == "infinity":
+            output_cost = 0
+        else:
+            try:
+                output_cost = output_tokens / float(inverse_output_price)
+            except Exception as e:
+                return f"Could not compute output price - {e}"
+
+        return input_cost + output_cost
+
+    @property
+    def is_initialized(self) -> bool:
+        """
+        Check if the PriceManager has been initialized.
+
+        Returns:
+            bool: True if initialized, False otherwise
+        """
+        return self._is_initialized

edsl/language_models/RawResponseHandler.py

@@ -0,0 +1,106 @@
+import json
+from typing import Optional, Any, List
+from edsl.exceptions.language_models import LanguageModelBadResponseError
+
+from json_repair import repair_json
+
+
+def _extract_item_from_raw_response(data, sequence):
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except json.JSONDecodeError as e:
+            return data
+    current_data = data
+    for i, key in enumerate(sequence):
+        try:
+            if isinstance(current_data, (list, tuple)):
+                if not isinstance(key, int):
+                    raise TypeError(
+                        f"Expected integer index for sequence at position {i}, got {type(key).__name__}"
+                    )
+                if key < 0 or key >= len(current_data):
+                    raise IndexError(
+                        f"Index {key} out of range for sequence of length {len(current_data)} at position {i}"
+                    )
+            elif isinstance(current_data, dict):
+                if key not in current_data:
+                    raise KeyError(
+                        f"Key '{key}' not found in dictionary at position {i}"
+                    )
+            else:
+                raise TypeError(
+                    f"Cannot index into {type(current_data).__name__} at position {i}. Full response is: {data} of type {type(data)}. Key sequence is: {sequence}"
+                )
+
+            current_data = current_data[key]
+        except Exception as e:
+            path = " -> ".join(map(str, sequence[: i + 1]))
+            if "error" in data:
+                msg = data["error"]
+            else:
+                msg = f"Error accessing path: {path}. {str(e)}. Full response is: '{data}'"
+            raise LanguageModelBadResponseError(message=msg, response_json=data)
+    if isinstance(current_data, str):
+        return current_data.strip()
+    else:
+        return current_data
+
+
+class RawResponseHandler:
+    """Class to handle raw responses from language models."""
+
+    def __init__(self, key_sequence: list, usage_sequence: Optional[list] = None):
+        self.key_sequence = key_sequence
+        self.usage_sequence = usage_sequence
+
+    def get_generated_token_string(self, raw_response):
+        return _extract_item_from_raw_response(raw_response, self.key_sequence)
+
+    def get_usage_dict(self, raw_response):
+        if self.usage_sequence is None:
+            return {}
+        return _extract_item_from_raw_response(raw_response, self.usage_sequence)
+
+    def parse_response(self, raw_response: dict[str, Any]) -> "EDSLOutput":
+        """Parses the API response and returns the response text."""
+
+        from edsl.data_transfer_models import EDSLOutput
+
+        generated_token_string = self.get_generated_token_string(raw_response)
+        last_newline = generated_token_string.rfind("\n")
+
+        if last_newline == -1:
+            # There is no comment
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string),
+                "generated_tokens": generated_token_string,
+                "comment": None,
+            }
+        else:
+            edsl_dict = {
+                "answer": self.convert_answer(generated_token_string[:last_newline]),
+                "comment": generated_token_string[last_newline + 1 :].strip(),
+                "generated_tokens": generated_token_string,
+            }
+        return EDSLOutput(**edsl_dict)
+
+    @staticmethod
+    def convert_answer(response_part):
+        import json
+
+        response_part = response_part.strip()
+
+        if response_part == "None":
+            return None
+
+        repaired = repair_json(response_part)
+        if repaired == '""':
+            # it was a literal string
+            return response_part
+
+        try:
+            return json.loads(repaired)
+        except json.JSONDecodeError as j:
+            # last resort
+            return response_part