edsl 0.1.37.dev5__py3-none-any.whl → 0.1.38__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -41,17 +41,19 @@ from edsl.data_transfer_models import (
     AgentResponseDict,
 )
 
+if TYPE_CHECKING:
+    from edsl.data.Cache import Cache
+    from edsl.scenarios.FileStore import FileStore
+    from edsl.questions.QuestionBase import QuestionBase
 
 from edsl.config import CONFIG
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
-from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
-from edsl.language_models.repair import repair
-from edsl.enums import InferenceServiceType
-from edsl.Base import RichPrintingMixin, PersistenceMixin
-from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
-from edsl.exceptions.language_models import LanguageModelBadResponseError
+from edsl.utilities.decorators import remove_edsl_version
 
+from edsl.Base import PersistenceMixin
+from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
 from edsl.language_models.KeyLookup import KeyLookup
+from edsl.exceptions.language_models import LanguageModelBadResponseError
 
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
@@ -116,29 +118,11 @@ def handle_key_error(func):
 
 
 class LanguageModel(
-    RichPrintingMixin, PersistenceMixin, ABC, metaclass=RegisterLanguageModelsMeta
+    PersistenceMixin,
+    ABC,
+    metaclass=RegisterLanguageModelsMeta,
 ):
-    """ABC for LLM subclasses.
-
-    TODO:
-
-    1) Need better, more descriptive names for functions
-
-    get_model_response_no_cache  (currently called async_execute_model_call)
-
-    get_model_response (currently called async_get_raw_response; uses cache & adds tracking info)
-      Calls:
-        - async_execute_model_call
-        - _updated_model_response_with_tracking
-
-    get_answer (currently called async_get_response)
-        This parses out the answer block and does some error-handling.
-        Calls:
-            - async_get_raw_response
-            - parse_response
-
-
-    """
+    """ABC for Language Models."""
 
     _model_ = None
     key_sequence = (
@@ -196,7 +180,7 @@ class LanguageModel(
         system_prompt = "You are a helpful agent pretending to be a human."
         return self.execute_model_call(user_prompt, system_prompt)
 
-    def set_key_lookup(self, key_lookup: KeyLookup):
+    def set_key_lookup(self, key_lookup: KeyLookup) -> None:
         del self._api_token
         self.key_lookup = key_lookup
 
@@ -211,10 +195,14 @@ class LanguageModel(
     def __getitem__(self, key):
         return getattr(self, key)
 
-    def _repr_html_(self):
-        from edsl.utilities.utilities import data_to_html
+    def _repr_html_(self) -> str:
+        d = {"model": self.model}
+        d.update(self.parameters)
+        data = [[k, v] for k, v in d.items()]
+        from tabulate import tabulate
 
-        return data_to_html(self.to_dict())
+        table = str(tabulate(data, headers=["keys", "values"], tablefmt="html"))
+        return f"<pre>{table}</pre>"
 
     def hello(self, verbose=False):
         """Runs a simple test to check if the model is working."""
@@ -235,7 +223,6 @@ class LanguageModel(
         This method is used to check if the model has a valid API key.
         """
         from edsl.enums import service_to_api_keyname
-        import os
 
         if self._model_ == "test":
             return True
@@ -248,9 +235,9 @@ class LanguageModel(
         """Allow the model to be used as a key in a dictionary."""
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self.to_dict())
+        return dict_hash(self.to_dict(add_edsl_version=False))
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
         """Check is two models are the same.
 
         >>> m1 = LanguageModel.example()
@@ -278,15 +265,11 @@ class LanguageModel(
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
-        # self._set_rate_limits()
-        # return self._safety_factor * self.__rate_limits["rpm"]
         return self._rpm
 
     @property
     def TPM(self):
         """Model's tokens-per-minute limit."""
-        # self._set_rate_limits()
-        # return self._safety_factor * self.__rate_limits["tpm"]
         return self._tpm
 
     @property
@@ -314,8 +297,6 @@ class LanguageModel(
         >>> LanguageModel._overide_default_parameters(passed_parameter_dict={"temperature": 0.5}, default_parameter_dict={"temperature":0.9, "max_tokens": 1000})
         {'temperature': 0.5, 'max_tokens': 1000}
         """
-        # parameters = dict({})
-
         # this is the case when data is loaded from a dict after serialization
         if "parameters" in passed_parameter_dict:
             passed_parameter_dict = passed_parameter_dict["parameters"]
@@ -429,9 +410,10 @@ class LanguageModel(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache: "Cache",
+        cache: Cache,
         iteration: int = 0,
-        files_list=None,
+        files_list: Optional[List[FileStore]] = None,
+        invigilator=None,
     ) -> ModelResponse:
         """Handle caching of responses.
 
@@ -455,7 +437,6 @@ class LanguageModel(
 
         if files_list:
             files_hash = "+".join([str(hash(file)) for file in files_list])
-            # print(f"Files hash: {files_hash}")
             user_prompt_with_hashes = user_prompt + f" {files_hash}"
         else:
             user_prompt_with_hashes = user_prompt
@@ -481,9 +462,7 @@ class LanguageModel(
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
                 "files_list": files_list,
-                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
-            # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
             new_cache_key = cache.store(
                 **cache_call_params, response=response
@@ -504,11 +483,9 @@ class LanguageModel(
         _async_get_intended_model_call_outcome
     )
 
-    # get_raw_response = sync_wrapper(async_get_raw_response)
-
     def simple_ask(
         self,
-        question: "QuestionBase",
+        question: QuestionBase,
         system_prompt="You are a helpful agent pretending to be a human.",
         top_logprobs=2,
     ):
@@ -523,9 +500,10 @@ class LanguageModel(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache: "Cache",
+        cache: Cache,
         iteration: int = 1,
-        files_list: Optional[List["File"]] = None,
+        files_list: Optional[List[FileStore]] = None,
+        **kwargs,
     ) -> dict:
         """Get response, parse, and return as string.
 
@@ -543,6 +521,9 @@ class LanguageModel(
             "cache": cache,
             "files_list": files_list,
         }
+        if "invigilator" in kwargs:
+            params.update({"invigilator": kwargs["invigilator"]})
+
         model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
         model_outputs = await self._async_get_intended_model_call_outcome(**params)
         edsl_dict = self.parse_response(model_outputs.response)
@@ -553,8 +534,6 @@ class LanguageModel(
         )
         return agent_response_dict
 
-        # return await self._async_prepare_response(model_call_outcome, cache=cache)
-
     get_response = sync_wrapper(async_get_response)
 
     def cost(self, raw_response: dict[str, Any]) -> Union[float, str]:
@@ -604,21 +583,20 @@ class LanguageModel(
 
         return input_cost + output_cost
 
-    #######################
-    # SERIALIZATION METHODS
-    #######################
-    def _to_dict(self) -> dict[str, Any]:
-        return {"model": self.model, "parameters": self.parameters}
-
-    @add_edsl_version
-    def to_dict(self) -> dict[str, Any]:
-        """Convert instance to a dictionary.
+    def to_dict(self, add_edsl_version: bool = True) -> dict[str, Any]:
+        """Convert instance to a dictionary
 
         >>> m = LanguageModel.example()
         >>> m.to_dict()
         {'model': '...', 'parameters': {'temperature': ..., 'max_tokens': ..., 'top_p': ..., 'frequency_penalty': ..., 'presence_penalty': ..., 'logprobs': False, 'top_logprobs': ...}, 'edsl_version': '...', 'edsl_class_name': 'LanguageModel'}
         """
-        return self._to_dict()
+        d = {"model": self.model, "parameters": self.parameters}
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = self.__class__.__name__
+        return d
 
     @classmethod
     @remove_edsl_version
@@ -627,18 +605,8 @@ class LanguageModel(
         from edsl.language_models.registry import get_model_class
 
         model_class = get_model_class(data["model"])
-        # data["use_cache"] = True
         return model_class(**data)
 
-    #######################
-    # DUNDER METHODS
-    #######################
-    def print(self):
-        from rich import print_json
-        import json
-
-        print_json(json.dumps(self.to_dict()))
-
     def __repr__(self) -> str:
         """Return a string representation of the object."""
         param_string = ", ".join(
@@ -652,33 +620,21 @@ class LanguageModel(
 
     def __add__(self, other_model: Type[LanguageModel]) -> Type[LanguageModel]:
         """Combine two models into a single model (other_model takes precedence over self)."""
-        print(
+        import warnings
+
+        warnings.warn(
             f"""Warning: one model is replacing another. If you want to run both models, use a single `by` e.g., 
               by(m1, m2, m3) not by(m1).by(m2).by(m3)."""
         )
         return other_model or self
 
-    def rich_print(self):
-        """Display an object as a table."""
-        from rich.table import Table
-
-        table = Table(title="Language Model")
-        table.add_column("Attribute", style="bold")
-        table.add_column("Value")
-
-        to_display = self.__dict__.copy()
-        for attr_name, attr_value in to_display.items():
-            table.add_row(attr_name, repr(attr_value))
-
-        return table
-
     @classmethod
     def example(
         cls,
         test_model: bool = False,
         canned_response: str = "Hello world",
         throw_exception: bool = False,
-    ):
+    ) -> LanguageModel:
         """Return a default instance of the class.
 
         >>> from edsl.language_models import LanguageModel
@@ -689,11 +645,17 @@ class LanguageModel(
         >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
         >>> q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True).select('example').first()
         'WOWZA!'
+        >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!", throw_exception = True)
+        >>> r = q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True, print_exceptions = True)
+        Exception report saved to ...
+        Also see: ...
         """
         from edsl import Model
 
         if test_model:
-            m = Model("test", canned_response=canned_response)
+            m = Model(
+                "test", canned_response=canned_response, throw_exception=throw_exception
+            )
             return m
         else:
             return Model(skip_api_key_check=True)

edsl/language_models/ModelList.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from typing import Optional, List
 from collections import UserList
 from edsl import Model
 
@@ -10,6 +10,8 @@ from edsl.utilities.utilities import dict_hash
 
 
 class ModelList(Base, UserList):
+    __documentation__ = """https://docs.expectedparrot.com/en/latest/language_models.html#module-edsl.language_models.ModelList"""
+
     def __init__(self, data: Optional[list] = None):
         """Initialize the ScenarioList class.
 
@@ -37,6 +39,9 @@ class ModelList(Base, UserList):
     def __repr__(self):
         return f"ModelList({super().__repr__()})"
 
+    def _summary(self):
+        return {"EDSL Class": "ModelList", "Number of Models": len(self)}
+
     def __hash__(self):
         """Return a hash of the ModelList. This is used for comparison of ModelLists.
 
@@ -46,14 +51,71 @@ class ModelList(Base, UserList):
         """
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict(sort=True))
+        return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
+
+    def to_scenario_list(self):
+        from edsl import ScenarioList, Scenario
+
+        sl = ScenarioList()
+        for model in self:
+            d = {"model": model.model}
+            d.update(model.parameters)
+            sl.append(Scenario(d))
+        return sl
+
+    def tree(self, node_list: Optional[List[str]] = None):
+        return self.to_scenario_list().tree(node_list)
+
+    def table(
+        self,
+        *fields,
+        tablefmt: Optional[str] = None,
+        pretty_labels: Optional[dict] = None,
+    ):
+        """
+        >>> ModelList.example().table("model")
+        model
+        -------
+        gpt-4o
+        gpt-4o
+        gpt-4o
+        """
+        return (
+            self.to_scenario_list()
+            .to_dataset()
+            .table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
+        )
 
-    def _to_dict(self, sort=False):
+    def to_list(self):
+        return self.to_scenario_list().to_list()
+
+    def to_dict(self, sort=False, add_edsl_version=True):
         if sort:
             model_list = sorted([model for model in self], key=lambda x: hash(x))
-            return {"models": [model._to_dict() for model in model_list]}
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version)
+                    for model in model_list
+                ]
+            }
         else:
-            return {"models": [model._to_dict() for model in self]}
+            d = {
+                "models": [
+                    model.to_dict(add_edsl_version=add_edsl_version) for model in self
+                ]
+            }
+        if add_edsl_version:
+            from edsl import __version__
+
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "ModelList"
+
+        return d
+
+    def _repr_html_(self):
+        """Return an HTML representation of the ModelList."""
+        footer = f"<a href={self.__documentation__}>(docs)</a>"
+        return str(self.summary(format="html")) + footer
 
     @classmethod
     def from_names(self, *args, **kwargs):
@@ -62,15 +124,6 @@ class ModelList(Base, UserList):
             args = args[0]
         return ModelList([Model(model_name, **kwargs) for model_name in args])
 
-    @add_edsl_version
-    def to_dict(self):
-        """
-        Convert the ModelList to a dictionary.
-        >>> ModelList.example().to_dict()
-        {'models': [...], 'edsl_version': '...', 'edsl_class_name': 'ModelList'}
-        """
-        return self._to_dict()
-
     @classmethod
     @remove_edsl_version
     def from_dict(cls, data):

edsl/language_models/registry.py

@@ -7,6 +7,49 @@ from edsl.config import CONFIG
 # else:
 #     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
+from collections import UserList
+
+
+class PrettyList(UserList):
+    def __init__(self, data=None, columns=None):
+        super().__init__(data)
+        self.columns = columns
+
+    def _repr_html_(self):
+        if isinstance(self[0], list) or isinstance(self[0], tuple):
+            num_cols = len(self[0])
+        else:
+            num_cols = 1
+
+        if self.columns:
+            columns = self.columns
+        else:
+            columns = list(range(num_cols))
+
+        if num_cols > 1:
+            return (
+                "<pre><table>"
+                + "".join(["<th>" + str(column) + "</th>" for column in columns])
+                + "".join(
+                    [
+                        "<tr>"
+                        + "".join(["<td>" + str(x) + "</td>" for x in row])
+                        + "</tr>"
+                        for row in self
+                    ]
+                )
+                + "</table></pre>"
+            )
+        else:
+            return (
+                "<pre><table>"
+                + "".join(["<th>" + str(index) + "</th>" for index in columns])
+                + "".join(
+                    ["<tr>" + "<td>" + str(row) + "</td>" + "</tr>" for row in self]
+                )
+                + "</table></pre>"
+            )
+
 
 def get_model_class(model_name, registry=None):
     from edsl.inference_services.registry import default
@@ -82,17 +125,27 @@ class Model(metaclass=Meta):
 
         if search_term is None:
             if name_only:
-                return [m[0] for m in full_list]
+                return PrettyList(
+                    [m[0] for m in full_list],
+                    columns=["Model Name", "Service Name", "Code"],
+                )
             else:
-                return full_list
+                return PrettyList(
+                    full_list, columns=["Model Name", "Service Name", "Code"]
+                )
         else:
             filtered_results = [
                 m for m in full_list if search_term in m[0] or search_term in m[1]
             ]
             if name_only:
-                return [m[0] for m in filtered_results]
+                return PrettyList(
+                    [m[0] for m in filtered_results],
+                    columns=["Model Name", "Service Name", "Code"],
+                )
             else:
-                return filtered_results
+                return PrettyList(
+                    filtered_results, columns=["Model Name", "Service Name", "Code"]
+                )
 
     @classmethod
     def check_models(cls, verbose=False):

edsl/notebooks/Notebook.py

@@ -102,18 +102,17 @@ class Notebook(Base):
 
         return dict_hash(self.data["cells"])
 
-    def _to_dict(self) -> dict:
+    def to_dict(self, add_edsl_version=False) -> dict:
         """
         Serialize to a dictionary.
         """
-        return {"name": self.name, "data": self.data}
+        d = {"name": self.name, "data": self.data}
+        if add_edsl_version:
+            from edsl import __version__
 
-    @add_edsl_version
-    def to_dict(self) -> dict:
-        """
-        Convert a Notebook to a dictionary.
-        """
-        return self._to_dict()
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = self.__class__.__name__
+        return d
 
     @classmethod
     @remove_edsl_version

edsl/prompts/Prompt.py

@@ -29,9 +29,14 @@ class Prompt(PersistenceMixin, RichPrintingMixin):
 
     def _repr_html_(self):
         """Return an HTML representation of the Prompt."""
-        from edsl.utilities.utilities import data_to_html
-
-        return data_to_html(self.to_dict())
+        # from edsl.utilities.utilities import data_to_html
+        # return data_to_html(self.to_dict())
+        d = self.to_dict()
+        data = [[k, v] for k, v in d.items()]
+        from tabulate import tabulate
+
+        table = str(tabulate(data, headers=["keys", "values"], tablefmt="html"))
+        return f"<pre>{table}</pre>"
 
     def __len__(self):
         """Return the length of the prompt text."""

edsl/questions/QuestionBase.py

@@ -135,7 +135,7 @@ class QuestionBase(
         """
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self._to_dict())
+        return dict_hash(self.to_dict(add_edsl_version=False))
 
     @property
     def data(self) -> dict:
@@ -147,13 +147,15 @@ class QuestionBase(
         """
         exclude_list = [
             "question_type",
-            "_include_comment",
+            # "_include_comment",
             "_fake_data_factory",
-            "_use_code",
+            # "_use_code",
             "_model_instructions",
         ]
         only_if_not_na_list = ["_answering_instructions", "_question_presentation"]
 
+        only_if_not_default_list = {"_include_comment": True, "_use_code": False}
+
         def ok(key, value):
             if not key.startswith("_"):
                 return False
@@ -161,6 +163,12 @@ class QuestionBase(
                 return False
             if key in only_if_not_na_list and value is None:
                 return False
+            if (
+                key in only_if_not_default_list
+                and value == only_if_not_default_list[key]
+            ):
+                return False
+
             return True
 
         candidate_data = {
@@ -175,25 +183,22 @@ class QuestionBase(
 
         return candidate_data
 
-    def _to_dict(self):
+    def to_dict(self, add_edsl_version=True):
         """Convert the question to a dictionary that includes the question type (used in deserialization).
 
-        >>> from edsl import QuestionFreeText as Q; Q.example()._to_dict()
+        >>> from edsl import QuestionFreeText as Q; Q.example().to_dict(add_edsl_version = False)
         {'question_name': 'how_are_you', 'question_text': 'How are you?', 'question_type': 'free_text'}
         """
         candidate_data = self.data.copy()
         candidate_data["question_type"] = self.question_type
-        return {
-            key: value for key, value in candidate_data.items() if value is not None
-        }
+        d = {key: value for key, value in candidate_data.items() if value is not None}
+        if add_edsl_version:
+            from edsl import __version__
 
-    @add_edsl_version
-    def to_dict(self) -> dict[str, Any]:
-        """Convert the question to a dictionary that includes the question type (used in deserialization).
-        >>> from edsl import QuestionFreeText as Q; Q.example().to_dict()
-        {'question_name': 'how_are_you', 'question_text': 'How are you?', 'question_type': 'free_text', 'edsl_version': '...'}
-        """
-        return self._to_dict()
+            d["edsl_version"] = __version__
+            d["edsl_class_name"] = "QuestionBase"
+
+        return d
 
     @classmethod
     @remove_edsl_version
@@ -259,12 +264,9 @@ class QuestionBase(
         >>> m.execute_model_call("", "")
         {'message': [{'text': "Yo, what's up?"}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
         >>> Q.run_example(show_answer = True, model = m, disable_remote_cache = True, disable_remote_inference = True)
-        ┏━━━━━━━━━━━━━━━━┓
-        ┃ answer         ┃
-        ┃ .how_are_you   ┃
-        ┡━━━━━━━━━━━━━━━━┩
-        │ Yo, what's up? │
-        └────────────────┘
+        answer.how_are_you
+        --------------------
+        Yo, what's up?
         """
         if model is None:
             from edsl import Model
@@ -280,7 +282,7 @@ class QuestionBase(
             )
         )
         if show_answer:
-            results.select("answer.*").print()
+            return results.select("answer.*").print()
         else:
             return results
 
@@ -358,16 +360,22 @@ class QuestionBase(
 
     # region: Magic methods
     def _repr_html_(self):
-        from edsl.utilities.utilities import data_to_html
+        # from edsl.utilities.utilities import data_to_html
 
-        data = self.to_dict()
-        try:
-            _ = data.pop("edsl_version")
-            _ = data.pop("edsl_class_name")
-        except KeyError:
-            print("Serialized question lacks edsl version, but is should have it.")
+        data = self.to_dict(add_edsl_version=False)
+        # keys = list(data.keys())
+        # values = list(data.values())
+        from tabulate import tabulate
+
+        return tabulate(data.items(), headers=["keys", "values"], tablefmt="html")
+
+        # try:
+        #     _ = data.pop("edsl_version")
+        #     _ = data.pop("edsl_class_name")
+        # except KeyError:
+        #     print("Serialized question lacks edsl version, but is should have it.")
 
-        return data_to_html(data)
+        # return data_to_html(data)
 
     def __getitem__(self, key: str) -> Any:
         """Get an attribute of the question so it can be treated like a dictionary.