edsl 0.1.38__py3-none-any.whl → 0.1.38.dev2__py3-none-any.whl

edsl/language_models/LanguageModel.py

@@ -41,20 +41,18 @@ from edsl.data_transfer_models import (
     AgentResponseDict,
 )
 
-if TYPE_CHECKING:
-    from edsl.data.Cache import Cache
-    from edsl.scenarios.FileStore import FileStore
-    from edsl.questions.QuestionBase import QuestionBase
 
 from edsl.config import CONFIG
 from edsl.utilities.decorators import sync_wrapper, jupyter_nb_handler
-from edsl.utilities.decorators import remove_edsl_version
-
-from edsl.Base import PersistenceMixin
+from edsl.utilities.decorators import add_edsl_version, remove_edsl_version
+from edsl.language_models.repair import repair
+from edsl.enums import InferenceServiceType
+from edsl.Base import RichPrintingMixin, PersistenceMixin
 from edsl.language_models.RegisterLanguageModelsMeta import RegisterLanguageModelsMeta
-from edsl.language_models.KeyLookup import KeyLookup
 from edsl.exceptions.language_models import LanguageModelBadResponseError
 
+from edsl.language_models.KeyLookup import KeyLookup
+
 TIMEOUT = float(CONFIG.get("EDSL_API_TIMEOUT"))
 
 
@@ -118,11 +116,29 @@ def handle_key_error(func):
 
 
 class LanguageModel(
-    PersistenceMixin,
-    ABC,
-    metaclass=RegisterLanguageModelsMeta,
+    RichPrintingMixin, PersistenceMixin, ABC, metaclass=RegisterLanguageModelsMeta
 ):
-    """ABC for Language Models."""
+    """ABC for LLM subclasses.
+
+    TODO:
+
+    1) Need better, more descriptive names for functions
+
+    get_model_response_no_cache  (currently called async_execute_model_call)
+
+    get_model_response (currently called async_get_raw_response; uses cache & adds tracking info)
+      Calls:
+        - async_execute_model_call
+        - _updated_model_response_with_tracking
+
+    get_answer (currently called async_get_response)
+        This parses out the answer block and does some error-handling.
+        Calls:
+            - async_get_raw_response
+            - parse_response
+
+
+    """
 
     _model_ = None
     key_sequence = (
@@ -180,7 +196,7 @@ class LanguageModel(
         system_prompt = "You are a helpful agent pretending to be a human."
         return self.execute_model_call(user_prompt, system_prompt)
 
-    def set_key_lookup(self, key_lookup: KeyLookup) -> None:
+    def set_key_lookup(self, key_lookup: KeyLookup):
         del self._api_token
         self.key_lookup = key_lookup
 
@@ -195,14 +211,10 @@ class LanguageModel(
     def __getitem__(self, key):
         return getattr(self, key)
 
-    def _repr_html_(self) -> str:
-        d = {"model": self.model}
-        d.update(self.parameters)
-        data = [[k, v] for k, v in d.items()]
-        from tabulate import tabulate
+    def _repr_html_(self):
+        from edsl.utilities.utilities import data_to_html
 
-        table = str(tabulate(data, headers=["keys", "values"], tablefmt="html"))
-        return f"<pre>{table}</pre>"
+        return data_to_html(self.to_dict())
 
     def hello(self, verbose=False):
         """Runs a simple test to check if the model is working."""
@@ -223,6 +235,7 @@ class LanguageModel(
         This method is used to check if the model has a valid API key.
         """
         from edsl.enums import service_to_api_keyname
+        import os
 
         if self._model_ == "test":
             return True
@@ -235,9 +248,9 @@ class LanguageModel(
         """Allow the model to be used as a key in a dictionary."""
         from edsl.utilities.utilities import dict_hash
 
-        return dict_hash(self.to_dict(add_edsl_version=False))
+        return dict_hash(self.to_dict())
 
-    def __eq__(self, other) -> bool:
+    def __eq__(self, other):
         """Check is two models are the same.
 
         >>> m1 = LanguageModel.example()
@@ -265,11 +278,15 @@ class LanguageModel(
     @property
     def RPM(self):
         """Model's requests-per-minute limit."""
+        # self._set_rate_limits()
+        # return self._safety_factor * self.__rate_limits["rpm"]
         return self._rpm
 
     @property
     def TPM(self):
         """Model's tokens-per-minute limit."""
+        # self._set_rate_limits()
+        # return self._safety_factor * self.__rate_limits["tpm"]
         return self._tpm
 
     @property
@@ -297,6 +314,8 @@ class LanguageModel(
         >>> LanguageModel._overide_default_parameters(passed_parameter_dict={"temperature": 0.5}, default_parameter_dict={"temperature":0.9, "max_tokens": 1000})
         {'temperature': 0.5, 'max_tokens': 1000}
         """
+        # parameters = dict({})
+
         # this is the case when data is loaded from a dict after serialization
         if "parameters" in passed_parameter_dict:
             passed_parameter_dict = passed_parameter_dict["parameters"]
@@ -410,10 +429,9 @@ class LanguageModel(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache: Cache,
+        cache: "Cache",
         iteration: int = 0,
-        files_list: Optional[List[FileStore]] = None,
-        invigilator=None,
+        files_list=None,
     ) -> ModelResponse:
         """Handle caching of responses.
 
@@ -437,6 +455,7 @@ class LanguageModel(
 
         if files_list:
             files_hash = "+".join([str(hash(file)) for file in files_list])
+            # print(f"Files hash: {files_hash}")
             user_prompt_with_hashes = user_prompt + f" {files_hash}"
         else:
             user_prompt_with_hashes = user_prompt
@@ -462,7 +481,9 @@ class LanguageModel(
                 "user_prompt": user_prompt,
                 "system_prompt": system_prompt,
                 "files_list": files_list,
+                # **({"encoded_image": encoded_image} if encoded_image else {}),
             }
+            # response = await f(**params)
             response = await asyncio.wait_for(f(**params), timeout=TIMEOUT)
             new_cache_key = cache.store(
                 **cache_call_params, response=response
@@ -483,9 +504,11 @@ class LanguageModel(
         _async_get_intended_model_call_outcome
     )
 
+    # get_raw_response = sync_wrapper(async_get_raw_response)
+
     def simple_ask(
         self,
-        question: QuestionBase,
+        question: "QuestionBase",
         system_prompt="You are a helpful agent pretending to be a human.",
         top_logprobs=2,
     ):
@@ -500,10 +523,9 @@ class LanguageModel(
         self,
         user_prompt: str,
         system_prompt: str,
-        cache: Cache,
+        cache: "Cache",
         iteration: int = 1,
-        files_list: Optional[List[FileStore]] = None,
-        **kwargs,
+        files_list: Optional[List["File"]] = None,
     ) -> dict:
         """Get response, parse, and return as string.
 
@@ -521,9 +543,6 @@ class LanguageModel(
             "cache": cache,
             "files_list": files_list,
         }
-        if "invigilator" in kwargs:
-            params.update({"invigilator": kwargs["invigilator"]})
-
         model_inputs = ModelInputs(user_prompt=user_prompt, system_prompt=system_prompt)
         model_outputs = await self._async_get_intended_model_call_outcome(**params)
         edsl_dict = self.parse_response(model_outputs.response)
@@ -534,6 +553,8 @@ class LanguageModel(
         )
         return agent_response_dict
 
+        # return await self._async_prepare_response(model_call_outcome, cache=cache)
+
     get_response = sync_wrapper(async_get_response)
 
     def cost(self, raw_response: dict[str, Any]) -> Union[float, str]:
@@ -583,7 +604,10 @@ class LanguageModel(
 
         return input_cost + output_cost
 
-    def to_dict(self, add_edsl_version: bool = True) -> dict[str, Any]:
+    #######################
+    # SERIALIZATION METHODS
+    #######################
+    def to_dict(self, add_edsl_version=True) -> dict[str, Any]:
         """Convert instance to a dictionary
 
         >>> m = LanguageModel.example()
@@ -605,8 +629,18 @@ class LanguageModel(
         from edsl.language_models.registry import get_model_class
 
         model_class = get_model_class(data["model"])
+        # data["use_cache"] = True
         return model_class(**data)
 
+    #######################
+    # DUNDER METHODS
+    #######################
+    def print(self):
+        from rich import print_json
+        import json
+
+        print_json(json.dumps(self.to_dict()))
+
     def __repr__(self) -> str:
         """Return a string representation of the object."""
         param_string = ", ".join(
@@ -620,21 +654,33 @@ class LanguageModel(
 
     def __add__(self, other_model: Type[LanguageModel]) -> Type[LanguageModel]:
         """Combine two models into a single model (other_model takes precedence over self)."""
-        import warnings
-
-        warnings.warn(
+        print(
             f"""Warning: one model is replacing another. If you want to run both models, use a single `by` e.g., 
               by(m1, m2, m3) not by(m1).by(m2).by(m3)."""
         )
         return other_model or self
 
+    def rich_print(self):
+        """Display an object as a table."""
+        from rich.table import Table
+
+        table = Table(title="Language Model")
+        table.add_column("Attribute", style="bold")
+        table.add_column("Value")
+
+        to_display = self.__dict__.copy()
+        for attr_name, attr_value in to_display.items():
+            table.add_row(attr_name, repr(attr_value))
+
+        return table
+
     @classmethod
     def example(
         cls,
         test_model: bool = False,
         canned_response: str = "Hello world",
         throw_exception: bool = False,
-    ) -> LanguageModel:
+    ):
         """Return a default instance of the class.
 
         >>> from edsl.language_models import LanguageModel
@@ -645,17 +691,11 @@ class LanguageModel(
         >>> q = QuestionFreeText(question_text = "What is your name?", question_name = 'example')
         >>> q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True).select('example').first()
         'WOWZA!'
-        >>> m = LanguageModel.example(test_model = True, canned_response = "WOWZA!", throw_exception = True)
-        >>> r = q.by(m).run(cache = False, disable_remote_cache = True, disable_remote_inference = True, print_exceptions = True)
-        Exception report saved to ...
-        Also see: ...
         """
         from edsl import Model
 
         if test_model:
-            m = Model(
-                "test", canned_response=canned_response, throw_exception=throw_exception
-            )
+            m = Model("test", canned_response=canned_response)
             return m
         else:
             return Model(skip_api_key_check=True)

edsl/language_models/ModelList.py

@@ -1,4 +1,4 @@
-from typing import Optional, List
+from typing import Optional
 from collections import UserList
 from edsl import Model
 
@@ -10,8 +10,6 @@ from edsl.utilities.utilities import dict_hash
 
 
 class ModelList(Base, UserList):
-    __documentation__ = """https://docs.expectedparrot.com/en/latest/language_models.html#module-edsl.language_models.ModelList"""
-
     def __init__(self, data: Optional[list] = None):
         """Initialize the ScenarioList class.
 
@@ -39,9 +37,6 @@ class ModelList(Base, UserList):
     def __repr__(self):
         return f"ModelList({super().__repr__()})"
 
-    def _summary(self):
-        return {"EDSL Class": "ModelList", "Number of Models": len(self)}
-
     def __hash__(self):
         """Return a hash of the ModelList. This is used for comparison of ModelLists.
 
@@ -53,42 +48,6 @@ class ModelList(Base, UserList):
 
         return dict_hash(self.to_dict(sort=True, add_edsl_version=False))
 
-    def to_scenario_list(self):
-        from edsl import ScenarioList, Scenario
-
-        sl = ScenarioList()
-        for model in self:
-            d = {"model": model.model}
-            d.update(model.parameters)
-            sl.append(Scenario(d))
-        return sl
-
-    def tree(self, node_list: Optional[List[str]] = None):
-        return self.to_scenario_list().tree(node_list)
-
-    def table(
-        self,
-        *fields,
-        tablefmt: Optional[str] = None,
-        pretty_labels: Optional[dict] = None,
-    ):
-        """
-        >>> ModelList.example().table("model")
-        model
-        -------
-        gpt-4o
-        gpt-4o
-        gpt-4o
-        """
-        return (
-            self.to_scenario_list()
-            .to_dataset()
-            .table(*fields, tablefmt=tablefmt, pretty_labels=pretty_labels)
-        )
-
-    def to_list(self):
-        return self.to_scenario_list().to_list()
-
     def to_dict(self, sort=False, add_edsl_version=True):
         if sort:
             model_list = sorted([model for model in self], key=lambda x: hash(x))
@@ -112,11 +71,6 @@ class ModelList(Base, UserList):
 
         return d
 
-    def _repr_html_(self):
-        """Return an HTML representation of the ModelList."""
-        footer = f"<a href={self.__documentation__}>(docs)</a>"
-        return str(self.summary(format="html")) + footer
-
     @classmethod
     def from_names(self, *args, **kwargs):
         """A a model list from a list of names"""

edsl/language_models/registry.py

@@ -7,49 +7,6 @@ from edsl.config import CONFIG
 # else:
 #     default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
 
-from collections import UserList
-
-
-class PrettyList(UserList):
-    def __init__(self, data=None, columns=None):
-        super().__init__(data)
-        self.columns = columns
-
-    def _repr_html_(self):
-        if isinstance(self[0], list) or isinstance(self[0], tuple):
-            num_cols = len(self[0])
-        else:
-            num_cols = 1
-
-        if self.columns:
-            columns = self.columns
-        else:
-            columns = list(range(num_cols))
-
-        if num_cols > 1:
-            return (
-                "<pre><table>"
-                + "".join(["<th>" + str(column) + "</th>" for column in columns])
-                + "".join(
-                    [
-                        "<tr>"
-                        + "".join(["<td>" + str(x) + "</td>" for x in row])
-                        + "</tr>"
-                        for row in self
-                    ]
-                )
-                + "</table></pre>"
-            )
-        else:
-            return (
-                "<pre><table>"
-                + "".join(["<th>" + str(index) + "</th>" for index in columns])
-                + "".join(
-                    ["<tr>" + "<td>" + str(row) + "</td>" + "</tr>" for row in self]
-                )
-                + "</table></pre>"
-            )
-
 
 def get_model_class(model_name, registry=None):
     from edsl.inference_services.registry import default
@@ -125,27 +82,17 @@ class Model(metaclass=Meta):
 
         if search_term is None:
             if name_only:
-                return PrettyList(
-                    [m[0] for m in full_list],
-                    columns=["Model Name", "Service Name", "Code"],
-                )
+                return [m[0] for m in full_list]
             else:
-                return PrettyList(
-                    full_list, columns=["Model Name", "Service Name", "Code"]
-                )
+                return full_list
         else:
             filtered_results = [
                 m for m in full_list if search_term in m[0] or search_term in m[1]
             ]
             if name_only:
-                return PrettyList(
-                    [m[0] for m in filtered_results],
-                    columns=["Model Name", "Service Name", "Code"],
-                )
+                return [m[0] for m in filtered_results]
             else:
-                return PrettyList(
-                    filtered_results, columns=["Model Name", "Service Name", "Code"]
-                )
+                return filtered_results
 
     @classmethod
     def check_models(cls, verbose=False):

edsl/prompts/Prompt.py

@@ -29,14 +29,9 @@ class Prompt(PersistenceMixin, RichPrintingMixin):
 
     def _repr_html_(self):
         """Return an HTML representation of the Prompt."""
-        # from edsl.utilities.utilities import data_to_html
-        # return data_to_html(self.to_dict())
-        d = self.to_dict()
-        data = [[k, v] for k, v in d.items()]
-        from tabulate import tabulate
-
-        table = str(tabulate(data, headers=["keys", "values"], tablefmt="html"))
-        return f"<pre>{table}</pre>"
+        from edsl.utilities.utilities import data_to_html
+
+        return data_to_html(self.to_dict())
 
     def __len__(self):
         """Return the length of the prompt text."""

edsl/questions/QuestionBase.py

@@ -82,8 +82,7 @@ class QuestionBase(
         if not hasattr(self, "_fake_data_factory"):
             from polyfactory.factories.pydantic_factory import ModelFactory
 
-            class FakeData(ModelFactory[self.response_model]):
-                ...
+            class FakeData(ModelFactory[self.response_model]): ...
 
             self._fake_data_factory = FakeData
         return self._fake_data_factory
@@ -264,9 +263,12 @@ class QuestionBase(
         >>> m.execute_model_call("", "")
         {'message': [{'text': "Yo, what's up?"}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1}}
         >>> Q.run_example(show_answer = True, model = m, disable_remote_cache = True, disable_remote_inference = True)
-        answer.how_are_you
-        --------------------
-        Yo, what's up?
+        ┏━━━━━━━━━━━━━━━━┓
+        ┃ answer         ┃
+        ┃ .how_are_you   ┃
+        ┡━━━━━━━━━━━━━━━━┩
+        │ Yo, what's up? │
+        └────────────────┘
         """
         if model is None:
             from edsl import Model
@@ -282,7 +284,7 @@ class QuestionBase(
             )
         )
         if show_answer:
-            return results.select("answer.*").print()
+            results.select("answer.*").print()
         else:
             return results
 
@@ -360,22 +362,16 @@ class QuestionBase(
 
     # region: Magic methods
     def _repr_html_(self):
-        # from edsl.utilities.utilities import data_to_html
+        from edsl.utilities.utilities import data_to_html
 
-        data = self.to_dict(add_edsl_version=False)
-        # keys = list(data.keys())
-        # values = list(data.values())
-        from tabulate import tabulate
-
-        return tabulate(data.items(), headers=["keys", "values"], tablefmt="html")
-
-        # try:
-        #     _ = data.pop("edsl_version")
-        #     _ = data.pop("edsl_class_name")
-        # except KeyError:
-        #     print("Serialized question lacks edsl version, but is should have it.")
+        data = self.to_dict()
+        try:
+            _ = data.pop("edsl_version")
+            _ = data.pop("edsl_class_name")
+        except KeyError:
+            print("Serialized question lacks edsl version, but is should have it.")
 
-        # return data_to_html(data)
+        return data_to_html(data)
 
     def __getitem__(self, key: str) -> Any:
         """Get an attribute of the question so it can be treated like a dictionary.

edsl/questions/QuestionExtract.py

@@ -1,7 +1,4 @@
 from __future__ import annotations
-import json
-import re
-
 from typing import Any, Optional, Dict
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import AnswerTemplateDescriptor
@@ -14,6 +11,9 @@ from edsl.questions.decorators import inject_exception
 from typing import Dict, Any
 from pydantic import create_model, Field
 
+import json
+import re
+
 
 def extract_json(text, expected_keys, verbose=False):
     # Escape special regex characters in keys
@@ -112,6 +112,7 @@ class QuestionExtract(QuestionBase):
 
         :param question_name: The name of the question.
         :param question_text: The text of the question.
+        :param question_options: The options the respondent should select from.
         :param answer_template: The template for the answer.
         """
         self.question_name = question_name

edsl/questions/question_registry.py

@@ -90,22 +90,6 @@ class Question(metaclass=Meta):
         coop = Coop()
         return coop.patch(uuid, url, description, value, visibility)
 
-    @classmethod
-    def list_question_types(cls):
-        """Return a list of available question types.
-
-        >>> from edsl import Question
-        >>> Question.list_question_types()
-        ['checkbox', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
-        """
-        return [
-            q
-            for q in sorted(
-                list(RegisterQuestionsMeta.question_types_to_classes().keys())
-            )
-            if q not in ["budget"]
-        ]
-
     @classmethod
     def available(cls, show_class_names: bool = False) -> Union[list, dict]:
         """Return a list of available question types.
@@ -114,32 +98,18 @@ class Question(metaclass=Meta):
 
         Example usage:
 
+        >>> from edsl import Question
+        >>> Question.available()
+        ['checkbox', 'extract', 'free_text', 'functional', 'likert_five', 'linear_scale', 'list', 'multiple_choice', 'numerical', 'rank', 'top_k', 'yes_no']
         """
-        from edsl.results.Dataset import Dataset
-
         exclude = ["budget"]
         if show_class_names:
             return RegisterQuestionsMeta.question_types_to_classes()
         else:
-            question_list = [
-                q
-                for q in sorted(
-                    set(RegisterQuestionsMeta.question_types_to_classes().keys())
-                )
-                if q not in exclude
-            ]
-            d = RegisterQuestionsMeta.question_types_to_classes()
-            question_classes = [d[q] for q in question_list]
-            example_questions = [repr(q.example()) for q in question_classes]
-
-            return Dataset(
-                [
-                    {"question_type": [q for q in question_list]},
-                    {"question_class": [q.__name__ for q in question_classes]},
-                    {"example_question": example_questions},
-                ],
-                print_parameters={"containerHeight": "auto"},
+            question_list = sorted(
+                set(RegisterQuestionsMeta.question_types_to_classes().keys())
             )
+            return [q for q in question_list if q not in exclude]
 
 
 def get_question_class(question_type):