edsl 0.1.39__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/language_models/registry.py

@@ -0,0 +1,180 @@
+import textwrap
+from random import random
+from edsl.config import CONFIG
+from functools import lru_cache
+from edsl.utilities.PrettyList import PrettyList
+from typing import Optional
+
+
+def get_model_class(model_name, registry=None):
+    from edsl.inference_services.registry import default
+
+    registry = registry or default
+    factory = registry.create_model_factory(model_name)
+    return factory
+
+
+class Meta(type):
+    def __repr__(cls):
+        return textwrap.dedent(
+            f"""\
+        Available models: {cls.available()}
+        
+        To create an instance, you can do: 
+        >>> m = Model('gpt-4-1106-preview', temperature=0.5, ...)
+        
+        To get the default model, you can leave out the model name. 
+        To see the available models, you can do:
+        >>> Model.available()
+        """
+        )
+
+
+class Model(metaclass=Meta):
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+
+    def __new__(
+        cls, model_name=None, registry=None, service_name=None, *args, **kwargs
+    ):
+        # Map index to the respective subclass
+        if model_name is None:
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+
+        if isinstance(model_name, int):  # can refer to a model by index
+            model_name = cls.available(name_only=True)[model_name]
+
+        factory = registry.create_model_factory(model_name, service_name=service_name)
+        return factory(*args, **kwargs)
+
+    @classmethod
+    def add_model(cls, service_name, model_name):
+        from edsl.inference_services.registry import default
+
+        registry = default
+        registry.add_model(service_name, model_name)
+
+    @classmethod
+    def service_classes(cls, registry=None):
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+        return [r for r in registry.services]
+
+    @classmethod
+    def services(cls, registry=None):
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+        return PrettyList(
+            [r._inference_service_ for r in registry.services], columns=["Service Name"]
+        )
+
+    @classmethod
+    def key_info(cls):
+        from edsl.language_models.key_management import KeyLookupCollection
+        from edsl.scenarios import Scenario, ScenarioList
+
+        klc = KeyLookupCollection()
+        klc.add_key_lookup(fetch_order=None)
+        sl = ScenarioList()
+        for service, entry in list(klc.data.values())[0].items():
+            sl.append(Scenario({"service": service} | entry.to_dict()))
+        return sl.to_dataset()
+
+    @classmethod
+    def available(
+        cls,
+        search_term: str = None,
+        name_only: bool = False,
+        registry=None,
+        service: Optional[str] = None,
+    ):
+        from edsl.inference_services.registry import default
+
+        registry = registry or default
+        # full_list = registry.available()
+
+        if service is not None:
+            if service not in cls.services(registry=registry):
+                raise ValueError(f"Service {service} not found in available services.")
+
+        # import time
+        # start = time.time()
+        full_list = registry.available(service=service)
+        # end = time.time()
+        # print(f"Time taken to get available models: {end-start}")
+
+        if search_term is None:
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in full_list],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+        else:
+            filtered_results = [
+                m
+                for m in full_list
+                if search_term in m.model_name or search_term in m.service_name
+            ]
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in filtered_results],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+
+    @classmethod
+    def check_models(cls, verbose=False):
+        print("Checking all available models...\n")
+        for model in cls.available(name_only=True):
+            print(f"Now checking: {model}")
+            try:
+                m = cls(model)
+            except Exception as e:
+                print(f"Error creating instance of {model}: {e}")
+                continue
+            try:
+                results = m.hello(verbose)
+                if verbose:
+                    print(f"Results from model call: {results}")
+            except Exception as e:
+                print(f"Error calling 'hello' on {model}: {e}")
+                continue
+            print("OK!")
+            print("\n")
+
+    @classmethod
+    def example(cls, randomize: bool = False) -> "Model":
+        """
+        Returns an example Model instance.
+
+        :param randomize: If True, the temperature is set to a random decimal between 0 and 1.
+        """
+        temperature = 0.5 if not randomize else round(random(), 2)
+        model_name = cls.default_model
+        return cls(model_name, temperature=temperature)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    available = Model.available()
+    m = Model("gpt-4-1106-preview")
+    results = m.execute_model_call("Hello world")
+    print(results)

edsl/language_models/repair.py

@@ -32,7 +32,7 @@ async def async_repair(
     else:
         return valid_dict, success
 
-    from edsl.language_models.model import Model
+    from edsl.language_models.registry import Model
 
     m = Model()
 

edsl/questions/QuestionBase.py

@@ -2,31 +2,23 @@
 
 from __future__ import annotations
 from abc import ABC, abstractmethod
-from typing import Any, Type, Optional, List, Callable, Union, TypedDict, TYPE_CHECKING
+from typing import Any, Type, Optional, List, Callable, Union, TypedDict
 
 from edsl.exceptions.questions import (
     QuestionSerializationError,
 )
 from edsl.questions.descriptors import QuestionNameDescriptor, QuestionTextDescriptor
 
-from edsl.questions.answer_validator_mixin import AnswerValidatorMixin
-from edsl.questions.register_questions_meta import RegisterQuestionsMeta
+from edsl.questions.AnswerValidatorMixin import AnswerValidatorMixin
+from edsl.questions.RegisterQuestionsMeta import RegisterQuestionsMeta
 from edsl.Base import PersistenceMixin, RepresentationMixin
 from edsl.BaseDiff import BaseDiff, BaseDiffCollection
 
 from edsl.questions.SimpleAskMixin import SimpleAskMixin
 from edsl.questions.QuestionBasePromptsMixin import QuestionBasePromptsMixin
-from edsl.questions.question_base_gen_mixin import QuestionBaseGenMixin
+from edsl.questions.QuestionBaseGenMixin import QuestionBaseGenMixin
 from edsl.utilities.remove_edsl_version import remove_edsl_version
 
-if TYPE_CHECKING:
-    from edsl.questions.response_validator_abc import ResponseValidatorABC
-    from edsl.language_models.LanguageModel import LanguageModel
-    from edsl.results.Results import Results
-    from edsl.agents.Agent import Agent
-    from edsl.surveys.Survey import Survey
-    from edsl.jobs.Jobs import Jobs
-
 
 class QuestionBase(
     PersistenceMixin,
@@ -57,17 +49,13 @@ class QuestionBase(
     _question_presentation = None
 
     @property
-    def response_validator(self) -> "ResponseValidatorABC":
+    def response_validator(self) -> "ResponseValidatorBase":
         """Return the response validator."""
-        from edsl.questions.response_validator_factory import ResponseValidatorFactory
+        from edsl.questions.ResponseValidatorFactory import ResponseValidatorFactory
 
         rvf = ResponseValidatorFactory(self)
         return rvf.response_validator
 
-    def duplicate(self):
-        """Return a duplicate of the question."""
-        return self.from_dict(self.to_dict())
-
     @property
     def fake_data_factory(self):
         """Return the fake data factory."""
@@ -171,7 +159,7 @@ class QuestionBase(
 
         return candidate_data
 
-    def to_dict(self, add_edsl_version: bool = True):
+    def to_dict(self, add_edsl_version=True):
         """Convert the question to a dictionary that includes the question type (used in deserialization).
 
         >>> from edsl.questions import QuestionFreeText as Q; Q.example().to_dict(add_edsl_version = False)
@@ -225,6 +213,9 @@ class QuestionBase(
 
         return question_class(**local_data)
 
+    # endregion
+
+    # region: Running methods
     @classmethod
     def _get_test_model(self, canned_response: Optional[str] = None) -> "LanguageModel":
         """Get a test model for the question."""
@@ -252,7 +243,7 @@ class QuestionBase(
         Dataset([{'answer.how_are_you': ["Yo, what's up?"]}])
         """
         if model is None:
-            from edsl.language_models.model import Model
+            from edsl.language_models.registry import Model
 
             model = Model()
         results = (
@@ -271,22 +262,21 @@ class QuestionBase(
 
     def __call__(
         self,
-        just_answer: bool = True,
-        model: Optional["LanguageModel"] = None,
-        agent: Optional["Agent"] = None,
+        just_answer=True,
+        model=None,
+        agent=None,
         disable_remote_cache: bool = False,
         disable_remote_inference: bool = False,
         verbose: bool = False,
         **kwargs,
-    ) -> Union[Any, "Results"]:
+    ):
         """Call the question.
 
 
         >>> from edsl import QuestionFreeText as Q
-        >>> from edsl import Model
-        >>> m = Model("test", canned_response = "Yo, what's up?")
+        >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True, cache = False)
+        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True)
         "Yo, what's up?"
 
         """
@@ -295,6 +285,7 @@ class QuestionBase(
             model=model,
             agent=agent,
             **kwargs,
+            cache=False,
             verbose=verbose,
             disable_remote_cache=disable_remote_cache,
             disable_remote_inference=disable_remote_inference,
@@ -306,16 +297,15 @@ class QuestionBase(
 
     def run(self, *args, **kwargs) -> "Results":
         """Turn a single question into a survey and runs it."""
-        return self.to_survey().run(*args, **kwargs)
+        from edsl.surveys.Survey import Survey
 
-    def using(self, *args, **kwargs) -> "Jobs":
-        """Turn a single question into a survey and then a Job."""
-        return self.to_survey().to_jobs().using(*args, **kwargs)
+        s = self.to_survey()
+        return s.run(*args, **kwargs)
 
     async def run_async(
         self,
         just_answer: bool = True,
-        model: Optional["LanguageModel"] = None,
+        model: Optional["Model"] = None,
         agent: Optional["Agent"] = None,
         disable_remote_inference: bool = False,
         **kwargs,
@@ -326,7 +316,7 @@ class QuestionBase(
         >>> from edsl.questions import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Blue")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True, disable_remote_cache = True); print(result)
+        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True); print(result)
         >>> asyncio.run(test_run_async())
         Blue
         """
@@ -430,7 +420,8 @@ class QuestionBase(
         """
         from edsl.surveys.Survey import Survey
 
-        return Survey([self])
+        s = Survey([self])
+        return s
 
     def by(self, *args) -> "Jobs":
         """Turn a single question into a survey and then a Job."""
@@ -473,7 +464,7 @@ class QuestionBase(
 
     @classmethod
     def example_model(cls):
-        from edsl.language_models.model import Model
+        from edsl.language_models.registry import Model
 
         q = cls.example()
         m = Model("test", canned_response=cls._simulate_answer(q)["answer"])

edsl/questions/{question_base_gen_mixin.py → QuestionBaseGenMixin.py}

@@ -1,16 +1,11 @@
 from __future__ import annotations
 import copy
 import itertools
-from typing import Optional, List, Callable, Type, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.scenarios.ScenarioList import ScenarioList
+from typing import Optional, List, Callable, Type
+from typing import TypeVar
 
 
 class QuestionBaseGenMixin:
-    """Mixin for QuestionBase."""
-
     def copy(self) -> QuestionBase:
         """Return a deep copy of the question.
 
@@ -44,31 +39,6 @@ class QuestionBaseGenMixin:
             questions.append(question)
         return questions
 
-    def draw(self) -> "QuestionBase":
-        """Return a new question with a randomly selected permutation of the options.
-
-        If the question has no options, returns a copy of the original question.
-
-        >>> from edsl.questions.QuestionMultipleChoice import QuestionMultipleChoice as Q
-        >>> q = Q.example()
-        >>> drawn = q.draw()
-        >>> len(drawn.question_options) == len(q.question_options)
-        True
-        >>> q is drawn
-        False
-        """
-
-        if not hasattr(self, "question_options"):
-            return copy.deepcopy(self)
-
-        import random
-
-        question = copy.deepcopy(self)
-        question.question_options = list(
-            random.sample(self.question_options, len(self.question_options))
-        )
-        return question
-
     def loop(self, scenario_list: ScenarioList) -> List[QuestionBase]:
         """Return a list of questions with the question name modified for each scenario.
 
@@ -80,22 +50,57 @@ class QuestionBaseGenMixin:
         >>> len(q.loop(ScenarioList.from_list("subject", ["Math", "Economics", "Chemistry"])))
         3
         """
-        from edsl.questions.loop_processor import LoopProcessor
+        from edsl.questions.LoopProcessor import LoopProcessor
 
         lp = LoopProcessor(self)
         return lp.process_templates(scenario_list)
 
-    def render(self, replacement_dict: dict) -> "QuestionBase":
-        """Render the question components as jinja2 templates with the replacement dictionary.
-
-        :param replacement_dict: The dictionary of values to replace in the question components.
-
-        >>> from edsl.questions.QuestionFreeText import QuestionFreeText
-        >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite {{ thing }}?")
-        >>> q.render({"thing": "color"})
-        Question('free_text', question_name = \"""color\""", question_text = \"""What is your favorite color?\""")
+        # from jinja2 import Environment
+        # from edsl.questions.QuestionBase import QuestionBase
+
+        # starting_name = self.question_name
+        # questions = []
+        # for index, scenario in enumerate(scenario_list):
+        #     env = Environment()
+        #     new_data = self.to_dict().copy()
+        #     for key, value in [(k, v) for k, v in new_data.items() if v is not None]:
+        #         if (
+        #             isinstance(value, str) or isinstance(value, int)
+        #         ) and key != "question_options":
+        #             new_data[key] = env.from_string(value).render(scenario)
+        #         elif isinstance(value, list):
+        #             new_data[key] = [
+        #                 env.from_string(v).render(scenario) if isinstance(v, str) else v
+        #                 for v in value
+        #             ]
+        #         elif isinstance(value, dict):
+        #             new_data[key] = {
+        #                 (
+        #                     env.from_string(k).render(scenario)
+        #                     if isinstance(k, str)
+        #                     else k
+        #                 ): (
+        #                     env.from_string(v).render(scenario)
+        #                     if isinstance(v, str)
+        #                     else v
+        #                 )
+        #                 for k, v in value.items()
+        #             }
+        #         elif key == "question_options" and isinstance(value, str):
+        #             new_data[key] = value
+        #         else:
+        #             raise ValueError(
+        #                 f"Unexpected value type: {type(value)} for key '{key}'"
+        #             )
+
+        #     if new_data["question_name"] == starting_name:
+        #         new_data["question_name"] = new_data["question_name"] + f"_{index}"
+
+        #     questions.append(QuestionBase.from_dict(new_data))
+        # return questions
 
-        """
+    def render(self, replacement_dict: dict) -> "QuestionBase":
+        """Render the question components as jinja2 templates with the replacement dictionary."""
         from jinja2 import Environment
         from edsl.scenarios.Scenario import Scenario
 
@@ -122,23 +127,15 @@ class QuestionBaseGenMixin:
 
         return self.apply_function(render_string)
 
-    def apply_function(
-        self, func: Callable, exclude_components: List[str] = None
-    ) -> QuestionBase:
+    def apply_function(self, func: Callable, exclude_components=None) -> QuestionBase:
         """Apply a function to the question parts
 
-        :param func: The function to apply to the question parts.
-        :param exclude_components: The components to exclude from the function application.
-
         >>> from edsl.questions import QuestionFreeText
         >>> q = QuestionFreeText(question_name = "color", question_text = "What is your favorite color?")
         >>> shouting = lambda x: x.upper()
         >>> q.apply_function(shouting)
         Question('free_text', question_name = \"""color\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
 
-        >>> q.apply_function(shouting, exclude_components = ["question_type"])
-        Question('free_text', question_name = \"""COLOR\""", question_text = \"""WHAT IS YOUR FAVORITE COLOR?\""")
-
         """
         from edsl.questions.QuestionBase import QuestionBase
 

edsl/questions/QuestionBasePromptsMixin.py

@@ -69,7 +69,7 @@ class QuestionBasePromptsMixin:
         >>> q.get_instructions(model = "gpt3")
         Prompt(text=\"""{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}\""")
         """
-        from edsl.language_models.model import Model
+        from edsl.language_models.registry import Model
 
         if not hasattr(self, "_model_instructions"):
             self._model_instructions = {}

edsl/questions/QuestionBudget.py

@@ -5,7 +5,7 @@ from pydantic import Field, BaseModel, validator
 
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
-from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
 
 
 class BudgewResponseValidator(ResponseValidatorABC):

edsl/questions/QuestionCheckBox.py

@@ -13,8 +13,8 @@ from edsl.questions.descriptors import (
 from edsl.questions.decorators import inject_exception
 
 from pydantic import field_validator
-from edsl.questions.response_validator_abc import ResponseValidatorABC
-from edsl.questions.data_structures import BaseResponse
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
 
 from edsl.exceptions.questions import QuestionAnswerValidationError
 

edsl/questions/QuestionExtract.py

@@ -6,8 +6,9 @@ from typing import Any, Optional, Dict
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import AnswerTemplateDescriptor
 
-from edsl.questions.response_validator_abc import ResponseValidatorABC
-from edsl.questions.data_structures import BaseResponse
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.decorators import inject_exception
 
 from typing import Dict, Any
@@ -56,7 +57,7 @@ def dict_to_pydantic_model(input_dict: Dict[str, Any]) -> Any:
     DynamicModel = create_model("DynamicModel", **field_definitions)
 
     class AnswerModel(BaseResponse):
-        answer: "DynamicModel"
+        answer: DynamicModel
         generated_tokens: Optional[str] = None
         comment: Optional[str] = None
 
@@ -112,8 +113,6 @@ class QuestionExtract(QuestionBase):
         :param question_name: The name of the question.
         :param question_text: The text of the question.
         :param answer_template: The template for the answer.
-        :param answering_instructions: Instructions for answering the question.
-        :param question_presentation: The presentation of the question.
         """
         self.question_name = question_name
         self.question_text = question_text
@@ -143,6 +142,9 @@ class QuestionExtract(QuestionBase):
         )
         return question_html_content
 
+    ################
+    # Helpful methods
+    ################
     @classmethod
     @inject_exception
     def example(cls) -> QuestionExtract:

edsl/questions/QuestionFreeText.py

@@ -5,7 +5,7 @@ from uuid import uuid4
 from pydantic import field_validator
 
 from edsl.questions.QuestionBase import QuestionBase
-from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
 
 from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.decorators import inject_exception

edsl/questions/QuestionList.py

@@ -1,18 +1,21 @@
 from __future__ import annotations
 import json
-from typing import Any, Optional, Union
-
-from pydantic import Field
-from json_repair import repair_json
 
-from edsl.exceptions.questions import QuestionAnswerValidationError
+from typing import Any, Optional, Union
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerOrNoneDescriptor
 from edsl.questions.decorators import inject_exception
-from edsl.questions.response_validator_abc import ResponseValidatorABC
 
+from pydantic import field_validator, Field
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import BaseResponse
+
+from edsl.exceptions.questions import QuestionAnswerValidationError
+
+from json_repair import repair_json
 
-def convert_string(s: str) -> Union[float, int, str, dict]:
+
+def convert_string(s):
     """Convert a string to a more appropriate type if possible.
 
     >>> convert_string("3.14")
@@ -55,7 +58,7 @@ def convert_string(s: str) -> Union[float, int, str, dict]:
     return s
 
 
-def create_model(max_list_items: int, permissive: bool) -> "ListResponse":
+def create_model(max_list_items: int, permissive):
     from pydantic import BaseModel
 
     if permissive or max_list_items is None:
@@ -130,8 +133,8 @@ class QuestionList(QuestionBase):
         self,
         question_name: str,
         question_text: str,
-        include_comment: bool = True,
         max_list_items: Optional[int] = None,
+        include_comment: bool = True,
         answering_instructions: Optional[str] = None,
         question_presentation: Optional[str] = None,
         permissive: bool = False,
@@ -181,6 +184,9 @@ class QuestionList(QuestionBase):
         ).render(question_name=self.question_name)
         return question_html_content
 
+    ################
+    # Helpful methods
+    ################
     @classmethod
     @inject_exception
     def example(

edsl/questions/QuestionMatrix.py

@@ -10,7 +10,7 @@ from edsl.questions.descriptors import (
     OptionLabelDescriptor,
     QuestionTextDescriptor,
 )
-from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
 from edsl.questions.decorators import inject_exception
 from edsl.exceptions.questions import (
     QuestionAnswerValidationError,

edsl/questions/QuestionMultipleChoice.py

@@ -8,7 +8,7 @@ from edsl.scenarios.Scenario import Scenario
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import QuestionOptionsDescriptor
 from edsl.questions.decorators import inject_exception
-from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
 
 
 def create_response_model(choices: List[str], permissive: bool = False):

edsl/questions/QuestionNumerical.py

@@ -9,7 +9,7 @@ from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import NumericalOrNoneDescriptor
 from edsl.questions.decorators import inject_exception
-from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
 
 
 def create_numeric_response(