edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev4__py3-none-any.whl

edsl/language_models/model.py

@@ -0,0 +1,256 @@
+import textwrap
+from random import random
+from typing import Optional, TYPE_CHECKING, List
+
+from edsl.utilities.PrettyList import PrettyList
+from edsl.config import CONFIG
+
+from edsl.inference_services.InferenceServicesCollection import (
+    InferenceServicesCollection,
+)
+from edsl.inference_services.data_structures import AvailableModels
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.enums import InferenceServiceLiteral
+
+if TYPE_CHECKING:
+    from edsl.results.Dataset import Dataset
+
+
+def get_model_class(model_name, registry: Optional[InferenceServicesCollection] = None):
+    from edsl.inference_services.registry import default
+
+    registry = registry or default
+    factory = registry.create_model_factory(model_name)
+    return factory
+
+
+class Meta(type):
+    def __repr__(cls):
+        return textwrap.dedent(
+            f"""\
+        Available models: {cls.available()}
+        
+        To create an instance, you can do: 
+        >>> m = Model('gpt-4-1106-preview', temperature=0.5, ...)
+        
+        To get the default model, you can leave out the model name. 
+        To see the available models, you can do:
+        >>> Model.available()
+        """
+        )
+
+
+class Model(metaclass=Meta):
+    default_model = CONFIG.get("EDSL_DEFAULT_MODEL")
+    _registry: InferenceServicesCollection = None  # Class-level registry storage
+
+    @classmethod
+    def get_registry(cls) -> InferenceServicesCollection:
+        """Get the current registry or initialize with default if None"""
+        if cls._registry is None:
+            from edsl.inference_services.registry import default
+
+            cls._registry = default
+        return cls._registry
+
+    @classmethod
+    def set_registry(cls, registry: InferenceServicesCollection) -> None:
+        """Set a new registry"""
+        cls._registry = registry
+
+    def __new__(
+        cls,
+        model_name: Optional[str] = None,
+        service_name: Optional[InferenceServiceLiteral] = None,
+        registry: Optional[InferenceServicesCollection] = None,
+        *args,
+        **kwargs,
+    ):
+        "Instantiate a new language model."
+        # Map index to the respective subclass
+        if model_name is None:
+            model_name = (
+                cls.default_model
+            )  # when model_name is None, use the default model, set in the config file
+
+        if registry is not None:
+            cls.set_registry(registry)
+
+        if isinstance(model_name, int):  # can refer to a model by index
+            model_name = cls.available(name_only=True)[model_name]
+
+        factory = cls.get_registry().create_model_factory(
+            model_name, service_name=service_name
+        )
+        return factory(*args, **kwargs)
+
+    @classmethod
+    def add_model(cls, service_name, model_name) -> None:
+        cls.get_registry().add_model(service_name, model_name)
+
+    @classmethod
+    def service_classes(cls) -> List["InferenceServiceABC"]:
+        """Returns a list of service classes.
+
+        >>> Model.service_classes()
+        [...]
+        """
+        return [r for r in cls.services(name_only=True)]
+
+    @classmethod
+    def services(cls, name_only: bool = False) -> List[str]:
+        """Returns a list of services, annotated with whether the user has local keys for them."""
+        services_with_local_keys = set(cls.key_info().select("service").to_list())
+        f = lambda service_name: (
+            "yes" if service_name in services_with_local_keys else " "
+        )
+        if name_only:
+            return PrettyList(
+                [r._inference_service_ for r in cls.get_registry().services],
+                columns=["Service Name"],
+            )
+        else:
+            return PrettyList(
+                [
+                    (r._inference_service_, f(r._inference_service_))
+                    for r in cls.get_registry().services
+                ],
+                columns=["Service Name", "Local key?"],
+            )
+
+    @classmethod
+    def services_with_local_keys(cls) -> set:
+        """Returns a list of services for which the user has local keys."""
+        return set(cls.key_info().select("service").to_list())
+
+    @classmethod
+    def key_info(cls, obscure_api_key: bool = True) -> "Dataset":
+        """Returns a dataset of local key information."""
+        from edsl.language_models.key_management.KeyLookupCollection import (
+            KeyLookupCollection,
+        )
+        from edsl.scenarios import Scenario, ScenarioList
+
+        klc = KeyLookupCollection()
+        klc.add_key_lookup(fetch_order=None)
+        sl = ScenarioList()
+        for service, entry in list(klc.data.values())[0].items():
+            sl.append(Scenario({"service": service} | entry.to_dict()))
+        if obscure_api_key:
+            for service in sl:
+                service["api_token"] = (
+                    service["api_token"][:4] + "..." + service["api_token"][-4:]
+                )
+        return sl.to_dataset()
+
+    @classmethod
+    def search_models(cls, search_term: str):
+        return cls.available(search_term=search_term)
+
+    @classmethod
+    def all_known_models(cls) -> "AvailableModels":
+        return cls.get_registry().available()
+
+    @classmethod
+    def available_with_local_keys(cls):
+        services_with_local_keys = set(cls.key_info().select("service").to_list())
+        return [
+            m
+            for m in cls.get_registry().available()
+            if m.service_name in services_with_local_keys
+        ]
+
+    @classmethod
+    def available(
+        cls,
+        search_term: str = None,
+        name_only: bool = False,
+        service: Optional[str] = None,
+    ):
+        # if search_term is None and service is None:
+        #     print("Getting available models...")
+        #     print("You have local keys for the following services:")
+        #     print(cls.services_with_local_keys())
+        #     print("\n")
+        #     print("To see models by service, use the 'service' parameter.")
+        #     print("E.g., Model.available(service='openai')")
+        #     return None
+
+        if service is not None:
+            if service not in cls.services(name_only=True):
+                raise ValueError(
+                    f"Service {service} not found in available services.",
+                    f"Available services are: {cls.services()}",
+                )
+
+        full_list = cls.get_registry().available(service=service)
+
+        if search_term is None:
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in full_list],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+        else:
+            filtered_results = [
+                m
+                for m in full_list
+                if search_term in m.model_name or search_term in m.service_name
+            ]
+            if name_only:
+                return PrettyList(
+                    [m.model_name for m in filtered_results],
+                    columns=["Model Name"],
+                )
+            else:
+                return PrettyList(
+                    [[m.model_name, m.service_name] for m in full_list],
+                    columns=["Model Name", "Service Name"],
+                )
+
+    @classmethod
+    def check_models(cls, verbose=False):
+        print("Checking all available models...\n")
+        for model in cls.available(name_only=True):
+            print(f"Now checking: {model}")
+            try:
+                m = cls(model)
+            except Exception as e:
+                print(f"Error creating instance of {model}: {e}")
+                continue
+            try:
+                results = m.hello(verbose)
+                if verbose:
+                    print(f"Results from model call: {results}")
+            except Exception as e:
+                print(f"Error calling 'hello' on {model}: {e}")
+                continue
+            print("OK!")
+            print("\n")
+
+    @classmethod
+    def example(cls, randomize: bool = False) -> "Model":
+        """
+        Returns an example Model instance.
+
+        :param randomize: If True, the temperature is set to a random decimal between 0 and 1.
+        """
+        temperature = 0.5 if not randomize else round(random(), 2)
+        model_name = cls.default_model
+        return cls(model_name, temperature=temperature)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+
+    available = Model.available()
+    m = Model("gpt-4-1106-preview")
+    results = m.execute_model_call("Hello world")
+    print(results)

edsl/language_models/repair.py

@@ -32,7 +32,7 @@ async def async_repair(
     else:
         return valid_dict, success
 
-    from edsl.language_models.registry import Model
+    from edsl.language_models.model import Model
 
     m = Model()
 

edsl/questions/QuestionBase.py

@@ -2,23 +2,31 @@
 
 from __future__ import annotations
 from abc import ABC, abstractmethod
-from typing import Any, Type, Optional, List, Callable, Union, TypedDict
+from typing import Any, Type, Optional, List, Callable, Union, TypedDict, TYPE_CHECKING
 
 from edsl.exceptions.questions import (
     QuestionSerializationError,
 )
 from edsl.questions.descriptors import QuestionNameDescriptor, QuestionTextDescriptor
 
-from edsl.questions.AnswerValidatorMixin import AnswerValidatorMixin
-from edsl.questions.RegisterQuestionsMeta import RegisterQuestionsMeta
+from edsl.questions.answer_validator_mixin import AnswerValidatorMixin
+from edsl.questions.register_questions_meta import RegisterQuestionsMeta
 from edsl.Base import PersistenceMixin, RepresentationMixin
 from edsl.BaseDiff import BaseDiff, BaseDiffCollection
 
 from edsl.questions.SimpleAskMixin import SimpleAskMixin
 from edsl.questions.QuestionBasePromptsMixin import QuestionBasePromptsMixin
-from edsl.questions.QuestionBaseGenMixin import QuestionBaseGenMixin
+from edsl.questions.question_base_gen_mixin import QuestionBaseGenMixin
 from edsl.utilities.remove_edsl_version import remove_edsl_version
 
+if TYPE_CHECKING:
+    from edsl.questions.response_validator_abc import ResponseValidatorABC
+    from edsl.language_models.LanguageModel import LanguageModel
+    from edsl.results.Results import Results
+    from edsl.agents.Agent import Agent
+    from edsl.surveys.Survey import Survey
+    from edsl.jobs.Jobs import Jobs
+
 
 class QuestionBase(
     PersistenceMixin,
@@ -49,13 +57,17 @@ class QuestionBase(
     _question_presentation = None
 
     @property
-    def response_validator(self) -> "ResponseValidatorBase":
+    def response_validator(self) -> "ResponseValidatorABC":
         """Return the response validator."""
-        from edsl.questions.ResponseValidatorFactory import ResponseValidatorFactory
+        from edsl.questions.response_validator_factory import ResponseValidatorFactory
 
         rvf = ResponseValidatorFactory(self)
         return rvf.response_validator
 
+    def duplicate(self):
+        """Return a duplicate of the question."""
+        return self.from_dict(self.to_dict())
+
     @property
     def fake_data_factory(self):
         """Return the fake data factory."""
@@ -159,7 +171,7 @@ class QuestionBase(
 
         return candidate_data
 
-    def to_dict(self, add_edsl_version=True):
+    def to_dict(self, add_edsl_version: bool = True):
         """Convert the question to a dictionary that includes the question type (used in deserialization).
 
         >>> from edsl.questions import QuestionFreeText as Q; Q.example().to_dict(add_edsl_version = False)
@@ -213,9 +225,6 @@ class QuestionBase(
 
         return question_class(**local_data)
 
-    # endregion
-
-    # region: Running methods
     @classmethod
     def _get_test_model(self, canned_response: Optional[str] = None) -> "LanguageModel":
         """Get a test model for the question."""
@@ -243,7 +252,7 @@ class QuestionBase(
         Dataset([{'answer.how_are_you': ["Yo, what's up?"]}])
         """
         if model is None:
-            from edsl.language_models.registry import Model
+            from edsl.language_models.model import Model
 
             model = Model()
         results = (
@@ -262,21 +271,22 @@ class QuestionBase(
 
     def __call__(
         self,
-        just_answer=True,
-        model=None,
-        agent=None,
+        just_answer: bool = True,
+        model: Optional["LanguageModel"] = None,
+        agent: Optional["Agent"] = None,
         disable_remote_cache: bool = False,
         disable_remote_inference: bool = False,
         verbose: bool = False,
         **kwargs,
-    ):
+    ) -> Union[Any, "Results"]:
         """Call the question.
 
 
         >>> from edsl import QuestionFreeText as Q
-        >>> m = Q._get_test_model(canned_response = "Yo, what's up?")
+        >>> from edsl import Model
+        >>> m = Model("test", canned_response = "Yo, what's up?")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True)
+        >>> q(model = m, disable_remote_cache = True, disable_remote_inference = True, cache = False)
         "Yo, what's up?"
 
         """
@@ -285,7 +295,6 @@ class QuestionBase(
             model=model,
             agent=agent,
             **kwargs,
-            cache=False,
             verbose=verbose,
             disable_remote_cache=disable_remote_cache,
             disable_remote_inference=disable_remote_inference,
@@ -297,15 +306,16 @@ class QuestionBase(
 
     def run(self, *args, **kwargs) -> "Results":
         """Turn a single question into a survey and runs it."""
-        from edsl.surveys.Survey import Survey
+        return self.to_survey().run(*args, **kwargs)
 
-        s = self.to_survey()
-        return s.run(*args, **kwargs)
+    def using(self, *args, **kwargs) -> "Jobs":
+        """Turn a single question into a survey and then a Job."""
+        return self.to_survey().to_jobs().using(*args, **kwargs)
 
     async def run_async(
         self,
         just_answer: bool = True,
-        model: Optional["Model"] = None,
+        model: Optional["LanguageModel"] = None,
         agent: Optional["Agent"] = None,
         disable_remote_inference: bool = False,
         **kwargs,
@@ -316,7 +326,7 @@ class QuestionBase(
         >>> from edsl.questions import QuestionFreeText as Q
         >>> m = Q._get_test_model(canned_response = "Blue")
         >>> q = Q(question_name = "color", question_text = "What is your favorite color?")
-        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True); print(result)
+        >>> async def test_run_async(): result = await q.run_async(model=m, disable_remote_inference = True, disable_remote_cache = True); print(result)
         >>> asyncio.run(test_run_async())
         Blue
         """
@@ -420,8 +430,7 @@ class QuestionBase(
         """
         from edsl.surveys.Survey import Survey
 
-        s = Survey([self])
-        return s
+        return Survey([self])
 
     def by(self, *args) -> "Jobs":
         """Turn a single question into a survey and then a Job."""
@@ -464,7 +473,7 @@ class QuestionBase(
 
     @classmethod
     def example_model(cls):
-        from edsl.language_models.registry import Model
+        from edsl.language_models.model import Model
 
         q = cls.example()
         m = Model("test", canned_response=cls._simulate_answer(q)["answer"])

edsl/questions/QuestionBasePromptsMixin.py

@@ -69,7 +69,7 @@ class QuestionBasePromptsMixin:
         >>> q.get_instructions(model = "gpt3")
         Prompt(text=\"""{{question_text}}. Answer in valid JSON like so {'answer': 'comment: <>}\""")
         """
-        from edsl.language_models.registry import Model
+        from edsl.language_models.model import Model
 
         if not hasattr(self, "_model_instructions"):
             self._model_instructions = {}

edsl/questions/QuestionBudget.py

@@ -5,7 +5,7 @@ from pydantic import Field, BaseModel, validator
 
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import IntegerDescriptor, QuestionOptionsDescriptor
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
 class BudgewResponseValidator(ResponseValidatorABC):

edsl/questions/QuestionCheckBox.py

@@ -13,8 +13,8 @@ from edsl.questions.descriptors import (
 from edsl.questions.decorators import inject_exception
 
 from pydantic import field_validator
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.questions.ResponseValidatorABC import BaseResponse
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.data_structures import BaseResponse
 
 from edsl.exceptions.questions import QuestionAnswerValidationError
 

edsl/questions/QuestionExtract.py

@@ -6,9 +6,8 @@ from typing import Any, Optional, Dict
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import AnswerTemplateDescriptor
 
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.questions.ResponseValidatorABC import BaseResponse
-from edsl.exceptions.questions import QuestionAnswerValidationError
+from edsl.questions.response_validator_abc import ResponseValidatorABC
+from edsl.questions.data_structures import BaseResponse
 from edsl.questions.decorators import inject_exception
 
 from typing import Dict, Any
@@ -57,7 +56,7 @@ def dict_to_pydantic_model(input_dict: Dict[str, Any]) -> Any:
     DynamicModel = create_model("DynamicModel", **field_definitions)
 
     class AnswerModel(BaseResponse):
-        answer: DynamicModel
+        answer: "DynamicModel"
         generated_tokens: Optional[str] = None
         comment: Optional[str] = None
 
@@ -113,6 +112,8 @@ class QuestionExtract(QuestionBase):
         :param question_name: The name of the question.
         :param question_text: The text of the question.
         :param answer_template: The template for the answer.
+        :param answering_instructions: Instructions for answering the question.
+        :param question_presentation: The presentation of the question.
         """
         self.question_name = question_name
         self.question_text = question_text
@@ -142,9 +143,6 @@ class QuestionExtract(QuestionBase):
         )
         return question_html_content
 
-    ################
-    # Helpful methods
-    ################
     @classmethod
     @inject_exception
     def example(cls) -> QuestionExtract:

edsl/questions/QuestionFreeText.py

@@ -5,7 +5,7 @@ from uuid import uuid4
 from pydantic import field_validator
 
 from edsl.questions.QuestionBase import QuestionBase
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.decorators import inject_exception

edsl/questions/QuestionList.py

@@ -1,21 +1,18 @@
 from __future__ import annotations
 import json
-
 from typing import Any, Optional, Union
-from edsl.questions.QuestionBase import QuestionBase
-from edsl.questions.descriptors import IntegerOrNoneDescriptor
-from edsl.questions.decorators import inject_exception
 
-from pydantic import field_validator, Field
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
-from edsl.questions.ResponseValidatorABC import BaseResponse
+from pydantic import Field
+from json_repair import repair_json
 
 from edsl.exceptions.questions import QuestionAnswerValidationError
-
-from json_repair import repair_json
+from edsl.questions.QuestionBase import QuestionBase
+from edsl.questions.descriptors import IntegerOrNoneDescriptor
+from edsl.questions.decorators import inject_exception
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
-def convert_string(s):
+def convert_string(s: str) -> Union[float, int, str, dict]:
     """Convert a string to a more appropriate type if possible.
 
     >>> convert_string("3.14")
@@ -58,7 +55,7 @@ def convert_string(s):
     return s
 
 
-def create_model(max_list_items: int, permissive):
+def create_model(max_list_items: int, permissive: bool) -> "ListResponse":
     from pydantic import BaseModel
 
     if permissive or max_list_items is None:
@@ -133,8 +130,8 @@ class QuestionList(QuestionBase):
         self,
         question_name: str,
         question_text: str,
-        max_list_items: Optional[int] = None,
         include_comment: bool = True,
+        max_list_items: Optional[int] = None,
         answering_instructions: Optional[str] = None,
         question_presentation: Optional[str] = None,
         permissive: bool = False,
@@ -184,9 +181,6 @@ class QuestionList(QuestionBase):
         ).render(question_name=self.question_name)
         return question_html_content
 
-    ################
-    # Helpful methods
-    ################
     @classmethod
     @inject_exception
     def example(

edsl/questions/QuestionMatrix.py

@@ -10,7 +10,7 @@ from edsl.questions.descriptors import (
     OptionLabelDescriptor,
     QuestionTextDescriptor,
 )
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 from edsl.questions.decorators import inject_exception
 from edsl.exceptions.questions import (
     QuestionAnswerValidationError,

edsl/questions/QuestionMultipleChoice.py

@@ -8,7 +8,7 @@ from edsl.scenarios.Scenario import Scenario
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import QuestionOptionsDescriptor
 from edsl.questions.decorators import inject_exception
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
 def create_response_model(choices: List[str], permissive: bool = False):

edsl/questions/QuestionNumerical.py

@@ -9,7 +9,7 @@ from edsl.exceptions.questions import QuestionAnswerValidationError
 from edsl.questions.QuestionBase import QuestionBase
 from edsl.questions.descriptors import NumericalOrNoneDescriptor
 from edsl.questions.decorators import inject_exception
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
 def create_numeric_response(

edsl/questions/QuestionRank.py

@@ -8,7 +8,7 @@ from edsl.questions.descriptors import (
     QuestionOptionsDescriptor,
     NumSelectionsDescriptor,
 )
-from edsl.questions.ResponseValidatorABC import ResponseValidatorABC
+from edsl.questions.response_validator_abc import ResponseValidatorABC
 
 
 def create_response_model(

edsl/questions/SimpleAskMixin.py

@@ -66,7 +66,7 @@ class SimpleAskMixin:
         system_prompt="You are a helpful agent pretending to be a human. Do not break character",
         top_logprobs=4,
     ):
-        from edsl.language_models.registry import Model
+        from edsl.language_models.model import Model
 
         if model is None:
             model = Model()

edsl/questions/__init__.py

@@ -1,6 +1,6 @@
 # Schemas
 from edsl.questions.settings import Settings
-from edsl.questions.RegisterQuestionsMeta import RegisterQuestionsMeta
+from edsl.questions.register_questions_meta import RegisterQuestionsMeta
 
 # Base Class
 from edsl.questions.QuestionBase import QuestionBase

edsl/questions/data_structures.py

@@ -0,0 +1,20 @@
+from typing import Any, Optional, TypedDict
+from pydantic import BaseModel
+
+
+class RawEdslAnswerDict(TypedDict):
+    answer: Any
+    comment: Optional[str]
+    generated_tokens: Optional[str]
+
+
+class BaseResponse(BaseModel):
+    answer: Any
+    comment: Optional[str] = None
+    generated_tokens: Optional[str] = None
+
+
+class EdslAnswerDict(TypedDict):
+    answer: Any
+    comment: Optional[str]
+    generated_tokens: Optional[str]