edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev4__py3-none-any.whl

edsl/inference_services/TestService.py

@@ -51,6 +51,7 @@ class TestService(InferenceServiceABC):
             @property
             def _canned_response(self):
                 if hasattr(self, "canned_response"):
+
                     return self.canned_response
                 else:
                     return "Hello, world"
@@ -63,9 +64,6 @@ class TestService(InferenceServiceABC):
                 files_list: Optional[List["File"]] = None,
             ) -> dict[str, Any]:
                 await asyncio.sleep(0.1)
-                # return {"message": """{"answer": "Hello, world"}"""}
-
-                # breakpoint()
 
                 if hasattr(self, "throw_exception") and self.throw_exception:
                     if hasattr(self, "exception_probability"):

edsl/inference_services/TogetherAIService.py

@@ -143,15 +143,17 @@ class TogetherAIService(OpenAIService):
     _async_client_ = openai.AsyncOpenAI
 
     @classmethod
-    def get_model_list(cls):
+    def get_model_list(cls, api_token=None):
         # Togheter.ai has a different response in model list then openai
         # and the OpenAI class returns an error when calling .models.list()
         import requests
         import os
 
         url = "https://api.together.xyz/v1/models?filter=serverless"
-        token = os.getenv(cls._env_key_name_)
-        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
+        if api_token is None:
+            api_token = os.getenv(cls._env_key_name_)
+
+        headers = {"accept": "application/json", "authorization": f"Bearer {api_token}"}
 
         response = requests.get(url, headers=headers)
         return response.json()

edsl/inference_services/data_structures.py

@@ -1,14 +1,33 @@
 from collections import UserDict, defaultdict, UserList
-from typing import Union
+from typing import Union, Optional, List
 from edsl.enums import InferenceServiceLiteral
 from dataclasses import dataclass
 
 
 @dataclass
 class LanguageModelInfo:
+    """A dataclass for storing information about a language model.
+
+
+    >>> LanguageModelInfo("gpt-4-1106-preview", "openai")
+    LanguageModelInfo(model_name='gpt-4-1106-preview', service_name='openai')
+
+    >>> model_name, service = LanguageModelInfo.example()
+    >>> model_name
+    'gpt-4-1106-preview'
+
+    >>> LanguageModelInfo.example().service_name
+    'openai'
+
+    """
+
     model_name: str
     service_name: str
 
+    def __iter__(self):
+        yield self.model_name
+        yield self.service_name
+
     def __getitem__(self, key: int) -> str:
         import warnings
 
@@ -26,13 +45,18 @@ class LanguageModelInfo:
         else:
             raise IndexError("Index out of range")
 
+    @classmethod
+    def example(cls) -> "LanguageModelInfo":
+        return cls("gpt-4-1106-preview", "openai")
+
 
 class ModelNamesList(UserList):
     pass
 
 
 class AvailableModels(UserList):
-    def __init__(self, data: list) -> None:
+
+    def __init__(self, data: List[LanguageModelInfo]) -> None:
         super().__init__(data)
 
     def __contains__(self, model_name: str) -> bool:
@@ -41,6 +65,54 @@ class AvailableModels(UserList):
                 return True
         return False
 
+    def print(self):
+        return self.to_dataset().print()
+
+    def to_dataset(self):
+        from edsl.scenarios.ScenarioList import ScenarioList
+
+        models, services = zip(
+            *[(model.model_name, model.service_name) for model in self]
+        )
+        return (
+            ScenarioList.from_list("model", models)
+            .add_list("service", services)
+            .to_dataset()
+        )
+
+    def to_model_list(self):
+        from edsl.language_models.ModelList import ModelList
+
+        return ModelList.from_available_models(self)
+
+    def search(
+        self, pattern: str, service_name: Optional[str] = None, regex: bool = False
+    ) -> "AvailableModels":
+        import re
+
+        if not regex:
+            # Escape special regex characters except *
+            pattern = re.escape(pattern).replace(r"\*", ".*")
+
+        try:
+            regex = re.compile(pattern)
+            avm = AvailableModels(
+                [
+                    entry
+                    for entry in self
+                    if regex.search(entry.model_name)
+                    and (service_name is None or entry.service_name == service_name)
+                ]
+            )
+            if len(avm) == 0:
+                raise ValueError(
+                    "No models found matching the search pattern: " + pattern
+                )
+            else:
+                return avm
+        except re.error as e:
+            raise ValueError(f"Invalid regular expression pattern: {e}")
+
 
 class ServiceToModelsMapping(UserDict):
     def __init__(self, data: dict) -> None:

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -5,6 +5,8 @@ from typing import Union, Type, Callable, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
+    from edsl.jobs.interviews.Interview import Interview
+    from edsl.language_models.key_management.KeyLookup import KeyLookup
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.tasks.task_status_enum import TaskStatus
@@ -17,34 +19,97 @@ from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
 from edsl.jobs.Answers import Answers
 
 
-class AnswerQuestionFunctionConstructor:
-    def __init__(self, interview):
+class RetryConfig:
+    from edsl.config import CONFIG
+
+    EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+    EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
+    EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+
+
+class SkipHandler:
+
+    def __init__(self, interview: "Interview"):
         self.interview = interview
-        self.had_language_model_no_response_error = False
         self.question_index = self.interview.to_index
 
         self.skip_function: Callable = (
             self.interview.survey.rule_collection.skip_question_before_running
         )
 
-    def _combined_answers(self) -> Answers:
-        return self.answers | self.interview.scenario | self.interview.agent["traits"]
-
-    @property
-    def answers(self) -> Answers:
-        return self.interview.answers
-
-    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
+    def should_skip(self, current_question: "QuestionBase") -> bool:
+        """Determine if the current question should be skipped."""
         current_question_index = self.question_index[current_question.question_name]
-        combined_answers = self._combined_answers()
+        combined_answers = (
+            self.interview.answers
+            | self.interview.scenario
+            | self.interview.agent["traits"]
+        )
         return self.skip_function(current_question_index, combined_answers)
 
+    def cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
+        """Cancel the tasks for questions that should be skipped."""
+        current_question_index: int = self.interview.to_index[
+            current_question.question_name
+        ]
+        answers = (
+            self.interview.answers
+            | self.interview.scenario
+            | self.interview.agent["traits"]
+        )
+
+        # Get the index of the next question, which could also be the end of the survey
+        next_question: Union[int, EndOfSurvey] = (
+            self.interview.survey.rule_collection.next_question(
+                q_now=current_question_index,
+                answers=answers,
+            )
+        )
+
+        def cancel_between(start, end):
+            """Cancel the tasks for questions between the start and end indices."""
+            for i in range(start, end):
+                self.interview.tasks[i].cancel()
+
+        if (next_question_index := next_question.next_q) == EndOfSurvey:
+            cancel_between(
+                current_question_index + 1, len(self.interview.survey.questions)
+            )
+            return
+
+        if next_question_index > (current_question_index + 1):
+            cancel_between(current_question_index + 1, next_question_index)
+
+
+class AnswerQuestionFunctionConstructor:
+    """Constructs a function that answers a question and records the answer."""
+
+    def __init__(self, interview: "Interview", key_lookup: "KeyLookup"):
+        self.interview = interview
+        self.key_lookup = key_lookup
+
+        self.had_language_model_no_response_error: bool = False
+        self.question_index = self.interview.to_index
+
+        self.skip_function: Callable = (
+            self.interview.survey.rule_collection.skip_question_before_running
+        )
+
+        self.invigilator_fetcher = FetchInvigilator(
+            self.interview, key_lookup=self.key_lookup
+        )
+        self.skip_handler = SkipHandler(self.interview)
+
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
+        """Handle an exception that occurred while answering a question."""
+
         from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-        answers = copy.copy(self.answers)  # copy to freeze the answers here for logging
+        answers = copy.copy(
+            self.interview.answers
+        )  # copy to freeze the answers here for logging
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
@@ -52,6 +117,7 @@ class AnswerQuestionFunctionConstructor:
         )
         if task:
             task.task_status = TaskStatus.FAILED
+
         self.interview.exceptions.add(
             invigilator.question.question_name, exception_entry
         )
@@ -65,41 +131,15 @@ class AnswerQuestionFunctionConstructor:
         if stop_on_exception:
             raise e
 
-    def _cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
-        current_question_index: int = self.interview.to_index[
-            current_question.question_name
-        ]
-        answers = (
-            self.answers | self.interview.scenario | self.interview.agent["traits"]
-        )
-
-        # Get the index of the next question, which could also be the end of the survey
-        next_question: Union[
-            int, EndOfSurvey
-        ] = self.interview.survey.rule_collection.next_question(
-            q_now=current_question_index,
-            answers=answers,
-        )
-
-        def cancel_between(start, end):
-            for i in range(start, end):
-                self.interview.tasks[i].cancel()
-
-        if (next_question_index := next_question.next_q) == EndOfSurvey:
-            cancel_between(
-                current_question_index + 1, len(self.interview.survey.questions)
-            )
-            return
-
-        if next_question_index > (current_question_index + 1):
-            cancel_between(current_question_index + 1, next_question_index)
-
     def __call__(self):
-        from edsl.config import CONFIG
+        return self.answer_question_and_record_task
 
-        EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-        EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-        EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
+    async def answer_question_and_record_task(
+        self,
+        *,
+        question: "QuestionBase",
+        task=None,
+    ) -> "AgentResponseDict":
 
         from tenacity import (
             retry,
@@ -109,80 +149,75 @@ class AnswerQuestionFunctionConstructor:
             RetryError,
         )
 
-        async def answer_question_and_record_task(
-            *,
-            question: "QuestionBase",
-            task=None,
-        ) -> "AgentResponseDict":
-            @retry(
-                stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
-                wait=wait_exponential(
-                    multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
-                ),
-                retry=retry_if_exception_type(LanguageModelNoResponseError),
-                reraise=True,
-            )
-            async def attempt_answer():
-                invigilator = FetchInvigilator(self.interview)(question)
+        @retry(
+            stop=stop_after_attempt(RetryConfig.EDSL_MAX_ATTEMPTS),
+            wait=wait_exponential(
+                multiplier=RetryConfig.EDSL_BACKOFF_START_SEC,
+                max=RetryConfig.EDSL_BACKOFF_MAX_SEC,
+            ),
+            retry=retry_if_exception_type(LanguageModelNoResponseError),
+            reraise=True,
+        )
+        async def attempt_answer():
+            invigilator = self.invigilator_fetcher(question)
 
-                if self._skip_this_question(question):
-                    return invigilator.get_failed_task_result(
-                        failure_reason="Question skipped."
-                    )
+            if self.skip_handler.should_skip(question):
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question skipped."
+                )
 
-                try:
-                    response: EDSLResultObjectInput = (
-                        await invigilator.async_answer_question()
-                    )
-                    if response.validated:
-                        self.answers.add_answer(response=response, question=question)
-                        self._cancel_skipped_questions(question)
-                    else:
-                        if (
-                            hasattr(response, "exception_occurred")
-                            and response.exception_occurred
-                        ):
-                            raise response.exception_occurred
-
-                except QuestionAnswerValidationError as e:
-                    self._handle_exception(e, invigilator, task)
-                    return invigilator.get_failed_task_result(
-                        failure_reason="Question answer validation failed."
+            try:
+                response: EDSLResultObjectInput = (
+                    await invigilator.async_answer_question()
+                )
+                if response.validated:
+                    self.interview.answers.add_answer(
+                        response=response, question=question
                     )
 
-                except asyncio.TimeoutError as e:
-                    self._handle_exception(e, invigilator, task)
-                    had_language_model_no_response_error = True
-                    raise LanguageModelNoResponseError(
-                        f"Language model timed out for question '{question.question_name}.'"
-                    )
+                    self.skip_handler.cancel_skipped_questions(question)
+                else:
+                    if (
+                        hasattr(response, "exception_occurred")
+                        and response.exception_occurred
+                    ):
+                        raise response.exception_occurred
+
+            except QuestionAnswerValidationError as e:
+                self._handle_exception(e, invigilator, task)
+                return invigilator.get_failed_task_result(
+                    failure_reason="Question answer validation failed."
+                )
 
-                except Exception as e:
-                    self._handle_exception(e, invigilator, task)
+            except asyncio.TimeoutError as e:
+                self._handle_exception(e, invigilator, task)
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model timed out for question '{question.question_name}.'"
+                )
 
-                if "response" not in locals():
-                    had_language_model_no_response_error = True
-                    raise LanguageModelNoResponseError(
-                        f"Language model did not return a response for question '{question.question_name}.'"
-                    )
+            except Exception as e:
+                self._handle_exception(e, invigilator, task)
 
-                if (
-                    question.question_name in self.interview.exceptions
-                    and had_language_model_no_response_error
-                ):
-                    self.interview.exceptions.record_fixed_question(
-                        question.question_name
-                    )
+            if "response" not in locals():
+                had_language_model_no_response_error = True
+                raise LanguageModelNoResponseError(
+                    f"Language model did not return a response for question '{question.question_name}.'"
+                )
 
-                return response
+            if (
+                question.question_name in self.interview.exceptions
+                and had_language_model_no_response_error
+            ):
+                self.interview.exceptions.record_fixed_question(question.question_name)
 
-            try:
-                return await attempt_answer()
-            except RetryError as retry_error:
-                original_error = retry_error.last_attempt.exception()
-                self._handle_exception(
-                    original_error, FetchInvigilator(self.interview)(question), task
-                )
-                raise original_error
+            return response
 
-        return answer_question_and_record_task
+        try:
+            return await attempt_answer()
+        except RetryError as retry_error:
+            original_error = retry_error.last_attempt.exception()
+            self._handle_exception(
+                original_error, self.invigilator_fetcher(question), task
+            )
+            raise original_error

edsl/jobs/FetchInvigilator.py

@@ -3,15 +3,23 @@ from typing import List, Dict, Any, Optional, TYPE_CHECKING
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
     from edsl.agents.InvigilatorBase import InvigilatorBase
+    from edsl.language_models.key_management.KeyLookup import KeyLookup
+    from edsl.jobs.interviews.Interview import Interview
 
 
 class FetchInvigilator:
-    def __init__(self, interview, current_answers: Optional[Dict[str, Any]] = None):
+    def __init__(
+        self,
+        interview: "Interview",
+        current_answers: Optional[Dict[str, Any]] = None,
+        key_lookup: Optional["KeyLookup"] = None,
+    ):
         self.interview = interview
         if current_answers is None:
             self.current_answers = self.interview.answers
         else:
             self.current_answers = current_answers
+        self.key_lookup = key_lookup
 
     def get_invigilator(self, question: "QuestionBase") -> "InvigilatorBase":
         """Return an invigilator for the given question.
@@ -24,14 +32,13 @@ class FetchInvigilator:
             question=question,
             scenario=self.interview.scenario,
             model=self.interview.model,
-            #            debug=False,
             survey=self.interview.survey,
             memory_plan=self.interview.survey.memory_plan,
             current_answers=self.current_answers,  # not yet known
             iteration=self.interview.iteration,
             cache=self.interview.cache,
-            #           sidecar_model=self.interview.sidecar_model,
             raise_validation_errors=self.interview.raise_validation_errors,
+            key_lookup=self.key_lookup,
         )
         """Return an invigilator for the given question."""
         return invigilator

edsl/jobs/InterviewsConstructor.py

@@ -6,8 +6,9 @@ if TYPE_CHECKING:
 
 
 class InterviewsConstructor:
-    def __init__(self, jobs):
+    def __init__(self, jobs: "Jobs", cache: "Cache"):
         self.jobs = jobs
+        self.cache = cache
 
     def create_interviews(self) -> Generator["Interview", None, None]:
         """
@@ -34,12 +35,13 @@ class InterviewsConstructor:
             self.jobs.agents, self.jobs.scenarios, self.jobs.models
         ):
             yield Interview(
-                survey=self.jobs.survey,
+                survey=self.jobs.survey.draw(),
                 agent=agent,
                 scenario=scenario,
                 model=model,
-                skip_retry=self.jobs.skip_retry,
-                raise_validation_errors=self.jobs.raise_validation_errors,
+                cache=self.cache,
+                skip_retry=self.jobs.run_config.parameters.skip_retry,
+                raise_validation_errors=self.jobs.run_config.parameters.raise_validation_errors,
                 indices={
                     "agent": agent_index[hash(agent)],
                     "model": model_index[hash(model)],