edsl 0.1.39__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/inference_services/TogetherAIService.py

@@ -143,17 +143,15 @@ class TogetherAIService(OpenAIService):
     _async_client_ = openai.AsyncOpenAI
 
     @classmethod
-    def get_model_list(cls, api_token=None):
+    def get_model_list(cls):
         # Togheter.ai has a different response in model list then openai
         # and the OpenAI class returns an error when calling .models.list()
         import requests
         import os
 
         url = "https://api.together.xyz/v1/models?filter=serverless"
-        if api_token is None:
-            api_token = os.getenv(cls._env_key_name_)
-
-        headers = {"accept": "application/json", "authorization": f"Bearer {api_token}"}
+        token = os.getenv(cls._env_key_name_)
+        headers = {"accept": "application/json", "authorization": f"Bearer {token}"}
 
         response = requests.get(url, headers=headers)
         return response.json()

edsl/inference_services/data_structures.py

@@ -1,33 +1,14 @@
 from collections import UserDict, defaultdict, UserList
-from typing import Union, Optional, List
+from typing import Union
 from edsl.enums import InferenceServiceLiteral
 from dataclasses import dataclass
 
 
 @dataclass
 class LanguageModelInfo:
-    """A dataclass for storing information about a language model.
-
-
-    >>> LanguageModelInfo("gpt-4-1106-preview", "openai")
-    LanguageModelInfo(model_name='gpt-4-1106-preview', service_name='openai')
-
-    >>> model_name, service = LanguageModelInfo.example()
-    >>> model_name
-    'gpt-4-1106-preview'
-
-    >>> LanguageModelInfo.example().service_name
-    'openai'
-
-    """
-
     model_name: str
     service_name: str
 
-    def __iter__(self):
-        yield self.model_name
-        yield self.service_name
-
     def __getitem__(self, key: int) -> str:
         import warnings
 
@@ -45,18 +26,13 @@ class LanguageModelInfo:
         else:
             raise IndexError("Index out of range")
 
-    @classmethod
-    def example(cls) -> "LanguageModelInfo":
-        return cls("gpt-4-1106-preview", "openai")
-
 
 class ModelNamesList(UserList):
     pass
 
 
 class AvailableModels(UserList):
-
-    def __init__(self, data: List[LanguageModelInfo]) -> None:
+    def __init__(self, data: list) -> None:
         super().__init__(data)
 
     def __contains__(self, model_name: str) -> bool:
@@ -65,54 +41,6 @@ class AvailableModels(UserList):
                 return True
         return False
 
-    def print(self):
-        return self.to_dataset().print()
-
-    def to_dataset(self):
-        from edsl.scenarios.ScenarioList import ScenarioList
-
-        models, services = zip(
-            *[(model.model_name, model.service_name) for model in self]
-        )
-        return (
-            ScenarioList.from_list("model", models)
-            .add_list("service", services)
-            .to_dataset()
-        )
-
-    def to_model_list(self):
-        from edsl.language_models.ModelList import ModelList
-
-        return ModelList.from_available_models(self)
-
-    def search(
-        self, pattern: str, service_name: Optional[str] = None, regex: bool = False
-    ) -> "AvailableModels":
-        import re
-
-        if not regex:
-            # Escape special regex characters except *
-            pattern = re.escape(pattern).replace(r"\*", ".*")
-
-        try:
-            regex = re.compile(pattern)
-            avm = AvailableModels(
-                [
-                    entry
-                    for entry in self
-                    if regex.search(entry.model_name)
-                    and (service_name is None or entry.service_name == service_name)
-                ]
-            )
-            if len(avm) == 0:
-                raise ValueError(
-                    "No models found matching the search pattern: " + pattern
-                )
-            else:
-                return avm
-        except re.error as e:
-            raise ValueError(f"Invalid regular expression pattern: {e}")
-
 
 class ServiceToModelsMapping(UserDict):
     def __init__(self, data: dict) -> None:

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -5,8 +5,6 @@ from typing import Union, Type, Callable, TYPE_CHECKING
 
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
-    from edsl.jobs.interviews.Interview import Interview
-    from edsl.language_models.key_management.KeyLookup import KeyLookup
 
 from edsl.surveys.base import EndOfSurvey
 from edsl.jobs.tasks.task_status_enum import TaskStatus
@@ -19,97 +17,34 @@ from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
 from edsl.jobs.Answers import Answers
 
 
-class RetryConfig:
-    from edsl.config import CONFIG
-
-    EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-    EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-    EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
-
-
-class SkipHandler:
-
-    def __init__(self, interview: "Interview"):
+class AnswerQuestionFunctionConstructor:
+    def __init__(self, interview):
         self.interview = interview
+        self.had_language_model_no_response_error = False
         self.question_index = self.interview.to_index
 
         self.skip_function: Callable = (
             self.interview.survey.rule_collection.skip_question_before_running
         )
 
-    def should_skip(self, current_question: "QuestionBase") -> bool:
-        """Determine if the current question should be skipped."""
-        current_question_index = self.question_index[current_question.question_name]
-        combined_answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
-        return self.skip_function(current_question_index, combined_answers)
-
-    def cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
-        """Cancel the tasks for questions that should be skipped."""
-        current_question_index: int = self.interview.to_index[
-            current_question.question_name
-        ]
-        answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
-
-        # Get the index of the next question, which could also be the end of the survey
-        next_question: Union[int, EndOfSurvey] = (
-            self.interview.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=answers,
-            )
-        )
+    def _combined_answers(self) -> Answers:
+        return self.answers | self.interview.scenario | self.interview.agent["traits"]
 
-        def cancel_between(start, end):
-            """Cancel the tasks for questions between the start and end indices."""
-            for i in range(start, end):
-                self.interview.tasks[i].cancel()
+    @property
+    def answers(self) -> Answers:
+        return self.interview.answers
 
-        if (next_question_index := next_question.next_q) == EndOfSurvey:
-            cancel_between(
-                current_question_index + 1, len(self.interview.survey.questions)
-            )
-            return
-
-        if next_question_index > (current_question_index + 1):
-            cancel_between(current_question_index + 1, next_question_index)
-
-
-class AnswerQuestionFunctionConstructor:
-    """Constructs a function that answers a question and records the answer."""
-
-    def __init__(self, interview: "Interview", key_lookup: "KeyLookup"):
-        self.interview = interview
-        self.key_lookup = key_lookup
-
-        self.had_language_model_no_response_error: bool = False
-        self.question_index = self.interview.to_index
-
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
-        )
-
-        self.invigilator_fetcher = FetchInvigilator(
-            self.interview, key_lookup=self.key_lookup
-        )
-        self.skip_handler = SkipHandler(self.interview)
+    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
+        current_question_index = self.question_index[current_question.question_name]
+        combined_answers = self._combined_answers()
+        return self.skip_function(current_question_index, combined_answers)
 
     def _handle_exception(
         self, e: Exception, invigilator: "InvigilatorBase", task=None
     ):
-        """Handle an exception that occurred while answering a question."""
-
         from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
 
-        answers = copy.copy(
-            self.interview.answers
-        )  # copy to freeze the answers here for logging
+        answers = copy.copy(self.answers)  # copy to freeze the answers here for logging
         exception_entry = InterviewExceptionEntry(
             exception=e,
             invigilator=invigilator,
@@ -117,7 +52,6 @@ class AnswerQuestionFunctionConstructor:
         )
         if task:
             task.task_status = TaskStatus.FAILED
-
         self.interview.exceptions.add(
             invigilator.question.question_name, exception_entry
         )
@@ -131,15 +65,41 @@ class AnswerQuestionFunctionConstructor:
         if stop_on_exception:
             raise e
 
+    def _cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
+        current_question_index: int = self.interview.to_index[
+            current_question.question_name
+        ]
+        answers = (
+            self.answers | self.interview.scenario | self.interview.agent["traits"]
+        )
+
+        # Get the index of the next question, which could also be the end of the survey
+        next_question: Union[
+            int, EndOfSurvey
+        ] = self.interview.survey.rule_collection.next_question(
+            q_now=current_question_index,
+            answers=answers,
+        )
+
+        def cancel_between(start, end):
+            for i in range(start, end):
+                self.interview.tasks[i].cancel()
+
+        if (next_question_index := next_question.next_q) == EndOfSurvey:
+            cancel_between(
+                current_question_index + 1, len(self.interview.survey.questions)
+            )
+            return
+
+        if next_question_index > (current_question_index + 1):
+            cancel_between(current_question_index + 1, next_question_index)
+
     def __call__(self):
-        return self.answer_question_and_record_task
+        from edsl.config import CONFIG
 
-    async def answer_question_and_record_task(
-        self,
-        *,
-        question: "QuestionBase",
-        task=None,
-    ) -> "AgentResponseDict":
+        EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
+        EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
+        EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
 
         from tenacity import (
             retry,
@@ -149,75 +109,80 @@ class AnswerQuestionFunctionConstructor:
             RetryError,
         )
 
-        @retry(
-            stop=stop_after_attempt(RetryConfig.EDSL_MAX_ATTEMPTS),
-            wait=wait_exponential(
-                multiplier=RetryConfig.EDSL_BACKOFF_START_SEC,
-                max=RetryConfig.EDSL_BACKOFF_MAX_SEC,
-            ),
-            retry=retry_if_exception_type(LanguageModelNoResponseError),
-            reraise=True,
-        )
-        async def attempt_answer():
-            invigilator = self.invigilator_fetcher(question)
+        async def answer_question_and_record_task(
+            *,
+            question: "QuestionBase",
+            task=None,
+        ) -> "AgentResponseDict":
+            @retry(
+                stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
+                wait=wait_exponential(
+                    multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
+                ),
+                retry=retry_if_exception_type(LanguageModelNoResponseError),
+                reraise=True,
+            )
+            async def attempt_answer():
+                invigilator = FetchInvigilator(self.interview)(question)
 
-            if self.skip_handler.should_skip(question):
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question skipped."
-                )
+                if self._skip_this_question(question):
+                    return invigilator.get_failed_task_result(
+                        failure_reason="Question skipped."
+                    )
 
-            try:
-                response: EDSLResultObjectInput = (
-                    await invigilator.async_answer_question()
-                )
-                if response.validated:
-                    self.interview.answers.add_answer(
-                        response=response, question=question
+                try:
+                    response: EDSLResultObjectInput = (
+                        await invigilator.async_answer_question()
+                    )
+                    if response.validated:
+                        self.answers.add_answer(response=response, question=question)
+                        self._cancel_skipped_questions(question)
+                    else:
+                        if (
+                            hasattr(response, "exception_occurred")
+                            and response.exception_occurred
+                        ):
+                            raise response.exception_occurred
+
+                except QuestionAnswerValidationError as e:
+                    self._handle_exception(e, invigilator, task)
+                    return invigilator.get_failed_task_result(
+                        failure_reason="Question answer validation failed."
                     )
 
-                    self.skip_handler.cancel_skipped_questions(question)
-                else:
-                    if (
-                        hasattr(response, "exception_occurred")
-                        and response.exception_occurred
-                    ):
-                        raise response.exception_occurred
-
-            except QuestionAnswerValidationError as e:
-                self._handle_exception(e, invigilator, task)
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question answer validation failed."
-                )
+                except asyncio.TimeoutError as e:
+                    self._handle_exception(e, invigilator, task)
+                    had_language_model_no_response_error = True
+                    raise LanguageModelNoResponseError(
+                        f"Language model timed out for question '{question.question_name}.'"
+                    )
 
-            except asyncio.TimeoutError as e:
-                self._handle_exception(e, invigilator, task)
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model timed out for question '{question.question_name}.'"
-                )
+                except Exception as e:
+                    self._handle_exception(e, invigilator, task)
 
-            except Exception as e:
-                self._handle_exception(e, invigilator, task)
+                if "response" not in locals():
+                    had_language_model_no_response_error = True
+                    raise LanguageModelNoResponseError(
+                        f"Language model did not return a response for question '{question.question_name}.'"
+                    )
 
-            if "response" not in locals():
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model did not return a response for question '{question.question_name}.'"
-                )
+                if (
+                    question.question_name in self.interview.exceptions
+                    and had_language_model_no_response_error
+                ):
+                    self.interview.exceptions.record_fixed_question(
+                        question.question_name
+                    )
 
-            if (
-                question.question_name in self.interview.exceptions
-                and had_language_model_no_response_error
-            ):
-                self.interview.exceptions.record_fixed_question(question.question_name)
+                return response
 
-            return response
+            try:
+                return await attempt_answer()
+            except RetryError as retry_error:
+                original_error = retry_error.last_attempt.exception()
+                self._handle_exception(
+                    original_error, FetchInvigilator(self.interview)(question), task
+                )
+                raise original_error
 
-        try:
-            return await attempt_answer()
-        except RetryError as retry_error:
-            original_error = retry_error.last_attempt.exception()
-            self._handle_exception(
-                original_error, self.invigilator_fetcher(question), task
-            )
-            raise original_error
+        return answer_question_and_record_task

edsl/jobs/FetchInvigilator.py

@@ -3,23 +3,15 @@ from typing import List, Dict, Any, Optional, TYPE_CHECKING
 if TYPE_CHECKING:
     from edsl.questions.QuestionBase import QuestionBase
     from edsl.agents.InvigilatorBase import InvigilatorBase
-    from edsl.language_models.key_management.KeyLookup import KeyLookup
-    from edsl.jobs.interviews.Interview import Interview
 
 
 class FetchInvigilator:
-    def __init__(
-        self,
-        interview: "Interview",
-        current_answers: Optional[Dict[str, Any]] = None,
-        key_lookup: Optional["KeyLookup"] = None,
-    ):
+    def __init__(self, interview, current_answers: Optional[Dict[str, Any]] = None):
         self.interview = interview
         if current_answers is None:
             self.current_answers = self.interview.answers
         else:
             self.current_answers = current_answers
-        self.key_lookup = key_lookup
 
     def get_invigilator(self, question: "QuestionBase") -> "InvigilatorBase":
         """Return an invigilator for the given question.
@@ -32,13 +24,14 @@ class FetchInvigilator:
             question=question,
             scenario=self.interview.scenario,
             model=self.interview.model,
+            #            debug=False,
             survey=self.interview.survey,
             memory_plan=self.interview.survey.memory_plan,
             current_answers=self.current_answers,  # not yet known
             iteration=self.interview.iteration,
             cache=self.interview.cache,
+            #           sidecar_model=self.interview.sidecar_model,
             raise_validation_errors=self.interview.raise_validation_errors,
-            key_lookup=self.key_lookup,
         )
         """Return an invigilator for the given question."""
         return invigilator

edsl/jobs/InterviewsConstructor.py

@@ -6,9 +6,8 @@ if TYPE_CHECKING:
 
 
 class InterviewsConstructor:
-    def __init__(self, jobs: "Jobs", cache: "Cache"):
+    def __init__(self, jobs):
         self.jobs = jobs
-        self.cache = cache
 
     def create_interviews(self) -> Generator["Interview", None, None]:
         """
@@ -35,13 +34,12 @@ class InterviewsConstructor:
             self.jobs.agents, self.jobs.scenarios, self.jobs.models
         ):
             yield Interview(
-                survey=self.jobs.survey.draw(),
+                survey=self.jobs.survey,
                 agent=agent,
                 scenario=scenario,
                 model=model,
-                cache=self.cache,
-                skip_retry=self.jobs.run_config.parameters.skip_retry,
-                raise_validation_errors=self.jobs.run_config.parameters.raise_validation_errors,
+                skip_retry=self.jobs.skip_retry,
+                raise_validation_errors=self.jobs.raise_validation_errors,
                 indices={
                     "agent": agent_index[hash(agent)],
                     "model": model_index[hash(model)],