edsl 0.1.39.dev2__py3-none-any.whl → 0.1.39.dev3__py3-none-any.whl

edsl/inference_services/AvailableModelFetcher.py

@@ -1,209 +0,0 @@
-from typing import Any, List, Tuple, Optional, Dict, TYPE_CHECKING, Union, Generator
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from collections import UserList
-
-from edsl.inference_services.ServiceAvailability import ServiceAvailability
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.inference_services.data_structures import ModelNamesList
-from edsl.enums import InferenceServiceLiteral
-
-from edsl.inference_services.data_structures import LanguageModelInfo
-from edsl.inference_services.AvailableModelCacheHandler import (
-    AvailableModelCacheHandler,
-)
-
-
-from edsl.inference_services.data_structures import AvailableModels
-
-
-class AvailableModelFetcher:
-    """Fetches available models from the various services with JSON caching."""
-
-    service_availability = ServiceAvailability()
-    CACHE_VALIDITY_HOURS = 48  # Cache validity period in hours
-
-    def __init__(
-        self,
-        services: List["InferenceServiceABC"],
-        added_models: Dict[str, List[str]],
-        verbose: bool = False,
-        use_cache: bool = True,
-    ):
-        self.services = services
-        self.added_models = added_models
-        self._service_map = {
-            service._inference_service_: service for service in services
-        }
-        self.verbose = verbose
-        if use_cache:
-            self.cache_handler = AvailableModelCacheHandler()
-        else:
-            self.cache_handler = None
-
-    @property
-    def num_cache_entries(self):
-        return self.cache_handler.num_cache_entries
-
-    @property
-    def path_to_db(self):
-        return self.cache_handler.path_to_db
-
-    def reset_cache(self):
-        if self.cache_handler:
-            self.cache_handler.reset_cache()
-
-    def available(
-        self,
-        service: Optional[InferenceServiceABC] = None,
-        force_refresh: bool = False,
-    ) -> List[LanguageModelInfo]:
-        """
-        Get available models from all services, using cached data when available.
-
-        :param service: Optional[InferenceServiceABC] - If specified, only fetch models for this service.
-
-        >>> from edsl.inference_services.OpenAIService import OpenAIService
-        >>> af = AvailableModelFetcher([OpenAIService()], {})
-        >>> af.available(service="openai")
-        [LanguageModelInfo(model_name='...', service_name='openai'), ...]
-
-        Returns a list of [model, service_name, index] entries.
-        """
-
-        if service:  # they passed a specific service
-            matching_models, _ = self.get_available_models_by_service(
-                service=service, force_refresh=force_refresh
-            )
-            return matching_models
-
-        # Nope, we need to fetch them all
-        all_models = self._get_all_models()
-
-        # if self.cache_handler:
-        #    self.cache_handler.add_models_to_cache(all_models)
-
-        return all_models
-
-    def get_available_models_by_service(
-        self,
-        service: Union["InferenceServiceABC", InferenceServiceLiteral],
-        force_refresh: bool = False,
-    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
-        """Get models for a single service.
-
-        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
-        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
-        """
-        if isinstance(service, str):
-            service = self._fetch_service_by_service_name(service)
-
-        if not force_refresh:
-            models_from_cache = self.cache_handler.models(
-                service=service._inference_service_
-            )
-            if self.verbose:
-                print(
-                    "Searching cache for models with service name:",
-                    service._inference_service_,
-                )
-                print("Got models from cache:", models_from_cache)
-        else:
-            models_from_cache = None
-
-        if models_from_cache:
-            # print(f"Models from cache for {service}: {models_from_cache}")
-            # print(hasattr(models_from_cache[0], "service_name"))
-            return models_from_cache, service._inference_service_
-        else:
-            return self.get_available_models_by_service_fresh(service)
-
-    def get_available_models_by_service_fresh(
-        self, service: Union["InferenceServiceABC", InferenceServiceLiteral]
-    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
-        """Get models for a single service. This method always fetches fresh data.
-
-        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
-        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
-        """
-        if isinstance(service, str):
-            service = self._fetch_service_by_service_name(service)
-
-        service_models: ModelNamesList = (
-            self.service_availability.get_service_available(service, warn=False)
-        )
-        service_name = service._inference_service_
-
-        models_list = AvailableModels(
-            [
-                LanguageModelInfo(
-                    model_name=model_name,
-                    service_name=service_name,
-                )
-                for model_name in service_models
-            ]
-        )
-        self.cache_handler.add_models_to_cache(models_list)  # update the cache
-        return models_list, service_name
-
-    def _fetch_service_by_service_name(
-        self, service_name: InferenceServiceLiteral
-    ) -> "InferenceServiceABC":
-        """The service name is the _inference_service_ attribute of the service."""
-        if service_name in self._service_map:
-            return self._service_map[service_name]
-        raise ValueError(f"Service {service_name} not found")
-
-    def _get_all_models(self, force_refresh=False) -> List[LanguageModelInfo]:
-        all_models = []
-        with ThreadPoolExecutor(max_workers=min(len(self.services), 10)) as executor:
-            future_to_service = {
-                executor.submit(
-                    self.get_available_models_by_service, service, force_refresh
-                ): service
-                for service in self.services
-            }
-
-            for future in as_completed(future_to_service):
-                try:
-                    models, service_name = future.result()
-                    all_models.extend(models)
-
-                    # Add any additional models for this service
-                    for model in self.added_models.get(service_name, []):
-                        all_models.append(
-                            LanguageModelInfo(
-                                model_name=model, service_name=service_name
-                            )
-                        )
-
-                except Exception as exc:
-                    print(f"Service query failed: {exc}")
-                    continue
-
-        return AvailableModels(all_models)
-
-
-def main():
-    from edsl.inference_services.OpenAIService import OpenAIService
-
-    af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
-    # print(af.available(service="openai"))
-    all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models(
-        force_refresh=True
-    )
-    print(all_models)
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
-    # main()
-
-    # from edsl.inference_services.OpenAIService import OpenAIService
-
-    # af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
-    # # print(af.available(service="openai"))
-
-    # all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models()
-    # print(all_models)

edsl/inference_services/ServiceAvailability.py

@@ -1,135 +0,0 @@
-from enum import Enum
-from typing import List, Optional, TYPE_CHECKING
-from functools import partial
-import warnings
-
-from edsl.inference_services.data_structures import AvailableModels, ModelNamesList
-
-if TYPE_CHECKING:
-    from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-
-
-class ModelSource(Enum):
-    LOCAL = "local"
-    COOP = "coop"
-    CACHE = "cache"
-
-
-class ServiceAvailability:
-    """This class is responsible for fetching the available models from different sources."""
-
-    _coop_model_list = None
-
-    def __init__(self, source_order: Optional[List[ModelSource]] = None):
-        """
-        Initialize with custom source order.
-        Default order is LOCAL -> COOP -> CACHE
-        """
-        self.source_order = source_order or [
-            ModelSource.LOCAL,
-            ModelSource.COOP,
-            ModelSource.CACHE,
-        ]
-
-        # Map sources to their fetch functions
-        self._source_fetchers = {
-            ModelSource.LOCAL: self._fetch_from_local_service,
-            ModelSource.COOP: self._fetch_from_coop,
-            ModelSource.CACHE: self._fetch_from_cache,
-        }
-
-    @classmethod
-    def models_from_coop(cls) -> AvailableModels:
-        if not cls._coop_model_list:
-            from edsl.coop.coop import Coop
-
-            c = Coop()
-            coop_model_list = c.fetch_models()
-            cls._coop_model_list = coop_model_list
-        return cls._coop_model_list
-
-    def get_service_available(
-        self, service: "InferenceServiceABC", warn: bool = False
-    ) -> ModelNamesList:
-        """
-        Try to fetch available models from sources in specified order.
-        Returns first successful result.
-        """
-        last_error = None
-
-        for source in self.source_order:
-            try:
-                fetch_func = partial(self._source_fetchers[source], service)
-                result = fetch_func()
-
-                # Cache successful result
-                service._models_list_cache = result
-                return result
-
-            except Exception as e:
-                last_error = e
-                if warn:
-                    self._warn_source_failed(service, source)
-                continue
-
-        # If we get here, all sources failed
-        raise RuntimeError(
-            f"All sources failed to fetch models. Last error: {last_error}"
-        )
-
-    @staticmethod
-    def _fetch_from_local_service(service: "InferenceServiceABC") -> ModelNamesList:
-        """Attempt to fetch models directly from the service."""
-        return service.available()
-
-    @classmethod
-    def _fetch_from_coop(cls, service: "InferenceServiceABC") -> ModelNamesList:
-        """Fetch models from Coop."""
-        models_from_coop = cls.models_from_coop()
-        return models_from_coop.get(service._inference_service_, [])
-
-    @staticmethod
-    def _fetch_from_cache(service: "InferenceServiceABC") -> ModelNamesList:
-        """Fetch models from local cache."""
-        from edsl.inference_services.models_available_cache import models_available
-
-        return models_available.get(service._inference_service_, [])
-
-    def _warn_source_failed(self, service: "InferenceServiceABC", source: ModelSource):
-        """Display appropriate warning message based on failed source."""
-        messages = {
-            ModelSource.LOCAL: f"""Error getting models for {service._inference_service_}. 
-                Check that you have properly stored your Expected Parrot API key and activated remote inference, 
-                or stored your own API keys for the language models that you want to use.
-                See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
-                Trying next source.""",
-            ModelSource.COOP: f"Error getting models from Coop for {service._inference_service_}. Trying next source.",
-            ModelSource.CACHE: f"Error getting models from cache for {service._inference_service_}.",
-        }
-        warnings.warn(messages[source], UserWarning)
-
-
-if __name__ == "__main__":
-    # sa = ServiceAvailability()
-    # models_from_coop = sa.models_from_coop()
-    # print(models_from_coop)
-    from edsl.inference_services.OpenAIService import OpenAIService
-
-    openai_models = ServiceAvailability._fetch_from_local_service(OpenAIService())
-    print(openai_models)
-
-# Example usage:
-"""
-# Default order (LOCAL -> COOP -> CACHE)
-availability = ServiceAvailability()
-
-# Custom order (COOP -> LOCAL -> CACHE)
-availability_coop_first = ServiceAvailability([
-    ModelSource.COOP,
-    ModelSource.LOCAL,
-    ModelSource.CACHE
-])
-
-# Get available models using custom order
-models = availability_coop_first.get_service_available(service, warn=True)
-"""

edsl/inference_services/data_structures.py

@@ -1,62 +0,0 @@
-from collections import UserDict, defaultdict, UserList
-from typing import Union
-from edsl.enums import InferenceServiceLiteral
-from dataclasses import dataclass
-
-
-@dataclass
-class LanguageModelInfo:
-    model_name: str
-    service_name: str
-
-    def __getitem__(self, key: int) -> str:
-        import warnings
-
-        warnings.warn(
-            "Accessing LanguageModelInfo via index is deprecated. "
-            "Please use .model_name, .service_name, or .index attributes instead.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-
-        if key == 0:
-            return self.model_name
-        elif key == 1:
-            return self.service_name
-        else:
-            raise IndexError("Index out of range")
-
-
-class ModelNamesList(UserList):
-    pass
-
-
-class AvailableModels(UserList):
-    def __init__(self, data: list) -> None:
-        super().__init__(data)
-
-    def __contains__(self, model_name: str) -> bool:
-        for model_entry in self:
-            if model_entry.model_name == model_name:
-                return True
-        return False
-
-
-class ServiceToModelsMapping(UserDict):
-    def __init__(self, data: dict) -> None:
-        super().__init__(data)
-
-    @property
-    def service_names(self) -> list[str]:
-        return list(self.data.keys())
-
-    def _validate_service_names(self):
-        for service in self.service_names:
-            if service not in InferenceServiceLiteral:
-                raise ValueError(f"Invalid service name: {service}")
-
-    def model_to_services(self) -> dict:
-        self._model_to_service = defaultdict(list)
-        for service, models in self.data.items():
-            for model in models:
-                self._model_to_service[model].append(service)

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -1,188 +0,0 @@
-import copy
-import asyncio
-
-from typing import Union, Type, Callable, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from edsl.questions.QuestionBase import QuestionBase
-
-from edsl.surveys.base import EndOfSurvey
-from edsl.jobs.tasks.task_status_enum import TaskStatus
-
-from edsl.jobs.FetchInvigilator import FetchInvigilator
-from edsl.exceptions.language_models import LanguageModelNoResponseError
-from edsl.exceptions.questions import QuestionAnswerValidationError
-from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
-
-from edsl.jobs.Answers import Answers
-
-
-class AnswerQuestionFunctionConstructor:
-    def __init__(self, interview):
-        self.interview = interview
-        self.had_language_model_no_response_error = False
-        self.question_index = self.interview.to_index
-
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
-        )
-
-    def _combined_answers(self) -> Answers:
-        return self.answers | self.interview.scenario | self.interview.agent["traits"]
-
-    @property
-    def answers(self) -> Answers:
-        return self.interview.answers
-
-    def _skip_this_question(self, current_question: "QuestionBase") -> bool:
-        current_question_index = self.question_index[current_question.question_name]
-        combined_answers = self._combined_answers()
-        return self.skip_function(current_question_index, combined_answers)
-
-    def _handle_exception(
-        self, e: Exception, invigilator: "InvigilatorBase", task=None
-    ):
-        from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-
-        answers = copy.copy(self.answers)  # copy to freeze the answers here for logging
-        exception_entry = InterviewExceptionEntry(
-            exception=e,
-            invigilator=invigilator,
-            answers=answers,
-        )
-        if task:
-            task.task_status = TaskStatus.FAILED
-        self.interview.exceptions.add(
-            invigilator.question.question_name, exception_entry
-        )
-
-        if self.interview.raise_validation_errors and isinstance(
-            e, QuestionAnswerValidationError
-        ):
-            raise e
-
-        stop_on_exception = getattr(self.interview, "stop_on_exception", False)
-        if stop_on_exception:
-            raise e
-
-    def _cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
-        current_question_index: int = self.interview.to_index[
-            current_question.question_name
-        ]
-        answers = (
-            self.answers | self.interview.scenario | self.interview.agent["traits"]
-        )
-
-        # Get the index of the next question, which could also be the end of the survey
-        next_question: Union[
-            int, EndOfSurvey
-        ] = self.interview.survey.rule_collection.next_question(
-            q_now=current_question_index,
-            answers=answers,
-        )
-
-        def cancel_between(start, end):
-            for i in range(start, end):
-                self.interview.tasks[i].cancel()
-
-        if (next_question_index := next_question.next_q) == EndOfSurvey:
-            cancel_between(
-                current_question_index + 1, len(self.interview.survey.questions)
-            )
-            return
-
-        if next_question_index > (current_question_index + 1):
-            cancel_between(current_question_index + 1, next_question_index)
-
-    def __call__(self):
-        from edsl.config import CONFIG
-
-        EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-        EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-        EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
-
-        from tenacity import (
-            retry,
-            stop_after_attempt,
-            wait_exponential,
-            retry_if_exception_type,
-            RetryError,
-        )
-
-        async def answer_question_and_record_task(
-            *,
-            question: "QuestionBase",
-            task=None,
-        ) -> "AgentResponseDict":
-            @retry(
-                stop=stop_after_attempt(EDSL_MAX_ATTEMPTS),
-                wait=wait_exponential(
-                    multiplier=EDSL_BACKOFF_START_SEC, max=EDSL_BACKOFF_MAX_SEC
-                ),
-                retry=retry_if_exception_type(LanguageModelNoResponseError),
-                reraise=True,
-            )
-            async def attempt_answer():
-                invigilator = FetchInvigilator(self.interview)(question)
-
-                if self._skip_this_question(question):
-                    return invigilator.get_failed_task_result(
-                        failure_reason="Question skipped."
-                    )
-
-                try:
-                    response: EDSLResultObjectInput = (
-                        await invigilator.async_answer_question()
-                    )
-                    if response.validated:
-                        self.answers.add_answer(response=response, question=question)
-                        self._cancel_skipped_questions(question)
-                    else:
-                        if (
-                            hasattr(response, "exception_occurred")
-                            and response.exception_occurred
-                        ):
-                            raise response.exception_occurred
-
-                except QuestionAnswerValidationError as e:
-                    self._handle_exception(e, invigilator, task)
-                    return invigilator.get_failed_task_result(
-                        failure_reason="Question answer validation failed."
-                    )
-
-                except asyncio.TimeoutError as e:
-                    self._handle_exception(e, invigilator, task)
-                    had_language_model_no_response_error = True
-                    raise LanguageModelNoResponseError(
-                        f"Language model timed out for question '{question.question_name}.'"
-                    )
-
-                except Exception as e:
-                    self._handle_exception(e, invigilator, task)
-
-                if "response" not in locals():
-                    had_language_model_no_response_error = True
-                    raise LanguageModelNoResponseError(
-                        f"Language model did not return a response for question '{question.question_name}.'"
-                    )
-
-                if (
-                    question.question_name in self.interview.exceptions
-                    and had_language_model_no_response_error
-                ):
-                    self.interview.exceptions.record_fixed_question(
-                        question.question_name
-                    )
-
-                return response
-
-            try:
-                return await attempt_answer()
-            except RetryError as retry_error:
-                original_error = retry_error.last_attempt.exception()
-                self._handle_exception(
-                    original_error, FetchInvigilator(self.interview)(question), task
-                )
-                raise original_error
-
-        return answer_question_and_record_task

edsl/jobs/FetchInvigilator.py

@@ -1,40 +0,0 @@
-from typing import List, Dict, Any, Optional, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.agents.InvigilatorBase import InvigilatorBase
-
-
-class FetchInvigilator:
-    def __init__(self, interview, current_answers: Optional[Dict[str, Any]] = None):
-        self.interview = interview
-        if current_answers is None:
-            self.current_answers = self.interview.answers
-        else:
-            self.current_answers = current_answers
-
-    def get_invigilator(self, question: "QuestionBase") -> "InvigilatorBase":
-        """Return an invigilator for the given question.
-
-        :param question: the question to be answered
-        :param debug: whether to use debug mode, in which case `InvigilatorDebug` is used.
-        """
-
-        invigilator = self.interview.agent.create_invigilator(
-            question=question,
-            scenario=self.interview.scenario,
-            model=self.interview.model,
-            #            debug=False,
-            survey=self.interview.survey,
-            memory_plan=self.interview.survey.memory_plan,
-            current_answers=self.current_answers,  # not yet known
-            iteration=self.interview.iteration,
-            cache=self.interview.cache,
-            #           sidecar_model=self.interview.sidecar_model,
-            raise_validation_errors=self.interview.raise_validation_errors,
-        )
-        """Return an invigilator for the given question."""
-        return invigilator
-
-    def __call__(self, question):
-        return self.get_invigilator(question)