edsl 0.1.39__py3-none-any.whl → 0.1.39.dev1__py3-none-any.whl

edsl/inference_services/AvailableModelFetcher.py

@@ -1,215 +0,0 @@
-from typing import Any, List, Tuple, Optional, Dict, TYPE_CHECKING, Union, Generator
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from collections import UserList
-
-from edsl.inference_services.ServiceAvailability import ServiceAvailability
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.inference_services.data_structures import ModelNamesList
-from edsl.enums import InferenceServiceLiteral
-
-from edsl.inference_services.data_structures import LanguageModelInfo
-from edsl.inference_services.AvailableModelCacheHandler import (
-    AvailableModelCacheHandler,
-)
-
-
-from edsl.inference_services.data_structures import AvailableModels
-
-
-class AvailableModelFetcher:
-    """Fetches available models from the various services with JSON caching."""
-
-    service_availability = ServiceAvailability()
-    CACHE_VALIDITY_HOURS = 48  # Cache validity period in hours
-
-    def __init__(
-        self,
-        services: List["InferenceServiceABC"],
-        added_models: Dict[str, List[str]],
-        verbose: bool = False,
-        use_cache: bool = True,
-    ):
-        self.services = services
-        self.added_models = added_models
-        self._service_map = {
-            service._inference_service_: service for service in services
-        }
-        self.verbose = verbose
-        if use_cache:
-            self.cache_handler = AvailableModelCacheHandler()
-        else:
-            self.cache_handler = None
-
-    @property
-    def num_cache_entries(self):
-        return self.cache_handler.num_cache_entries
-
-    @property
-    def path_to_db(self):
-        return self.cache_handler.path_to_db
-
-    def reset_cache(self):
-        if self.cache_handler:
-            self.cache_handler.reset_cache()
-
-    def available(
-        self,
-        service: Optional[InferenceServiceABC] = None,
-        force_refresh: bool = False,
-    ) -> List[LanguageModelInfo]:
-        """
-        Get available models from all services, using cached data when available.
-
-        :param service: Optional[InferenceServiceABC] - If specified, only fetch models for this service.
-
-        >>> from edsl.inference_services.OpenAIService import OpenAIService
-        >>> af = AvailableModelFetcher([OpenAIService()], {})
-        >>> af.available(service="openai")
-        [LanguageModelInfo(model_name='...', service_name='openai'), ...]
-
-        Returns a list of [model, service_name, index] entries.
-        """
-
-        if service:  # they passed a specific service
-            matching_models, _ = self.get_available_models_by_service(
-                service=service, force_refresh=force_refresh
-            )
-            return matching_models
-
-        # Nope, we need to fetch them all
-        all_models = self._get_all_models()
-
-        # if self.cache_handler:
-        #    self.cache_handler.add_models_to_cache(all_models)
-
-        return all_models
-
-    def get_available_models_by_service(
-        self,
-        service: Union["InferenceServiceABC", InferenceServiceLiteral],
-        force_refresh: bool = False,
-    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
-        """Get models for a single service.
-
-        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
-        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
-        """
-        if isinstance(service, str):
-            service = self._fetch_service_by_service_name(service)
-
-        if not force_refresh:
-            models_from_cache = self.cache_handler.models(
-                service=service._inference_service_
-            )
-            if self.verbose:
-                print(
-                    "Searching cache for models with service name:",
-                    service._inference_service_,
-                )
-                print("Got models from cache:", models_from_cache)
-        else:
-            models_from_cache = None
-
-        if models_from_cache:
-            # print(f"Models from cache for {service}: {models_from_cache}")
-            # print(hasattr(models_from_cache[0], "service_name"))
-            return models_from_cache, service._inference_service_
-        else:
-            return self.get_available_models_by_service_fresh(service)
-
-    def get_available_models_by_service_fresh(
-        self, service: Union["InferenceServiceABC", InferenceServiceLiteral]
-    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
-        """Get models for a single service. This method always fetches fresh data.
-
-        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
-        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
-        """
-        if isinstance(service, str):
-            service = self._fetch_service_by_service_name(service)
-
-        service_models: ModelNamesList = (
-            self.service_availability.get_service_available(service, warn=False)
-        )
-        service_name = service._inference_service_
-
-        if not service_models:
-            import warnings
-
-            warnings.warn(f"No models found for service {service_name}")
-            return [], service_name
-
-        models_list = AvailableModels(
-            [
-                LanguageModelInfo(
-                    model_name=model_name,
-                    service_name=service_name,
-                )
-                for model_name in service_models
-            ]
-        )
-        self.cache_handler.add_models_to_cache(models_list)  # update the cache
-        return models_list, service_name
-
-    def _fetch_service_by_service_name(
-        self, service_name: InferenceServiceLiteral
-    ) -> "InferenceServiceABC":
-        """The service name is the _inference_service_ attribute of the service."""
-        if service_name in self._service_map:
-            return self._service_map[service_name]
-        raise ValueError(f"Service {service_name} not found")
-
-    def _get_all_models(self, force_refresh=False) -> List[LanguageModelInfo]:
-        all_models = []
-        with ThreadPoolExecutor(max_workers=min(len(self.services), 10)) as executor:
-            future_to_service = {
-                executor.submit(
-                    self.get_available_models_by_service, service, force_refresh
-                ): service
-                for service in self.services
-            }
-
-            for future in as_completed(future_to_service):
-                try:
-                    models, service_name = future.result()
-                    all_models.extend(models)
-
-                    # Add any additional models for this service
-                    for model in self.added_models.get(service_name, []):
-                        all_models.append(
-                            LanguageModelInfo(
-                                model_name=model, service_name=service_name
-                            )
-                        )
-
-                except Exception as exc:
-                    print(f"Service query failed for service {service_name}: {exc}")
-                    continue
-
-        return AvailableModels(all_models)
-
-
-def main():
-    from edsl.inference_services.OpenAIService import OpenAIService
-
-    af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
-    # print(af.available(service="openai"))
-    all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models(
-        force_refresh=True
-    )
-    print(all_models)
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod(optionflags=doctest.ELLIPSIS)
-    # main()
-
-    # from edsl.inference_services.OpenAIService import OpenAIService
-
-    # af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
-    # # print(af.available(service="openai"))
-
-    # all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models()
-    # print(all_models)

edsl/inference_services/ServiceAvailability.py

@@ -1,135 +0,0 @@
-from enum import Enum
-from typing import List, Optional, TYPE_CHECKING
-from functools import partial
-import warnings
-
-from edsl.inference_services.data_structures import AvailableModels, ModelNamesList
-
-if TYPE_CHECKING:
-    from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-
-
-class ModelSource(Enum):
-    LOCAL = "local"
-    COOP = "coop"
-    CACHE = "cache"
-
-
-class ServiceAvailability:
-    """This class is responsible for fetching the available models from different sources."""
-
-    _coop_model_list = None
-
-    def __init__(self, source_order: Optional[List[ModelSource]] = None):
-        """
-        Initialize with custom source order.
-        Default order is LOCAL -> COOP -> CACHE
-        """
-        self.source_order = source_order or [
-            ModelSource.LOCAL,
-            ModelSource.COOP,
-            ModelSource.CACHE,
-        ]
-
-        # Map sources to their fetch functions
-        self._source_fetchers = {
-            ModelSource.LOCAL: self._fetch_from_local_service,
-            ModelSource.COOP: self._fetch_from_coop,
-            ModelSource.CACHE: self._fetch_from_cache,
-        }
-
-    @classmethod
-    def models_from_coop(cls) -> AvailableModels:
-        if not cls._coop_model_list:
-            from edsl.coop.coop import Coop
-
-            c = Coop()
-            coop_model_list = c.fetch_models()
-            cls._coop_model_list = coop_model_list
-        return cls._coop_model_list
-
-    def get_service_available(
-        self, service: "InferenceServiceABC", warn: bool = False
-    ) -> ModelNamesList:
-        """
-        Try to fetch available models from sources in specified order.
-        Returns first successful result.
-        """
-        last_error = None
-
-        for source in self.source_order:
-            try:
-                fetch_func = partial(self._source_fetchers[source], service)
-                result = fetch_func()
-
-                # Cache successful result
-                service._models_list_cache = result
-                return result
-
-            except Exception as e:
-                last_error = e
-                if warn:
-                    self._warn_source_failed(service, source)
-                continue
-
-        # If we get here, all sources failed
-        raise RuntimeError(
-            f"All sources failed to fetch models. Last error: {last_error}"
-        )
-
-    @staticmethod
-    def _fetch_from_local_service(service: "InferenceServiceABC") -> ModelNamesList:
-        """Attempt to fetch models directly from the service."""
-        return service.available()
-
-    @classmethod
-    def _fetch_from_coop(cls, service: "InferenceServiceABC") -> ModelNamesList:
-        """Fetch models from Coop."""
-        models_from_coop = cls.models_from_coop()
-        return models_from_coop.get(service._inference_service_, [])
-
-    @staticmethod
-    def _fetch_from_cache(service: "InferenceServiceABC") -> ModelNamesList:
-        """Fetch models from local cache."""
-        from edsl.inference_services.models_available_cache import models_available
-
-        return models_available.get(service._inference_service_, [])
-
-    def _warn_source_failed(self, service: "InferenceServiceABC", source: ModelSource):
-        """Display appropriate warning message based on failed source."""
-        messages = {
-            ModelSource.LOCAL: f"""Error getting models for {service._inference_service_}. 
-                Check that you have properly stored your Expected Parrot API key and activated remote inference, 
-                or stored your own API keys for the language models that you want to use.
-                See https://docs.expectedparrot.com/en/latest/api_keys.html for instructions on storing API keys.
-                Trying next source.""",
-            ModelSource.COOP: f"Error getting models from Coop for {service._inference_service_}. Trying next source.",
-            ModelSource.CACHE: f"Error getting models from cache for {service._inference_service_}.",
-        }
-        warnings.warn(messages[source], UserWarning)
-
-
-if __name__ == "__main__":
-    # sa = ServiceAvailability()
-    # models_from_coop = sa.models_from_coop()
-    # print(models_from_coop)
-    from edsl.inference_services.OpenAIService import OpenAIService
-
-    openai_models = ServiceAvailability._fetch_from_local_service(OpenAIService())
-    print(openai_models)
-
-# Example usage:
-"""
-# Default order (LOCAL -> COOP -> CACHE)
-availability = ServiceAvailability()
-
-# Custom order (COOP -> LOCAL -> CACHE)
-availability_coop_first = ServiceAvailability([
-    ModelSource.COOP,
-    ModelSource.LOCAL,
-    ModelSource.CACHE
-])
-
-# Get available models using custom order
-models = availability_coop_first.get_service_available(service, warn=True)
-"""

edsl/inference_services/data_structures.py

@@ -1,134 +0,0 @@
-from collections import UserDict, defaultdict, UserList
-from typing import Union, Optional, List
-from edsl.enums import InferenceServiceLiteral
-from dataclasses import dataclass
-
-
-@dataclass
-class LanguageModelInfo:
-    """A dataclass for storing information about a language model.
-
-
-    >>> LanguageModelInfo("gpt-4-1106-preview", "openai")
-    LanguageModelInfo(model_name='gpt-4-1106-preview', service_name='openai')
-
-    >>> model_name, service = LanguageModelInfo.example()
-    >>> model_name
-    'gpt-4-1106-preview'
-
-    >>> LanguageModelInfo.example().service_name
-    'openai'
-
-    """
-
-    model_name: str
-    service_name: str
-
-    def __iter__(self):
-        yield self.model_name
-        yield self.service_name
-
-    def __getitem__(self, key: int) -> str:
-        import warnings
-
-        warnings.warn(
-            "Accessing LanguageModelInfo via index is deprecated. "
-            "Please use .model_name, .service_name, or .index attributes instead.",
-            DeprecationWarning,
-            stacklevel=2,
-        )
-
-        if key == 0:
-            return self.model_name
-        elif key == 1:
-            return self.service_name
-        else:
-            raise IndexError("Index out of range")
-
-    @classmethod
-    def example(cls) -> "LanguageModelInfo":
-        return cls("gpt-4-1106-preview", "openai")
-
-
-class ModelNamesList(UserList):
-    pass
-
-
-class AvailableModels(UserList):
-
-    def __init__(self, data: List[LanguageModelInfo]) -> None:
-        super().__init__(data)
-
-    def __contains__(self, model_name: str) -> bool:
-        for model_entry in self:
-            if model_entry.model_name == model_name:
-                return True
-        return False
-
-    def print(self):
-        return self.to_dataset().print()
-
-    def to_dataset(self):
-        from edsl.scenarios.ScenarioList import ScenarioList
-
-        models, services = zip(
-            *[(model.model_name, model.service_name) for model in self]
-        )
-        return (
-            ScenarioList.from_list("model", models)
-            .add_list("service", services)
-            .to_dataset()
-        )
-
-    def to_model_list(self):
-        from edsl.language_models.ModelList import ModelList
-
-        return ModelList.from_available_models(self)
-
-    def search(
-        self, pattern: str, service_name: Optional[str] = None, regex: bool = False
-    ) -> "AvailableModels":
-        import re
-
-        if not regex:
-            # Escape special regex characters except *
-            pattern = re.escape(pattern).replace(r"\*", ".*")
-
-        try:
-            regex = re.compile(pattern)
-            avm = AvailableModels(
-                [
-                    entry
-                    for entry in self
-                    if regex.search(entry.model_name)
-                    and (service_name is None or entry.service_name == service_name)
-                ]
-            )
-            if len(avm) == 0:
-                raise ValueError(
-                    "No models found matching the search pattern: " + pattern
-                )
-            else:
-                return avm
-        except re.error as e:
-            raise ValueError(f"Invalid regular expression pattern: {e}")
-
-
-class ServiceToModelsMapping(UserDict):
-    def __init__(self, data: dict) -> None:
-        super().__init__(data)
-
-    @property
-    def service_names(self) -> list[str]:
-        return list(self.data.keys())
-
-    def _validate_service_names(self):
-        for service in self.service_names:
-            if service not in InferenceServiceLiteral:
-                raise ValueError(f"Invalid service name: {service}")
-
-    def model_to_services(self) -> dict:
-        self._model_to_service = defaultdict(list)
-        for service, models in self.data.items():
-            for model in models:
-                self._model_to_service[model].append(service)

edsl/jobs/AnswerQuestionFunctionConstructor.py

@@ -1,223 +0,0 @@
-import copy
-import asyncio
-
-from typing import Union, Type, Callable, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from edsl.questions.QuestionBase import QuestionBase
-    from edsl.jobs.interviews.Interview import Interview
-    from edsl.language_models.key_management.KeyLookup import KeyLookup
-
-from edsl.surveys.base import EndOfSurvey
-from edsl.jobs.tasks.task_status_enum import TaskStatus
-
-from edsl.jobs.FetchInvigilator import FetchInvigilator
-from edsl.exceptions.language_models import LanguageModelNoResponseError
-from edsl.exceptions.questions import QuestionAnswerValidationError
-from edsl.data_transfer_models import AgentResponseDict, EDSLResultObjectInput
-
-from edsl.jobs.Answers import Answers
-
-
-class RetryConfig:
-    from edsl.config import CONFIG
-
-    EDSL_BACKOFF_START_SEC = float(CONFIG.get("EDSL_BACKOFF_START_SEC"))
-    EDSL_BACKOFF_MAX_SEC = float(CONFIG.get("EDSL_BACKOFF_MAX_SEC"))
-    EDSL_MAX_ATTEMPTS = int(CONFIG.get("EDSL_MAX_ATTEMPTS"))
-
-
-class SkipHandler:
-
-    def __init__(self, interview: "Interview"):
-        self.interview = interview
-        self.question_index = self.interview.to_index
-
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
-        )
-
-    def should_skip(self, current_question: "QuestionBase") -> bool:
-        """Determine if the current question should be skipped."""
-        current_question_index = self.question_index[current_question.question_name]
-        combined_answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
-        return self.skip_function(current_question_index, combined_answers)
-
-    def cancel_skipped_questions(self, current_question: "QuestionBase") -> None:
-        """Cancel the tasks for questions that should be skipped."""
-        current_question_index: int = self.interview.to_index[
-            current_question.question_name
-        ]
-        answers = (
-            self.interview.answers
-            | self.interview.scenario
-            | self.interview.agent["traits"]
-        )
-
-        # Get the index of the next question, which could also be the end of the survey
-        next_question: Union[int, EndOfSurvey] = (
-            self.interview.survey.rule_collection.next_question(
-                q_now=current_question_index,
-                answers=answers,
-            )
-        )
-
-        def cancel_between(start, end):
-            """Cancel the tasks for questions between the start and end indices."""
-            for i in range(start, end):
-                self.interview.tasks[i].cancel()
-
-        if (next_question_index := next_question.next_q) == EndOfSurvey:
-            cancel_between(
-                current_question_index + 1, len(self.interview.survey.questions)
-            )
-            return
-
-        if next_question_index > (current_question_index + 1):
-            cancel_between(current_question_index + 1, next_question_index)
-
-
-class AnswerQuestionFunctionConstructor:
-    """Constructs a function that answers a question and records the answer."""
-
-    def __init__(self, interview: "Interview", key_lookup: "KeyLookup"):
-        self.interview = interview
-        self.key_lookup = key_lookup
-
-        self.had_language_model_no_response_error: bool = False
-        self.question_index = self.interview.to_index
-
-        self.skip_function: Callable = (
-            self.interview.survey.rule_collection.skip_question_before_running
-        )
-
-        self.invigilator_fetcher = FetchInvigilator(
-            self.interview, key_lookup=self.key_lookup
-        )
-        self.skip_handler = SkipHandler(self.interview)
-
-    def _handle_exception(
-        self, e: Exception, invigilator: "InvigilatorBase", task=None
-    ):
-        """Handle an exception that occurred while answering a question."""
-
-        from edsl.jobs.interviews.InterviewExceptionEntry import InterviewExceptionEntry
-
-        answers = copy.copy(
-            self.interview.answers
-        )  # copy to freeze the answers here for logging
-        exception_entry = InterviewExceptionEntry(
-            exception=e,
-            invigilator=invigilator,
-            answers=answers,
-        )
-        if task:
-            task.task_status = TaskStatus.FAILED
-
-        self.interview.exceptions.add(
-            invigilator.question.question_name, exception_entry
-        )
-
-        if self.interview.raise_validation_errors and isinstance(
-            e, QuestionAnswerValidationError
-        ):
-            raise e
-
-        stop_on_exception = getattr(self.interview, "stop_on_exception", False)
-        if stop_on_exception:
-            raise e
-
-    def __call__(self):
-        return self.answer_question_and_record_task
-
-    async def answer_question_and_record_task(
-        self,
-        *,
-        question: "QuestionBase",
-        task=None,
-    ) -> "AgentResponseDict":
-
-        from tenacity import (
-            retry,
-            stop_after_attempt,
-            wait_exponential,
-            retry_if_exception_type,
-            RetryError,
-        )
-
-        @retry(
-            stop=stop_after_attempt(RetryConfig.EDSL_MAX_ATTEMPTS),
-            wait=wait_exponential(
-                multiplier=RetryConfig.EDSL_BACKOFF_START_SEC,
-                max=RetryConfig.EDSL_BACKOFF_MAX_SEC,
-            ),
-            retry=retry_if_exception_type(LanguageModelNoResponseError),
-            reraise=True,
-        )
-        async def attempt_answer():
-            invigilator = self.invigilator_fetcher(question)
-
-            if self.skip_handler.should_skip(question):
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question skipped."
-                )
-
-            try:
-                response: EDSLResultObjectInput = (
-                    await invigilator.async_answer_question()
-                )
-                if response.validated:
-                    self.interview.answers.add_answer(
-                        response=response, question=question
-                    )
-
-                    self.skip_handler.cancel_skipped_questions(question)
-                else:
-                    if (
-                        hasattr(response, "exception_occurred")
-                        and response.exception_occurred
-                    ):
-                        raise response.exception_occurred
-
-            except QuestionAnswerValidationError as e:
-                self._handle_exception(e, invigilator, task)
-                return invigilator.get_failed_task_result(
-                    failure_reason="Question answer validation failed."
-                )
-
-            except asyncio.TimeoutError as e:
-                self._handle_exception(e, invigilator, task)
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model timed out for question '{question.question_name}.'"
-                )
-
-            except Exception as e:
-                self._handle_exception(e, invigilator, task)
-
-            if "response" not in locals():
-                had_language_model_no_response_error = True
-                raise LanguageModelNoResponseError(
-                    f"Language model did not return a response for question '{question.question_name}.'"
-                )
-
-            if (
-                question.question_name in self.interview.exceptions
-                and had_language_model_no_response_error
-            ):
-                self.interview.exceptions.record_fixed_question(question.question_name)
-
-            return response
-
-        try:
-            return await attempt_answer()
-        except RetryError as retry_error:
-            original_error = retry_error.last_attempt.exception()
-            self._handle_exception(
-                original_error, self.invigilator_fetcher(question), task
-            )
-            raise original_error