edsl 0.1.39.dev1__py3-none-any.whl → 0.1.39.dev2__py3-none-any.whl

edsl/exceptions/__init__.py

@@ -1,54 +1,54 @@
-from .agents import (
-    #    AgentAttributeLookupCallbackError,
-    AgentCombinationError,
-    #    AgentLacksLLMError,
-    #    AgentRespondedWithBadJSONError,
-)
-from .configuration import (
-    InvalidEnvironmentVariableError,
-    MissingEnvironmentVariableError,
-)
-from .data import (
-    DatabaseConnectionError,
-    DatabaseCRUDError,
-    DatabaseIntegrityError,
-)
+# from .agents import (
+#     #    AgentAttributeLookupCallbackError,
+#     AgentCombinationError,
+#     #    AgentLacksLLMError,
+#     #    AgentRespondedWithBadJSONError,
+# )
+# from .configuration import (
+#     InvalidEnvironmentVariableError,
+#     MissingEnvironmentVariableError,
+# )
+# from .data import (
+#     DatabaseConnectionError,
+#     DatabaseCRUDError,
+#     DatabaseIntegrityError,
+# )
 
-from .scenarios import (
-    ScenarioError,
-)
+# from .scenarios import (
+#     ScenarioError,
+# )
 
-from .general import MissingAPIKeyError
+# from .general import MissingAPIKeyError
 
-from .jobs import JobsRunError, InterviewErrorPriorTaskCanceled, InterviewTimeoutError
+# from .jobs import JobsRunError, InterviewErrorPriorTaskCanceled, InterviewTimeoutError
 
-from .language_models import (
-    LanguageModelResponseNotJSONError,
-    LanguageModelMissingAttributeError,
-    LanguageModelAttributeTypeError,
-    LanguageModelDoNotAddError,
-)
-from .questions import (
-    QuestionAnswerValidationError,
-    QuestionAttributeMissing,
-    QuestionCreationValidationError,
-    QuestionResponseValidationError,
-    QuestionSerializationError,
-    QuestionScenarioRenderError,
-)
-from .results import (
-    ResultsBadMutationstringError,
-    ResultsColumnNotFoundError,
-    ResultsInvalidNameError,
-    ResultsMutateError,
-)
-from .surveys import (
-    SurveyCreationError,
-    SurveyHasNoRulesError,
-    SurveyRuleCannotEvaluateError,
-    SurveyRuleCollectionHasNoRulesAtNodeError,
-    SurveyRuleReferenceInRuleToUnknownQuestionError,
-    SurveyRuleRefersToFutureStateError,
-    SurveyRuleSendsYouBackwardsError,
-    SurveyRuleSkipLogicSyntaxError,
-)
+# from .language_models import (
+#     LanguageModelResponseNotJSONError,
+#     LanguageModelMissingAttributeError,
+#     LanguageModelAttributeTypeError,
+#     LanguageModelDoNotAddError,
+# )
+# from .questions import (
+#     QuestionAnswerValidationError,
+#     QuestionAttributeMissing,
+#     QuestionCreationValidationError,
+#     QuestionResponseValidationError,
+#     QuestionSerializationError,
+#     QuestionScenarioRenderError,
+# )
+# from .results import (
+#     ResultsBadMutationstringError,
+#     ResultsColumnNotFoundError,
+#     ResultsInvalidNameError,
+#     ResultsMutateError,
+# )
+# from .surveys import (
+#     SurveyCreationError,
+#     SurveyHasNoRulesError,
+#     SurveyRuleCannotEvaluateError,
+#     SurveyRuleCollectionHasNoRulesAtNodeError,
+#     SurveyRuleReferenceInRuleToUnknownQuestionError,
+#     SurveyRuleRefersToFutureStateError,
+#     SurveyRuleSendsYouBackwardsError,
+#     SurveyRuleSkipLogicSyntaxError,
+# )

edsl/exceptions/agents.py

@@ -1,6 +1,18 @@
 from edsl.exceptions.BaseException import BaseException
 
 
+# from edsl.utilities.utilities import is_notebook
+
+# from IPython.core.error import UsageError
+
+# class AgentListErrorAlternative(UsageError):
+#     def __init__(self, message):
+#         super().__init__(message)
+
+import sys
+from edsl.utilities.is_notebook import is_notebook
+
+
 class AgentListError(BaseException):
     relevant_doc = "https://docs.expectedparrot.com/en/latest/agents.html#agent-lists"
 

edsl/exceptions/inference_services.py

@@ -0,0 +1,5 @@
+from edsl.exceptions.BaseException import BaseException
+
+
+class InferenceServiceError(BaseException):
+    relevant_doc = "https://docs.expectedparrot.com/"

edsl/exceptions/questions.py

@@ -1,6 +1,6 @@
 from typing import Any, SupportsIndex
-from jinja2 import Template
 import json
+from pydantic import ValidationError
 
 
 class QuestionErrors(Exception):
@@ -20,17 +20,35 @@ class QuestionAnswerValidationError(QuestionErrors):
     For example, if the question is a multiple choice question, the answer should be drawn from the list of options provided.
     """
 
-    def __init__(self, message="Invalid answer.", data=None, model=None):
+    def __init__(
+        self,
+        message="Invalid answer.",
+        pydantic_error: ValidationError = None,
+        data: dict = None,
+        model=None,
+    ):
         self.message = message
+        self.pydantic_error = pydantic_error
         self.data = data
         self.model = model
         super().__init__(self.message)
 
     def __str__(self):
-        return f"""{repr(self)} 
-        Data being validated: {self.data} 
-        Pydnantic Model: {self.model}.
-        Reported error: {self.message}."""
+        if isinstance(self.message, ValidationError):
+            # If it's a ValidationError, just return the core error message
+            return str(self.message)
+        elif hasattr(self.message, "errors"):
+            # Handle the case where it's already been converted to a string but has errors
+            error_list = self.message.errors()
+            if error_list:
+                return str(error_list[0].get("msg", "Unknown error"))
+        return str(self.message)
+
+    # def __str__(self):
+    #     return f"""{repr(self)}
+    #     Data being validated: {self.data}
+    #     Pydnantic Model: {self.model}.
+    #     Reported error: {self.message}."""
 
     def to_html_dict(self):
         return {

edsl/exceptions/scenarios.py

@@ -1,6 +1,13 @@
 import re
 import textwrap
 
+# from IPython.core.error import UsageError
+
+
+class AgentListError(Exception):
+    def __init__(self, message):
+        super().__init__(message)
+
 
 class ScenarioError(Exception):
     documentation = "https://docs.expectedparrot.com/en/latest/scenarios.html#module-edsl.scenarios.Scenario"

edsl/inference_services/AnthropicService.py

@@ -56,9 +56,6 @@ class AnthropicService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
-            _tpm = cls.get_tpm(cls)
-            _rpm = cls.get_rpm(cls)
-
             async def async_execute_model_call(
                 self,
                 user_prompt: str,

edsl/inference_services/AvailableModelCacheHandler.py

@@ -0,0 +1,184 @@
+from typing import List, Optional, get_args, Union
+from pathlib import Path
+import sqlite3
+from datetime import datetime
+import tempfile
+from platformdirs import user_cache_dir
+from dataclasses import dataclass
+import os
+
+from edsl.inference_services.data_structures import LanguageModelInfo, AvailableModels
+from edsl.enums import InferenceServiceLiteral
+
+
+class AvailableModelCacheHandler:
+    MAX_ROWS = 1000
+    CACHE_VALIDITY_HOURS = 48
+
+    def __init__(
+        self,
+        cache_validity_hours: int = 48,
+        verbose: bool = False,
+        testing_db_name: str = None,
+    ):
+        self.cache_validity_hours = cache_validity_hours
+        self.verbose = verbose
+
+        if testing_db_name:
+            self.cache_dir = Path(tempfile.mkdtemp())
+            self.db_path = self.cache_dir / testing_db_name
+        else:
+            self.cache_dir = Path(user_cache_dir("edsl", "model_availability"))
+            self.db_path = self.cache_dir / "available_models.db"
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        if os.path.exists(self.db_path):
+            if self.verbose:
+                print(f"Using existing cache DB: {self.db_path}")
+        else:
+            self._initialize_db()
+
+    @property
+    def path_to_db(self):
+        return self.db_path
+
+    def _initialize_db(self):
+        """Initialize the SQLite database with the required schema."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            # Drop the old table if it exists (for migration)
+            cursor.execute("DROP TABLE IF EXISTS model_cache")
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS model_cache (
+                    timestamp DATETIME NOT NULL,
+                    model_name TEXT NOT NULL,
+                    service_name TEXT NOT NULL,
+                    UNIQUE(model_name, service_name)
+                )
+            """
+            )
+            conn.commit()
+
+    def _prune_old_entries(self, conn: sqlite3.Connection):
+        """Delete oldest entries when MAX_ROWS is exceeded."""
+        cursor = conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM model_cache")
+        count = cursor.fetchone()[0]
+
+        if count > self.MAX_ROWS:
+            cursor.execute(
+                """
+                DELETE FROM model_cache 
+                WHERE rowid IN (
+                    SELECT rowid 
+                    FROM model_cache 
+                    ORDER BY timestamp ASC 
+                    LIMIT ?
+                )
+            """,
+                (count - self.MAX_ROWS,),
+            )
+            conn.commit()
+
+    @classmethod
+    def example_models(cls) -> List[LanguageModelInfo]:
+        return [
+            LanguageModelInfo(
+                "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "deep_infra"
+            ),
+            LanguageModelInfo("openai/gpt-4", "openai"),
+        ]
+
+    def add_models_to_cache(self, models_data: List[LanguageModelInfo]):
+        """Add new models to the cache, updating timestamps for existing entries."""
+        current_time = datetime.now()
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            for model in models_data:
+                cursor.execute(
+                    """
+                    INSERT INTO model_cache (timestamp, model_name, service_name)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(model_name, service_name) 
+                    DO UPDATE SET timestamp = excluded.timestamp
+                """,
+                    (current_time, model.model_name, model.service_name),
+                )
+
+            # self._prune_old_entries(conn)
+            conn.commit()
+
+    def reset_cache(self):
+        """Clear all entries from the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("DELETE FROM model_cache")
+            conn.commit()
+
+    @property
+    def num_cache_entries(self):
+        """Return the number of entries in the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM model_cache")
+            count = cursor.fetchone()[0]
+            return count
+
+    def models(
+        self,
+        service: Optional[InferenceServiceLiteral],
+    ) -> Union[None, AvailableModels]:
+        """Return the available models within the cache validity period."""
+        # if service is not None:
+        #    assert service in get_args(InferenceServiceLiteral)
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            valid_time = datetime.now().timestamp() - (self.cache_validity_hours * 3600)
+
+            if self.verbose:
+                print(f"Fetching all with timestamp greater than {valid_time}")
+
+            cursor.execute(
+                """
+                SELECT DISTINCT model_name, service_name
+                FROM model_cache
+                WHERE timestamp > ?
+                ORDER BY timestamp DESC
+            """,
+                (valid_time,),
+            )
+
+            results = cursor.fetchall()
+            if not results:
+                if self.verbose:
+                    print("No results found in cache DB.")
+                return None
+
+            matching_models = [
+                LanguageModelInfo(model_name=row[0], service_name=row[1])
+                for row in results
+            ]
+
+            if self.verbose:
+                print(f"Found {len(matching_models)} models in cache DB.")
+            if service:
+                matching_models = [
+                    model for model in matching_models if model.service_name == service
+                ]
+
+            return AvailableModels(matching_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
+    # cache_handler = AvailableModelCacheHandler(verbose=True)
+    # models_data = cache_handler.example_models()
+    # cache_handler.add_models_to_cache(models_data)
+    # print(cache_handler.models())
+    # cache_handler.clear_cache()
+    # print(cache_handler.models())

edsl/inference_services/AvailableModelFetcher.py

@@ -0,0 +1,209 @@
+from typing import Any, List, Tuple, Optional, Dict, TYPE_CHECKING, Union, Generator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections import UserList
+
+from edsl.inference_services.ServiceAvailability import ServiceAvailability
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.inference_services.data_structures import ModelNamesList
+from edsl.enums import InferenceServiceLiteral
+
+from edsl.inference_services.data_structures import LanguageModelInfo
+from edsl.inference_services.AvailableModelCacheHandler import (
+    AvailableModelCacheHandler,
+)
+
+
+from edsl.inference_services.data_structures import AvailableModels
+
+
+class AvailableModelFetcher:
+    """Fetches available models from the various services with JSON caching."""
+
+    service_availability = ServiceAvailability()
+    CACHE_VALIDITY_HOURS = 48  # Cache validity period in hours
+
+    def __init__(
+        self,
+        services: List["InferenceServiceABC"],
+        added_models: Dict[str, List[str]],
+        verbose: bool = False,
+        use_cache: bool = True,
+    ):
+        self.services = services
+        self.added_models = added_models
+        self._service_map = {
+            service._inference_service_: service for service in services
+        }
+        self.verbose = verbose
+        if use_cache:
+            self.cache_handler = AvailableModelCacheHandler()
+        else:
+            self.cache_handler = None
+
+    @property
+    def num_cache_entries(self):
+        return self.cache_handler.num_cache_entries
+
+    @property
+    def path_to_db(self):
+        return self.cache_handler.path_to_db
+
+    def reset_cache(self):
+        if self.cache_handler:
+            self.cache_handler.reset_cache()
+
+    def available(
+        self,
+        service: Optional[InferenceServiceABC] = None,
+        force_refresh: bool = False,
+    ) -> List[LanguageModelInfo]:
+        """
+        Get available models from all services, using cached data when available.
+
+        :param service: Optional[InferenceServiceABC] - If specified, only fetch models for this service.
+
+        >>> from edsl.inference_services.OpenAIService import OpenAIService
+        >>> af = AvailableModelFetcher([OpenAIService()], {})
+        >>> af.available(service="openai")
+        [LanguageModelInfo(model_name='...', service_name='openai'), ...]
+
+        Returns a list of [model, service_name, index] entries.
+        """
+
+        if service:  # they passed a specific service
+            matching_models, _ = self.get_available_models_by_service(
+                service=service, force_refresh=force_refresh
+            )
+            return matching_models
+
+        # Nope, we need to fetch them all
+        all_models = self._get_all_models()
+
+        # if self.cache_handler:
+        #    self.cache_handler.add_models_to_cache(all_models)
+
+        return all_models
+
+    def get_available_models_by_service(
+        self,
+        service: Union["InferenceServiceABC", InferenceServiceLiteral],
+        force_refresh: bool = False,
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        if not force_refresh:
+            models_from_cache = self.cache_handler.models(
+                service=service._inference_service_
+            )
+            if self.verbose:
+                print(
+                    "Searching cache for models with service name:",
+                    service._inference_service_,
+                )
+                print("Got models from cache:", models_from_cache)
+        else:
+            models_from_cache = None
+
+        if models_from_cache:
+            # print(f"Models from cache for {service}: {models_from_cache}")
+            # print(hasattr(models_from_cache[0], "service_name"))
+            return models_from_cache, service._inference_service_
+        else:
+            return self.get_available_models_by_service_fresh(service)
+
+    def get_available_models_by_service_fresh(
+        self, service: Union["InferenceServiceABC", InferenceServiceLiteral]
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service. This method always fetches fresh data.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        service_models: ModelNamesList = (
+            self.service_availability.get_service_available(service, warn=False)
+        )
+        service_name = service._inference_service_
+
+        models_list = AvailableModels(
+            [
+                LanguageModelInfo(
+                    model_name=model_name,
+                    service_name=service_name,
+                )
+                for model_name in service_models
+            ]
+        )
+        self.cache_handler.add_models_to_cache(models_list)  # update the cache
+        return models_list, service_name
+
+    def _fetch_service_by_service_name(
+        self, service_name: InferenceServiceLiteral
+    ) -> "InferenceServiceABC":
+        """The service name is the _inference_service_ attribute of the service."""
+        if service_name in self._service_map:
+            return self._service_map[service_name]
+        raise ValueError(f"Service {service_name} not found")
+
+    def _get_all_models(self, force_refresh=False) -> List[LanguageModelInfo]:
+        all_models = []
+        with ThreadPoolExecutor(max_workers=min(len(self.services), 10)) as executor:
+            future_to_service = {
+                executor.submit(
+                    self.get_available_models_by_service, service, force_refresh
+                ): service
+                for service in self.services
+            }
+
+            for future in as_completed(future_to_service):
+                try:
+                    models, service_name = future.result()
+                    all_models.extend(models)
+
+                    # Add any additional models for this service
+                    for model in self.added_models.get(service_name, []):
+                        all_models.append(
+                            LanguageModelInfo(
+                                model_name=model, service_name=service_name
+                            )
+                        )
+
+                except Exception as exc:
+                    print(f"Service query failed: {exc}")
+                    continue
+
+        return AvailableModels(all_models)
+
+
+def main():
+    from edsl.inference_services.OpenAIService import OpenAIService
+
+    af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # print(af.available(service="openai"))
+    all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models(
+        force_refresh=True
+    )
+    print(all_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    # main()
+
+    # from edsl.inference_services.OpenAIService import OpenAIService
+
+    # af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # # print(af.available(service="openai"))
+
+    # all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models()
+    # print(all_models)

edsl/inference_services/AwsBedrock.py

@@ -69,8 +69,6 @@ class AwsBedrockService(InferenceServiceABC):
             }
             input_token_name = cls.input_token_name
             output_token_name = cls.output_token_name
-            _rpm = cls.get_rpm(cls)
-            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/AzureAI.py

@@ -118,8 +118,6 @@ class AzureAIService(InferenceServiceABC):
                 "max_tokens": 512,
                 "top_p": 0.9,
             }
-            _rpm = cls.get_rpm(cls)
-            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/GoogleService.py

@@ -1,11 +1,11 @@
-import os
+# import os
 from typing import Any, Dict, List, Optional
 import google
 import google.generativeai as genai
 from google.generativeai.types import GenerationConfig
 from google.api_core.exceptions import InvalidArgument
 
-from edsl.exceptions import MissingAPIKeyError
+# from edsl.exceptions.general import MissingAPIKeyError
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.coop import Coop
@@ -39,10 +39,6 @@ class GoogleService(InferenceServiceABC):
 
     model_exclude_list = []
 
-    # @classmethod
-    # def available(cls) -> List[str]:
-    #     return ["gemini-pro", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.0-pro"]
-
     @classmethod
     def available(cls) -> List[str]:
         model_list = []
@@ -66,9 +62,6 @@ class GoogleService(InferenceServiceABC):
             output_token_name = cls.output_token_name
             _inference_service_ = cls._inference_service_
 
-            _tpm = cls.get_tpm(cls)
-            _rpm = cls.get_rpm(cls)
-
             _parameters_ = {
                 "temperature": 0.5,
                 "topP": 1,
@@ -77,7 +70,6 @@ class GoogleService(InferenceServiceABC):
                 "stopSequences": [],
             }
 
-            api_token = None
             model = None
 
             def __init__(self, *args, **kwargs):
@@ -102,7 +94,6 @@ class GoogleService(InferenceServiceABC):
 
                 if files_list is None:
                     files_list = []
-
                 genai.configure(api_key=self.api_token)
                 if (
                     system_prompt is not None