edsl 0.1.38.dev4__py3-none-any.whl → 0.1.39__py3-none-any.whl

edsl/inference_services/AnthropicService.py

@@ -11,21 +11,27 @@ class AnthropicService(InferenceServiceABC):
 
     _inference_service_ = "anthropic"
     _env_key_name_ = "ANTHROPIC_API_KEY"
-    key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    key_sequence = ["content", 0, "text"]
     usage_sequence = ["usage"]
     input_token_name = "input_tokens"
     output_token_name = "output_tokens"
     model_exclude_list = []
 
+    @classmethod
+    def get_model_list(cls, api_key: str = None):
+
+        import requests
+
+        if api_key is None:
+            api_key = os.environ.get("ANTHROPIC_API_KEY")
+        headers = {"x-api-key": api_key, "anthropic-version": "2023-06-01"}
+        response = requests.get("https://api.anthropic.com/v1/models", headers=headers)
+        model_names = [m["id"] for m in response.json()["data"]]
+        return model_names
+
     @classmethod
     def available(cls):
-        # TODO - replace with an API call
-        return [
-            "claude-3-5-sonnet-20240620",
-            "claude-3-opus-20240229",
-            "claude-3-sonnet-20240229",
-            "claude-3-haiku-20240307",
-        ]
+        return cls.get_model_list()
 
     @classmethod
     def create_model(
@@ -56,29 +62,42 @@ class AnthropicService(InferenceServiceABC):
                 "top_logprobs": 3,
             }
 
-            _tpm = cls.get_tpm(cls)
-            _rpm = cls.get_rpm(cls)
-
             async def async_execute_model_call(
                 self,
                 user_prompt: str,
                 system_prompt: str = "",
                 files_list: Optional[List["Files"]] = None,
             ) -> dict[str, Any]:
-                """Calls the OpenAI API and returns the API response."""
+                """Calls the Anthropic API and returns the API response."""
 
-                api_key = os.environ.get("ANTHROPIC_API_KEY")
-                client = AsyncAnthropic(api_key=api_key)
+                messages = [
+                    {
+                        "role": "user",
+                        "content": [{"type": "text", "text": user_prompt}],
+                    }
+                ]
+                if files_list:
+                    for file_entry in files_list:
+                        encoded_image = file_entry.base64_string
+                        messages[0]["content"].append(
+                            {
+                                "type": "image",
+                                "source": {
+                                    "type": "base64",
+                                    "media_type": file_entry.mime_type,
+                                    "data": encoded_image,
+                                },
+                            }
+                        )
+                # breakpoint()
+                client = AsyncAnthropic(api_key=self.api_token)
 
                 response = await client.messages.create(
                     model=model_name,
                     max_tokens=self.max_tokens,
                     temperature=self.temperature,
-                    system=system_prompt,
-                    messages=[
-                        #                        {"role": "system", "content": system_prompt},
-                        {"role": "user", "content": user_prompt},
-                    ],
+                    system=system_prompt,  # note that the Anthropic API uses "system" parameter rather than put it in the message
+                    messages=messages,
                 )
                 return response.model_dump()
 

edsl/inference_services/AvailableModelCacheHandler.py

@@ -0,0 +1,184 @@
+from typing import List, Optional, get_args, Union
+from pathlib import Path
+import sqlite3
+from datetime import datetime
+import tempfile
+from platformdirs import user_cache_dir
+from dataclasses import dataclass
+import os
+
+from edsl.inference_services.data_structures import LanguageModelInfo, AvailableModels
+from edsl.enums import InferenceServiceLiteral
+
+
+class AvailableModelCacheHandler:
+    MAX_ROWS = 1000
+    CACHE_VALIDITY_HOURS = 48
+
+    def __init__(
+        self,
+        cache_validity_hours: int = 48,
+        verbose: bool = False,
+        testing_db_name: str = None,
+    ):
+        self.cache_validity_hours = cache_validity_hours
+        self.verbose = verbose
+
+        if testing_db_name:
+            self.cache_dir = Path(tempfile.mkdtemp())
+            self.db_path = self.cache_dir / testing_db_name
+        else:
+            self.cache_dir = Path(user_cache_dir("edsl", "model_availability"))
+            self.db_path = self.cache_dir / "available_models.db"
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        if os.path.exists(self.db_path):
+            if self.verbose:
+                print(f"Using existing cache DB: {self.db_path}")
+        else:
+            self._initialize_db()
+
+    @property
+    def path_to_db(self):
+        return self.db_path
+
+    def _initialize_db(self):
+        """Initialize the SQLite database with the required schema."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            # Drop the old table if it exists (for migration)
+            cursor.execute("DROP TABLE IF EXISTS model_cache")
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS model_cache (
+                    timestamp DATETIME NOT NULL,
+                    model_name TEXT NOT NULL,
+                    service_name TEXT NOT NULL,
+                    UNIQUE(model_name, service_name)
+                )
+            """
+            )
+            conn.commit()
+
+    def _prune_old_entries(self, conn: sqlite3.Connection):
+        """Delete oldest entries when MAX_ROWS is exceeded."""
+        cursor = conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM model_cache")
+        count = cursor.fetchone()[0]
+
+        if count > self.MAX_ROWS:
+            cursor.execute(
+                """
+                DELETE FROM model_cache 
+                WHERE rowid IN (
+                    SELECT rowid 
+                    FROM model_cache 
+                    ORDER BY timestamp ASC 
+                    LIMIT ?
+                )
+            """,
+                (count - self.MAX_ROWS,),
+            )
+            conn.commit()
+
+    @classmethod
+    def example_models(cls) -> List[LanguageModelInfo]:
+        return [
+            LanguageModelInfo(
+                "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "deep_infra"
+            ),
+            LanguageModelInfo("openai/gpt-4", "openai"),
+        ]
+
+    def add_models_to_cache(self, models_data: List[LanguageModelInfo]):
+        """Add new models to the cache, updating timestamps for existing entries."""
+        current_time = datetime.now()
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            for model in models_data:
+                cursor.execute(
+                    """
+                    INSERT INTO model_cache (timestamp, model_name, service_name)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(model_name, service_name) 
+                    DO UPDATE SET timestamp = excluded.timestamp
+                """,
+                    (current_time, model.model_name, model.service_name),
+                )
+
+            # self._prune_old_entries(conn)
+            conn.commit()
+
+    def reset_cache(self):
+        """Clear all entries from the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("DELETE FROM model_cache")
+            conn.commit()
+
+    @property
+    def num_cache_entries(self):
+        """Return the number of entries in the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM model_cache")
+            count = cursor.fetchone()[0]
+            return count
+
+    def models(
+        self,
+        service: Optional[InferenceServiceLiteral],
+    ) -> Union[None, AvailableModels]:
+        """Return the available models within the cache validity period."""
+        # if service is not None:
+        #    assert service in get_args(InferenceServiceLiteral)
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            valid_time = datetime.now().timestamp() - (self.cache_validity_hours * 3600)
+
+            if self.verbose:
+                print(f"Fetching all with timestamp greater than {valid_time}")
+
+            cursor.execute(
+                """
+                SELECT DISTINCT model_name, service_name
+                FROM model_cache
+                WHERE timestamp > ?
+                ORDER BY timestamp DESC
+            """,
+                (valid_time,),
+            )
+
+            results = cursor.fetchall()
+            if not results:
+                if self.verbose:
+                    print("No results found in cache DB.")
+                return None
+
+            matching_models = [
+                LanguageModelInfo(model_name=row[0], service_name=row[1])
+                for row in results
+            ]
+
+            if self.verbose:
+                print(f"Found {len(matching_models)} models in cache DB.")
+            if service:
+                matching_models = [
+                    model for model in matching_models if model.service_name == service
+                ]
+
+            return AvailableModels(matching_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
+    # cache_handler = AvailableModelCacheHandler(verbose=True)
+    # models_data = cache_handler.example_models()
+    # cache_handler.add_models_to_cache(models_data)
+    # print(cache_handler.models())
+    # cache_handler.clear_cache()
+    # print(cache_handler.models())

edsl/inference_services/AvailableModelFetcher.py

@@ -0,0 +1,215 @@
+from typing import Any, List, Tuple, Optional, Dict, TYPE_CHECKING, Union, Generator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections import UserList
+
+from edsl.inference_services.ServiceAvailability import ServiceAvailability
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.inference_services.data_structures import ModelNamesList
+from edsl.enums import InferenceServiceLiteral
+
+from edsl.inference_services.data_structures import LanguageModelInfo
+from edsl.inference_services.AvailableModelCacheHandler import (
+    AvailableModelCacheHandler,
+)
+
+
+from edsl.inference_services.data_structures import AvailableModels
+
+
+class AvailableModelFetcher:
+    """Fetches available models from the various services with JSON caching."""
+
+    service_availability = ServiceAvailability()
+    CACHE_VALIDITY_HOURS = 48  # Cache validity period in hours
+
+    def __init__(
+        self,
+        services: List["InferenceServiceABC"],
+        added_models: Dict[str, List[str]],
+        verbose: bool = False,
+        use_cache: bool = True,
+    ):
+        self.services = services
+        self.added_models = added_models
+        self._service_map = {
+            service._inference_service_: service for service in services
+        }
+        self.verbose = verbose
+        if use_cache:
+            self.cache_handler = AvailableModelCacheHandler()
+        else:
+            self.cache_handler = None
+
+    @property
+    def num_cache_entries(self):
+        return self.cache_handler.num_cache_entries
+
+    @property
+    def path_to_db(self):
+        return self.cache_handler.path_to_db
+
+    def reset_cache(self):
+        if self.cache_handler:
+            self.cache_handler.reset_cache()
+
+    def available(
+        self,
+        service: Optional[InferenceServiceABC] = None,
+        force_refresh: bool = False,
+    ) -> List[LanguageModelInfo]:
+        """
+        Get available models from all services, using cached data when available.
+
+        :param service: Optional[InferenceServiceABC] - If specified, only fetch models for this service.
+
+        >>> from edsl.inference_services.OpenAIService import OpenAIService
+        >>> af = AvailableModelFetcher([OpenAIService()], {})
+        >>> af.available(service="openai")
+        [LanguageModelInfo(model_name='...', service_name='openai'), ...]
+
+        Returns a list of [model, service_name, index] entries.
+        """
+
+        if service:  # they passed a specific service
+            matching_models, _ = self.get_available_models_by_service(
+                service=service, force_refresh=force_refresh
+            )
+            return matching_models
+
+        # Nope, we need to fetch them all
+        all_models = self._get_all_models()
+
+        # if self.cache_handler:
+        #    self.cache_handler.add_models_to_cache(all_models)
+
+        return all_models
+
+    def get_available_models_by_service(
+        self,
+        service: Union["InferenceServiceABC", InferenceServiceLiteral],
+        force_refresh: bool = False,
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        if not force_refresh:
+            models_from_cache = self.cache_handler.models(
+                service=service._inference_service_
+            )
+            if self.verbose:
+                print(
+                    "Searching cache for models with service name:",
+                    service._inference_service_,
+                )
+                print("Got models from cache:", models_from_cache)
+        else:
+            models_from_cache = None
+
+        if models_from_cache:
+            # print(f"Models from cache for {service}: {models_from_cache}")
+            # print(hasattr(models_from_cache[0], "service_name"))
+            return models_from_cache, service._inference_service_
+        else:
+            return self.get_available_models_by_service_fresh(service)
+
+    def get_available_models_by_service_fresh(
+        self, service: Union["InferenceServiceABC", InferenceServiceLiteral]
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service. This method always fetches fresh data.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        service_models: ModelNamesList = (
+            self.service_availability.get_service_available(service, warn=False)
+        )
+        service_name = service._inference_service_
+
+        if not service_models:
+            import warnings
+
+            warnings.warn(f"No models found for service {service_name}")
+            return [], service_name
+
+        models_list = AvailableModels(
+            [
+                LanguageModelInfo(
+                    model_name=model_name,
+                    service_name=service_name,
+                )
+                for model_name in service_models
+            ]
+        )
+        self.cache_handler.add_models_to_cache(models_list)  # update the cache
+        return models_list, service_name
+
+    def _fetch_service_by_service_name(
+        self, service_name: InferenceServiceLiteral
+    ) -> "InferenceServiceABC":
+        """The service name is the _inference_service_ attribute of the service."""
+        if service_name in self._service_map:
+            return self._service_map[service_name]
+        raise ValueError(f"Service {service_name} not found")
+
+    def _get_all_models(self, force_refresh=False) -> List[LanguageModelInfo]:
+        all_models = []
+        with ThreadPoolExecutor(max_workers=min(len(self.services), 10)) as executor:
+            future_to_service = {
+                executor.submit(
+                    self.get_available_models_by_service, service, force_refresh
+                ): service
+                for service in self.services
+            }
+
+            for future in as_completed(future_to_service):
+                try:
+                    models, service_name = future.result()
+                    all_models.extend(models)
+
+                    # Add any additional models for this service
+                    for model in self.added_models.get(service_name, []):
+                        all_models.append(
+                            LanguageModelInfo(
+                                model_name=model, service_name=service_name
+                            )
+                        )
+
+                except Exception as exc:
+                    print(f"Service query failed for service {service_name}: {exc}")
+                    continue
+
+        return AvailableModels(all_models)
+
+
+def main():
+    from edsl.inference_services.OpenAIService import OpenAIService
+
+    af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # print(af.available(service="openai"))
+    all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models(
+        force_refresh=True
+    )
+    print(all_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    # main()
+
+    # from edsl.inference_services.OpenAIService import OpenAIService
+
+    # af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # # print(af.available(service="openai"))
+
+    # all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models()
+    # print(all_models)

edsl/inference_services/AwsBedrock.py

@@ -69,8 +69,6 @@ class AwsBedrockService(InferenceServiceABC):
             }
             input_token_name = cls.input_token_name
             output_token_name = cls.output_token_name
-            _rpm = cls.get_rpm(cls)
-            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/AzureAI.py

@@ -118,8 +118,6 @@ class AzureAIService(InferenceServiceABC):
                 "max_tokens": 512,
                 "top_p": 0.9,
             }
-            _rpm = cls.get_rpm(cls)
-            _tpm = cls.get_tpm(cls)
 
             async def async_execute_model_call(
                 self,

edsl/inference_services/GoogleService.py

@@ -1,11 +1,11 @@
-import os
+# import os
 from typing import Any, Dict, List, Optional
 import google
 import google.generativeai as genai
 from google.generativeai.types import GenerationConfig
 from google.api_core.exceptions import InvalidArgument
 
-from edsl.exceptions import MissingAPIKeyError
+# from edsl.exceptions.general import MissingAPIKeyError
 from edsl.language_models.LanguageModel import LanguageModel
 from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
 from edsl.coop import Coop
@@ -39,18 +39,18 @@ class GoogleService(InferenceServiceABC):
 
     model_exclude_list = []
 
-    # @classmethod
-    # def available(cls) -> List[str]:
-    #     return ["gemini-pro", "gemini-1.5-pro", "gemini-1.5-flash", "gemini-1.0-pro"]
-
     @classmethod
-    def available(cls) -> List[str]:
+    def get_model_list(cls):
         model_list = []
         for m in genai.list_models():
             if "generateContent" in m.supported_generation_methods:
                 model_list.append(m.name.split("/")[-1])
         return model_list
 
+    @classmethod
+    def available(cls) -> List[str]:
+        return cls.get_model_list()
+
     @classmethod
     def create_model(
         cls, model_name: str = "gemini-pro", model_class_name=None
@@ -66,9 +66,6 @@ class GoogleService(InferenceServiceABC):
             output_token_name = cls.output_token_name
             _inference_service_ = cls._inference_service_
 
-            _tpm = cls.get_tpm(cls)
-            _rpm = cls.get_rpm(cls)
-
             _parameters_ = {
                 "temperature": 0.5,
                 "topP": 1,
@@ -77,7 +74,6 @@ class GoogleService(InferenceServiceABC):
                 "stopSequences": [],
             }
 
-            api_token = None
             model = None
 
             def __init__(self, *args, **kwargs):
@@ -102,7 +98,6 @@ class GoogleService(InferenceServiceABC):
 
                 if files_list is None:
                     files_list = []
-
                 genai.configure(api_key=self.api_token)
                 if (
                     system_prompt is not None