edsl 0.1.39.dev3__py3-none-any.whl → 0.1.39.dev4__py3-none-any.whl

edsl/inference_services/AvailableModelCacheHandler.py

@@ -0,0 +1,184 @@
+from typing import List, Optional, get_args, Union
+from pathlib import Path
+import sqlite3
+from datetime import datetime
+import tempfile
+from platformdirs import user_cache_dir
+from dataclasses import dataclass
+import os
+
+from edsl.inference_services.data_structures import LanguageModelInfo, AvailableModels
+from edsl.enums import InferenceServiceLiteral
+
+
+class AvailableModelCacheHandler:
+    MAX_ROWS = 1000
+    CACHE_VALIDITY_HOURS = 48
+
+    def __init__(
+        self,
+        cache_validity_hours: int = 48,
+        verbose: bool = False,
+        testing_db_name: str = None,
+    ):
+        self.cache_validity_hours = cache_validity_hours
+        self.verbose = verbose
+
+        if testing_db_name:
+            self.cache_dir = Path(tempfile.mkdtemp())
+            self.db_path = self.cache_dir / testing_db_name
+        else:
+            self.cache_dir = Path(user_cache_dir("edsl", "model_availability"))
+            self.db_path = self.cache_dir / "available_models.db"
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        if os.path.exists(self.db_path):
+            if self.verbose:
+                print(f"Using existing cache DB: {self.db_path}")
+        else:
+            self._initialize_db()
+
+    @property
+    def path_to_db(self):
+        return self.db_path
+
+    def _initialize_db(self):
+        """Initialize the SQLite database with the required schema."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            # Drop the old table if it exists (for migration)
+            cursor.execute("DROP TABLE IF EXISTS model_cache")
+            cursor.execute(
+                """
+                CREATE TABLE IF NOT EXISTS model_cache (
+                    timestamp DATETIME NOT NULL,
+                    model_name TEXT NOT NULL,
+                    service_name TEXT NOT NULL,
+                    UNIQUE(model_name, service_name)
+                )
+            """
+            )
+            conn.commit()
+
+    def _prune_old_entries(self, conn: sqlite3.Connection):
+        """Delete oldest entries when MAX_ROWS is exceeded."""
+        cursor = conn.cursor()
+        cursor.execute("SELECT COUNT(*) FROM model_cache")
+        count = cursor.fetchone()[0]
+
+        if count > self.MAX_ROWS:
+            cursor.execute(
+                """
+                DELETE FROM model_cache 
+                WHERE rowid IN (
+                    SELECT rowid 
+                    FROM model_cache 
+                    ORDER BY timestamp ASC 
+                    LIMIT ?
+                )
+            """,
+                (count - self.MAX_ROWS,),
+            )
+            conn.commit()
+
+    @classmethod
+    def example_models(cls) -> List[LanguageModelInfo]:
+        return [
+            LanguageModelInfo(
+                "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "deep_infra"
+            ),
+            LanguageModelInfo("openai/gpt-4", "openai"),
+        ]
+
+    def add_models_to_cache(self, models_data: List[LanguageModelInfo]):
+        """Add new models to the cache, updating timestamps for existing entries."""
+        current_time = datetime.now()
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            for model in models_data:
+                cursor.execute(
+                    """
+                    INSERT INTO model_cache (timestamp, model_name, service_name)
+                    VALUES (?, ?, ?)
+                    ON CONFLICT(model_name, service_name) 
+                    DO UPDATE SET timestamp = excluded.timestamp
+                """,
+                    (current_time, model.model_name, model.service_name),
+                )
+
+            # self._prune_old_entries(conn)
+            conn.commit()
+
+    def reset_cache(self):
+        """Clear all entries from the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("DELETE FROM model_cache")
+            conn.commit()
+
+    @property
+    def num_cache_entries(self):
+        """Return the number of entries in the cache."""
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM model_cache")
+            count = cursor.fetchone()[0]
+            return count
+
+    def models(
+        self,
+        service: Optional[InferenceServiceLiteral],
+    ) -> Union[None, AvailableModels]:
+        """Return the available models within the cache validity period."""
+        # if service is not None:
+        #    assert service in get_args(InferenceServiceLiteral)
+
+        with sqlite3.connect(self.db_path) as conn:
+            cursor = conn.cursor()
+            valid_time = datetime.now().timestamp() - (self.cache_validity_hours * 3600)
+
+            if self.verbose:
+                print(f"Fetching all with timestamp greater than {valid_time}")
+
+            cursor.execute(
+                """
+                SELECT DISTINCT model_name, service_name
+                FROM model_cache
+                WHERE timestamp > ?
+                ORDER BY timestamp DESC
+            """,
+                (valid_time,),
+            )
+
+            results = cursor.fetchall()
+            if not results:
+                if self.verbose:
+                    print("No results found in cache DB.")
+                return None
+
+            matching_models = [
+                LanguageModelInfo(model_name=row[0], service_name=row[1])
+                for row in results
+            ]
+
+            if self.verbose:
+                print(f"Found {len(matching_models)} models in cache DB.")
+            if service:
+                matching_models = [
+                    model for model in matching_models if model.service_name == service
+                ]
+
+            return AvailableModels(matching_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()
+    # cache_handler = AvailableModelCacheHandler(verbose=True)
+    # models_data = cache_handler.example_models()
+    # cache_handler.add_models_to_cache(models_data)
+    # print(cache_handler.models())
+    # cache_handler.clear_cache()
+    # print(cache_handler.models())

edsl/inference_services/AvailableModelFetcher.py

@@ -0,0 +1,215 @@
+from typing import Any, List, Tuple, Optional, Dict, TYPE_CHECKING, Union, Generator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from collections import UserList
+
+from edsl.inference_services.ServiceAvailability import ServiceAvailability
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.inference_services.data_structures import ModelNamesList
+from edsl.enums import InferenceServiceLiteral
+
+from edsl.inference_services.data_structures import LanguageModelInfo
+from edsl.inference_services.AvailableModelCacheHandler import (
+    AvailableModelCacheHandler,
+)
+
+
+from edsl.inference_services.data_structures import AvailableModels
+
+
+class AvailableModelFetcher:
+    """Fetches available models from the various services with JSON caching."""
+
+    service_availability = ServiceAvailability()
+    CACHE_VALIDITY_HOURS = 48  # Cache validity period in hours
+
+    def __init__(
+        self,
+        services: List["InferenceServiceABC"],
+        added_models: Dict[str, List[str]],
+        verbose: bool = False,
+        use_cache: bool = True,
+    ):
+        self.services = services
+        self.added_models = added_models
+        self._service_map = {
+            service._inference_service_: service for service in services
+        }
+        self.verbose = verbose
+        if use_cache:
+            self.cache_handler = AvailableModelCacheHandler()
+        else:
+            self.cache_handler = None
+
+    @property
+    def num_cache_entries(self):
+        return self.cache_handler.num_cache_entries
+
+    @property
+    def path_to_db(self):
+        return self.cache_handler.path_to_db
+
+    def reset_cache(self):
+        if self.cache_handler:
+            self.cache_handler.reset_cache()
+
+    def available(
+        self,
+        service: Optional[InferenceServiceABC] = None,
+        force_refresh: bool = False,
+    ) -> List[LanguageModelInfo]:
+        """
+        Get available models from all services, using cached data when available.
+
+        :param service: Optional[InferenceServiceABC] - If specified, only fetch models for this service.
+
+        >>> from edsl.inference_services.OpenAIService import OpenAIService
+        >>> af = AvailableModelFetcher([OpenAIService()], {})
+        >>> af.available(service="openai")
+        [LanguageModelInfo(model_name='...', service_name='openai'), ...]
+
+        Returns a list of [model, service_name, index] entries.
+        """
+
+        if service:  # they passed a specific service
+            matching_models, _ = self.get_available_models_by_service(
+                service=service, force_refresh=force_refresh
+            )
+            return matching_models
+
+        # Nope, we need to fetch them all
+        all_models = self._get_all_models()
+
+        # if self.cache_handler:
+        #    self.cache_handler.add_models_to_cache(all_models)
+
+        return all_models
+
+    def get_available_models_by_service(
+        self,
+        service: Union["InferenceServiceABC", InferenceServiceLiteral],
+        force_refresh: bool = False,
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        if not force_refresh:
+            models_from_cache = self.cache_handler.models(
+                service=service._inference_service_
+            )
+            if self.verbose:
+                print(
+                    "Searching cache for models with service name:",
+                    service._inference_service_,
+                )
+                print("Got models from cache:", models_from_cache)
+        else:
+            models_from_cache = None
+
+        if models_from_cache:
+            # print(f"Models from cache for {service}: {models_from_cache}")
+            # print(hasattr(models_from_cache[0], "service_name"))
+            return models_from_cache, service._inference_service_
+        else:
+            return self.get_available_models_by_service_fresh(service)
+
+    def get_available_models_by_service_fresh(
+        self, service: Union["InferenceServiceABC", InferenceServiceLiteral]
+    ) -> Tuple[AvailableModels, InferenceServiceLiteral]:
+        """Get models for a single service. This method always fetches fresh data.
+
+        :param service: InferenceServiceABC - e.g., OpenAIService or "openai"
+        :return: Tuple[List[LanguageModelInfo], InferenceServiceLiteral]
+        """
+        if isinstance(service, str):
+            service = self._fetch_service_by_service_name(service)
+
+        service_models: ModelNamesList = (
+            self.service_availability.get_service_available(service, warn=False)
+        )
+        service_name = service._inference_service_
+
+        if not service_models:
+            import warnings
+
+            warnings.warn(f"No models found for service {service_name}")
+            return [], service_name
+
+        models_list = AvailableModels(
+            [
+                LanguageModelInfo(
+                    model_name=model_name,
+                    service_name=service_name,
+                )
+                for model_name in service_models
+            ]
+        )
+        self.cache_handler.add_models_to_cache(models_list)  # update the cache
+        return models_list, service_name
+
+    def _fetch_service_by_service_name(
+        self, service_name: InferenceServiceLiteral
+    ) -> "InferenceServiceABC":
+        """The service name is the _inference_service_ attribute of the service."""
+        if service_name in self._service_map:
+            return self._service_map[service_name]
+        raise ValueError(f"Service {service_name} not found")
+
+    def _get_all_models(self, force_refresh=False) -> List[LanguageModelInfo]:
+        all_models = []
+        with ThreadPoolExecutor(max_workers=min(len(self.services), 10)) as executor:
+            future_to_service = {
+                executor.submit(
+                    self.get_available_models_by_service, service, force_refresh
+                ): service
+                for service in self.services
+            }
+
+            for future in as_completed(future_to_service):
+                try:
+                    models, service_name = future.result()
+                    all_models.extend(models)
+
+                    # Add any additional models for this service
+                    for model in self.added_models.get(service_name, []):
+                        all_models.append(
+                            LanguageModelInfo(
+                                model_name=model, service_name=service_name
+                            )
+                        )
+
+                except Exception as exc:
+                    print(f"Service query failed for service {service_name}: {exc}")
+                    continue
+
+        return AvailableModels(all_models)
+
+
+def main():
+    from edsl.inference_services.OpenAIService import OpenAIService
+
+    af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # print(af.available(service="openai"))
+    all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models(
+        force_refresh=True
+    )
+    print(all_models)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod(optionflags=doctest.ELLIPSIS)
+    # main()
+
+    # from edsl.inference_services.OpenAIService import OpenAIService
+
+    # af = AvailableModelFetcher([OpenAIService()], {}, verbose=True)
+    # # print(af.available(service="openai"))
+
+    # all_models = AvailableModelFetcher([OpenAIService()], {})._get_all_models()
+    # print(all_models)

edsl/inference_services/AwsBedrock.py

@@ -1,120 +1,118 @@
-import os
-from typing import Any, List, Optional
-import re
-import boto3
-from botocore.exceptions import ClientError
-from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
-from edsl.language_models.LanguageModel import LanguageModel
-import json
-from edsl.utilities.utilities import fix_partial_correct_response
-
-
-class AwsBedrockService(InferenceServiceABC):
-    """AWS Bedrock service class."""
-
-    _inference_service_ = "bedrock"
-    _env_key_name_ = (
-        "AWS_ACCESS_KEY_ID"  # or any other environment key for AWS credentials
-    )
-    key_sequence = ["output", "message", "content", 0, "text"]
-    input_token_name = "inputTokens"
-    output_token_name = "outputTokens"
-    usage_sequence = ["usage"]
-    model_exclude_list = [
-        "ai21.j2-grande-instruct",
-        "ai21.j2-jumbo-instruct",
-        "ai21.j2-mid",
-        "ai21.j2-mid-v1",
-        "ai21.j2-ultra",
-        "ai21.j2-ultra-v1",
-    ]
-    _models_list_cache: List[str] = []
-
-    @classmethod
-    def available(cls):
-        """Fetch available models from AWS Bedrock."""
-
-        region = os.getenv("AWS_REGION", "us-east-1")
-
-        if not cls._models_list_cache:
-            client = boto3.client("bedrock", region_name=region)
-            all_models_ids = [
-                x["modelId"] for x in client.list_foundation_models()["modelSummaries"]
-            ]
-        else:
-            all_models_ids = cls._models_list_cache
-
-        return [m for m in all_models_ids if m not in cls.model_exclude_list]
-
-    @classmethod
-    def create_model(
-        cls, model_name: str = "amazon.titan-tg1-large", model_class_name=None
-    ) -> LanguageModel:
-        if model_class_name is None:
-            model_class_name = cls.to_class_name(model_name)
-
-        class LLM(LanguageModel):
-            """
-            Child class of LanguageModel for interacting with AWS Bedrock models.
-            """
-
-            key_sequence = cls.key_sequence
-            usage_sequence = cls.usage_sequence
-            _inference_service_ = cls._inference_service_
-            _model_ = model_name
-            _parameters_ = {
-                "temperature": 0.5,
-                "max_tokens": 512,
-                "top_p": 0.9,
-            }
-            input_token_name = cls.input_token_name
-            output_token_name = cls.output_token_name
-            _rpm = cls.get_rpm(cls)
-            _tpm = cls.get_tpm(cls)
-
-            async def async_execute_model_call(
-                self,
-                user_prompt: str,
-                system_prompt: str = "",
-                files_list: Optional[List["FileStore"]] = None,
-            ) -> dict[str, Any]:
-                """Calls the AWS Bedrock API and returns the API response."""
-
-                api_token = (
-                    self.api_token
-                )  # call to check the if env variables are set.
-
-                region = os.getenv("AWS_REGION", "us-east-1")
-                client = boto3.client("bedrock-runtime", region_name=region)
-
-                conversation = [
-                    {
-                        "role": "user",
-                        "content": [{"text": user_prompt}],
-                    }
-                ]
-                system = [
-                    {
-                        "text": system_prompt,
-                    }
-                ]
-                try:
-                    response = client.converse(
-                        modelId=self._model_,
-                        messages=conversation,
-                        inferenceConfig={
-                            "maxTokens": self.max_tokens,
-                            "temperature": self.temperature,
-                            "topP": self.top_p,
-                        },
-                        # system=system,
-                        additionalModelRequestFields={},
-                    )
-                    return response
-                except (ClientError, Exception) as e:
-                    print(e)
-                    return {"error": str(e)}
-
-        LLM.__name__ = model_class_name
-
-        return LLM
+import os
+from typing import Any, List, Optional
+import re
+import boto3
+from botocore.exceptions import ClientError
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models.LanguageModel import LanguageModel
+import json
+from edsl.utilities.utilities import fix_partial_correct_response
+
+
+class AwsBedrockService(InferenceServiceABC):
+    """AWS Bedrock service class."""
+
+    _inference_service_ = "bedrock"
+    _env_key_name_ = (
+        "AWS_ACCESS_KEY_ID"  # or any other environment key for AWS credentials
+    )
+    key_sequence = ["output", "message", "content", 0, "text"]
+    input_token_name = "inputTokens"
+    output_token_name = "outputTokens"
+    usage_sequence = ["usage"]
+    model_exclude_list = [
+        "ai21.j2-grande-instruct",
+        "ai21.j2-jumbo-instruct",
+        "ai21.j2-mid",
+        "ai21.j2-mid-v1",
+        "ai21.j2-ultra",
+        "ai21.j2-ultra-v1",
+    ]
+    _models_list_cache: List[str] = []
+
+    @classmethod
+    def available(cls):
+        """Fetch available models from AWS Bedrock."""
+
+        region = os.getenv("AWS_REGION", "us-east-1")
+
+        if not cls._models_list_cache:
+            client = boto3.client("bedrock", region_name=region)
+            all_models_ids = [
+                x["modelId"] for x in client.list_foundation_models()["modelSummaries"]
+            ]
+        else:
+            all_models_ids = cls._models_list_cache
+
+        return [m for m in all_models_ids if m not in cls.model_exclude_list]
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "amazon.titan-tg1-large", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            """
+            Child class of LanguageModel for interacting with AWS Bedrock models.
+            """
+
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            _inference_service_ = cls._inference_service_
+            _model_ = model_name
+            _parameters_ = {
+                "temperature": 0.5,
+                "max_tokens": 512,
+                "top_p": 0.9,
+            }
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str = "",
+                files_list: Optional[List["FileStore"]] = None,
+            ) -> dict[str, Any]:
+                """Calls the AWS Bedrock API and returns the API response."""
+
+                api_token = (
+                    self.api_token
+                )  # call to check the if env variables are set.
+
+                region = os.getenv("AWS_REGION", "us-east-1")
+                client = boto3.client("bedrock-runtime", region_name=region)
+
+                conversation = [
+                    {
+                        "role": "user",
+                        "content": [{"text": user_prompt}],
+                    }
+                ]
+                system = [
+                    {
+                        "text": system_prompt,
+                    }
+                ]
+                try:
+                    response = client.converse(
+                        modelId=self._model_,
+                        messages=conversation,
+                        inferenceConfig={
+                            "maxTokens": self.max_tokens,
+                            "temperature": self.temperature,
+                            "topP": self.top_p,
+                        },
+                        # system=system,
+                        additionalModelRequestFields={},
+                    )
+                    return response
+                except (ClientError, Exception) as e:
+                    print(e)
+                    return {"error": str(e)}
+
+        LLM.__name__ = model_class_name
+
+        return LLM