edsl 0.1.14__py3-none-any.whl → 0.1.40__py3-none-any.whl

edsl/inference_services/AzureAI.py

@@ -0,0 +1,215 @@
+import os
+from typing import Any, Optional, List
+import re
+from openai import AsyncAzureOpenAI
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models.LanguageModel import LanguageModel
+
+from azure.ai.inference.aio import ChatCompletionsClient
+from azure.core.credentials import AzureKeyCredential
+from azure.ai.inference.models import SystemMessage, UserMessage
+import asyncio
+import json
+from edsl.utilities.utilities import fix_partial_correct_response
+
+
+def json_handle_none(value: Any) -> Any:
+    """
+    Handle None values during JSON serialization.
+    - Return "null" if the value is None. Otherwise, don't return anything.
+    """
+    if value is None:
+        return "null"
+
+
+class AzureAIService(InferenceServiceABC):
+    """Azure AI service class."""
+
+    # key_sequence = ["content", 0, "text"]  # ["content"][0]["text"]
+    key_sequence = ["choices", 0, "message", "content"]
+    usage_sequence = ["usage"]
+    input_token_name = "prompt_tokens"
+    output_token_name = "completion_tokens"
+
+    _inference_service_ = "azure"
+    _env_key_name_ = (
+        "AZURE_ENDPOINT_URL_AND_KEY"  # Environment variable for Azure API key
+    )
+    _model_id_to_endpoint_and_key = {}
+    model_exclude_list = [
+        "Cohere-command-r-plus-xncmg",
+        "Mistral-Nemo-klfsi",
+        "Mistral-large-2407-ojfld",
+    ]
+
+    @classmethod
+    def available(cls):
+        out = []
+        azure_endpoints = os.getenv("AZURE_ENDPOINT_URL_AND_KEY", None)
+        if not azure_endpoints:
+            raise EnvironmentError(f"AZURE_ENDPOINT_URL_AND_KEY is not defined")
+        azure_endpoints = azure_endpoints.split(",")
+        for data in azure_endpoints:
+            try:
+                # data has this format for non openai models https://model_id.azure_endpoint:azure_key
+                _, endpoint, azure_endpoint_key = data.split(":")
+                if "openai" not in endpoint:
+                    model_id = endpoint.split(".")[0].replace("/", "")
+                    out.append(model_id)
+                    cls._model_id_to_endpoint_and_key[model_id] = {
+                        "endpoint": f"https:{endpoint}",
+                        "azure_endpoint_key": azure_endpoint_key,
+                    }
+                else:
+                    # data has this format for openai models ,https://azure_project_id.openai.azure.com/openai/deployments/gpt-4o-mini/chat/completions?api-version=2023-03-15-preview:azure_key
+                    if "/deployments/" in endpoint:
+                        start_idx = endpoint.index("/deployments/") + len(
+                            "/deployments/"
+                        )
+                        end_idx = (
+                            endpoint.index("/", start_idx)
+                            if "/" in endpoint[start_idx:]
+                            else len(endpoint)
+                        )
+                        model_id = endpoint[start_idx:end_idx]
+                        api_version_value = None
+                        if "api-version=" in endpoint:
+                            start_idx = endpoint.index("api-version=") + len(
+                                "api-version="
+                            )
+                            end_idx = (
+                                endpoint.index("&", start_idx)
+                                if "&" in endpoint[start_idx:]
+                                else len(endpoint)
+                            )
+                            api_version_value = endpoint[start_idx:end_idx]
+
+                        cls._model_id_to_endpoint_and_key[f"azure:{model_id}"] = {
+                            "endpoint": f"https:{endpoint}",
+                            "azure_endpoint_key": azure_endpoint_key,
+                            "api_version": api_version_value,
+                        }
+                        out.append(f"azure:{model_id}")
+
+            except Exception as e:
+                raise e
+        return [m for m in out if m not in cls.model_exclude_list]
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "azureai", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            """
+            Child class of LanguageModel for interacting with Azure OpenAI models.
+            """
+
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _inference_service_ = cls._inference_service_
+            _model_ = model_name
+            _parameters_ = {
+                "temperature": 0.5,
+                "max_tokens": 512,
+                "top_p": 0.9,
+            }
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str = "",
+                files_list: Optional[List["FileStore"]] = None,
+            ) -> dict[str, Any]:
+                """Calls the Azure OpenAI API and returns the API response."""
+
+                try:
+                    api_key = cls._model_id_to_endpoint_and_key[model_name][
+                        "azure_endpoint_key"
+                    ]
+                except:
+                    api_key = None
+
+                if not api_key:
+                    raise EnvironmentError(
+                        f"AZURE_ENDPOINT_URL_AND_KEY doesn't have the endpoint:key pair for your model: {model_name}"
+                    )
+
+                try:
+                    endpoint = cls._model_id_to_endpoint_and_key[model_name]["endpoint"]
+                except:
+                    endpoint = None
+
+                if not endpoint:
+                    raise EnvironmentError(
+                        f"AZURE_ENDPOINT_URL_AND_KEY doesn't have the endpoint:key pair for your model: {model_name}"
+                    )
+
+                if "openai" not in endpoint:
+                    client = ChatCompletionsClient(
+                        endpoint=endpoint,
+                        credential=AzureKeyCredential(api_key),
+                        temperature=self.temperature,
+                        top_p=self.top_p,
+                        max_tokens=self.max_tokens,
+                    )
+                    try:
+                        response = await client.complete(
+                            messages=[
+                                SystemMessage(content=system_prompt),
+                                UserMessage(content=user_prompt),
+                            ],
+                            # model_extras={"safe_mode": True},
+                        )
+                        await client.close()
+                        return response.as_dict()
+                    except Exception as e:
+                        await client.close()
+                        return {"error": str(e)}
+                else:
+                    api_version = cls._model_id_to_endpoint_and_key[model_name][
+                        "api_version"
+                    ]
+                    client = AsyncAzureOpenAI(
+                        azure_endpoint=endpoint,
+                        api_version=api_version,
+                        api_key=api_key,
+                    )
+                    response = await client.chat.completions.create(
+                        model=model_name,
+                        messages=[
+                            {
+                                "role": "user",
+                                "content": user_prompt,  # Your question can go here
+                            },
+                        ],
+                    )
+                    return response.model_dump()
+
+            # @staticmethod
+            # def parse_response(raw_response: dict[str, Any]) -> str:
+            #     """Parses the API response and returns the response text."""
+            #     if (
+            #         raw_response
+            #         and "choices" in raw_response
+            #         and raw_response["choices"]
+            #     ):
+            #         response = raw_response["choices"][0]["message"]["content"]
+            #         pattern = r"^```json(?:\\n|\n)(.+?)(?:\\n|\n)```$"
+            #         match = re.match(pattern, response, re.DOTALL)
+            #         if match:
+            #             return match.group(1)
+            #         else:
+            #             out = fix_partial_correct_response(response)
+            #             if "error" not in out:
+            #                 response = out["extracted_json"]
+            #             return response
+            #     return "Error parsing response"
+
+        LLM.__name__ = model_class_name
+
+        return LLM

edsl/inference_services/DeepInfraService.py

@@ -0,0 +1,18 @@
+import aiohttp
+import json
+import requests
+from typing import Any, List
+
+# from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.language_models import LanguageModel
+
+from edsl.inference_services.OpenAIService import OpenAIService
+
+
+class DeepInfraService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "deep_infra"
+    _env_key_name_ = "DEEP_INFRA_API_KEY"
+    _base_url_ = "https://api.deepinfra.com/v1/openai"
+    _models_list_cache: List[str] = []

edsl/inference_services/GoogleService.py

@@ -0,0 +1,143 @@
+# import os
+from typing import Any, Dict, List, Optional
+import google
+import google.generativeai as genai
+from google.generativeai.types import GenerationConfig
+from google.api_core.exceptions import InvalidArgument
+
+# from edsl.exceptions.general import MissingAPIKeyError
+from edsl.language_models.LanguageModel import LanguageModel
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.coop import Coop
+
+safety_settings = [
+    {
+        "category": "HARM_CATEGORY_HARASSMENT",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_HATE_SPEECH",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
+        "threshold": "BLOCK_NONE",
+    },
+    {
+        "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
+        "threshold": "BLOCK_NONE",
+    },
+]
+
+
+class GoogleService(InferenceServiceABC):
+    _inference_service_ = "google"
+    key_sequence = ["candidates", 0, "content", "parts", 0, "text"]
+    usage_sequence = ["usage_metadata"]
+    input_token_name = "prompt_token_count"
+    output_token_name = "candidates_token_count"
+
+    model_exclude_list = []
+
+    @classmethod
+    def get_model_list(cls):
+        model_list = []
+        for m in genai.list_models():
+            if "generateContent" in m.supported_generation_methods:
+                model_list.append(m.name.split("/")[-1])
+        return model_list
+
+    @classmethod
+    def available(cls) -> List[str]:
+        return cls.get_model_list()
+
+    @classmethod
+    def create_model(
+        cls, model_name: str = "gemini-pro", model_class_name=None
+    ) -> LanguageModel:
+        if model_class_name is None:
+            model_class_name = cls.to_class_name(model_name)
+
+        class LLM(LanguageModel):
+            _model_ = model_name
+            key_sequence = cls.key_sequence
+            usage_sequence = cls.usage_sequence
+            input_token_name = cls.input_token_name
+            output_token_name = cls.output_token_name
+            _inference_service_ = cls._inference_service_
+
+            _parameters_ = {
+                "temperature": 0.5,
+                "topP": 1,
+                "topK": 1,
+                "maxOutputTokens": 2048,
+                "stopSequences": [],
+            }
+
+            model = None
+
+            def __init__(self, *args, **kwargs):
+                super().__init__(*args, **kwargs)
+
+            def get_generation_config(self) -> GenerationConfig:
+                return GenerationConfig(
+                    temperature=self.temperature,
+                    top_p=self.topP,
+                    top_k=self.topK,
+                    max_output_tokens=self.maxOutputTokens,
+                    stop_sequences=self.stopSequences,
+                )
+
+            async def async_execute_model_call(
+                self,
+                user_prompt: str,
+                system_prompt: str = "",
+                files_list: Optional["Files"] = None,
+            ) -> Dict[str, Any]:
+                generation_config = self.get_generation_config()
+
+                if files_list is None:
+                    files_list = []
+                genai.configure(api_key=self.api_token)
+                if (
+                    system_prompt is not None
+                    and system_prompt != ""
+                    and self._model_ != "gemini-pro"
+                ):
+                    try:
+                        self.generative_model = genai.GenerativeModel(
+                            self._model_,
+                            safety_settings=safety_settings,
+                            system_instruction=system_prompt,
+                        )
+                    except InvalidArgument as e:
+                        print(
+                            f"This model, {self._model_}, does not support system_instruction"
+                        )
+                        print("Will add system_prompt to user_prompt")
+                        user_prompt = f"{system_prompt}\n{user_prompt}"
+                else:
+                    self.generative_model = genai.GenerativeModel(
+                        self._model_,
+                        safety_settings=safety_settings,
+                    )
+                combined_prompt = [user_prompt]
+                for file in files_list:
+                    if "google" not in file.external_locations:
+                        _ = file.upload_google()
+                    gen_ai_file = google.generativeai.types.file_types.File(
+                        file.external_locations["google"]
+                    )
+                    combined_prompt.append(gen_ai_file)
+
+                response = await self.generative_model.generate_content_async(
+                    combined_prompt, generation_config=generation_config
+                )
+                return response.to_dict()
+
+        LLM.__name__ = model_name
+        return LLM
+
+
+if __name__ == "__main__":
+    pass

edsl/inference_services/GroqService.py

@@ -0,0 +1,20 @@
+from typing import Any, List
+from edsl.inference_services.OpenAIService import OpenAIService
+
+import groq
+
+
+class GroqService(OpenAIService):
+    """DeepInfra service class."""
+
+    _inference_service_ = "groq"
+    _env_key_name_ = "GROQ_API_KEY"
+
+    _sync_client_ = groq.Groq
+    _async_client_ = groq.AsyncGroq
+
+    model_exclude_list = ["whisper-large-v3", "distil-whisper-large-v3-en"]
+
+    # _base_url_ = "https://api.deepinfra.com/v1/openai"
+    _base_url_ = None
+    _models_list_cache: List[str] = []

edsl/inference_services/InferenceServiceABC.py

@@ -0,0 +1,80 @@
+from abc import abstractmethod, ABC
+import re
+from datetime import datetime, timedelta
+from edsl.config import CONFIG
+
+
+class InferenceServiceABC(ABC):
+    """
+    Abstract class for inference services.
+    """
+
+    _coop_config_vars = None
+
+    def __init_subclass__(cls):
+        """
+        Check that the subclass has the required attributes.
+        - `key_sequence` attribute determines...
+        - `model_exclude_list` attribute determines...
+        """
+        must_have_attributes = [
+            "key_sequence",
+            "model_exclude_list",
+            "usage_sequence",
+            "input_token_name",
+            "output_token_name",
+        ]
+        for attr in must_have_attributes:
+            if not hasattr(cls, attr):
+                raise NotImplementedError(
+                    f"Class {cls.__name__} must have a '{attr}' attribute."
+                )
+
+    @property
+    def service_name(self):
+        return self._inference_service_
+
+    @classmethod
+    def _should_refresh_coop_config_vars(cls):
+        """
+        Returns True if config vars have been fetched over 24 hours ago, and False otherwise.
+        """
+
+        if cls._last_config_fetch is None:
+            return True
+        return (datetime.now() - cls._last_config_fetch) > timedelta(hours=24)
+
+    @abstractmethod
+    def available() -> list[str]:
+        """
+        Returns a list of available models for the service.
+        """
+        pass
+
+    @abstractmethod
+    def create_model():
+        """
+        Returns a LanguageModel object.
+        """
+        pass
+
+    @staticmethod
+    def to_class_name(s):
+        """
+        Converts a string to a valid class name.
+
+        >>> InferenceServiceABC.to_class_name("hello world")
+        'HelloWorld'
+        """
+
+        s = re.sub(r"[^a-zA-Z0-9 ]", "", s)
+        s = "".join(word.title() for word in s.split())
+        if s and s[0].isdigit():
+            s = "Class" + s
+        return s
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

edsl/inference_services/InferenceServicesCollection.py

@@ -0,0 +1,138 @@
+from functools import lru_cache
+from collections import defaultdict
+from typing import Optional, Protocol, Dict, List, Tuple, TYPE_CHECKING, Literal
+
+from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+from edsl.inference_services.AvailableModelFetcher import AvailableModelFetcher
+from edsl.exceptions.inference_services import InferenceServiceError
+
+if TYPE_CHECKING:
+    from edsl.language_models.LanguageModel import LanguageModel
+    from edsl.inference_services.InferenceServiceABC import InferenceServiceABC
+
+
+class ModelCreator(Protocol):
+    def create_model(self, model_name: str) -> "LanguageModel":
+        ...
+
+
+from edsl.enums import InferenceServiceLiteral
+
+
+class ModelResolver:
+    def __init__(
+        self,
+        services: List[InferenceServiceLiteral],
+        models_to_services: Dict[InferenceServiceLiteral, InferenceServiceABC],
+        availability_fetcher: "AvailableModelFetcher",
+    ):
+        """
+        Class for determining which service to use for a given model.
+        """
+        self.services = services
+        self._models_to_services = models_to_services
+        self.availability_fetcher = availability_fetcher
+        self._service_names_to_classes = {
+            service._inference_service_: service for service in services
+        }
+
+    def resolve_model(
+        self, model_name: str, service_name: Optional[InferenceServiceLiteral] = None
+    ) -> InferenceServiceABC:
+        """Returns an InferenceServiceABC object for the given model name.
+
+        :param model_name: The name of the model to resolve. E.g., 'gpt-4o'
+        :param service_name: The name of the service to use. E.g., 'openai'
+        :return: An InferenceServiceABC object
+
+        """
+        if model_name == "test":
+            from edsl.inference_services.TestService import TestService
+
+            return TestService()
+
+        if service_name is not None:
+            service: InferenceServiceABC = self._service_names_to_classes.get(
+                service_name
+            )
+            if not service:
+                raise InferenceServiceError(f"Service {service_name} not found")
+            return service
+
+        if model_name in self._models_to_services:  # maybe we've seen it before!
+            return self._models_to_services[model_name]
+
+        for service in self.services:
+            (
+                available_models,
+                service_name,
+            ) = self.availability_fetcher.get_available_models_by_service(service)
+            if model_name in available_models:
+                self._models_to_services[model_name] = service
+                return service
+
+        raise InferenceServiceError(
+            f"""Model {model_name} not found in any services. 
+                                    If you know the service that has this model, use the service_name parameter directly.
+                                    E.g., Model("gpt-4o", service_name="openai")
+                                    """
+        )
+
+
+class InferenceServicesCollection:
+    added_models = defaultdict(list)  # Moved back to class level
+
+    def __init__(self, services: Optional[List[InferenceServiceABC]] = None):
+        self.services = services or []
+        self._models_to_services: Dict[str, InferenceServiceABC] = {}
+
+        self.availability_fetcher = AvailableModelFetcher(
+            self.services, self.added_models
+        )
+        self.resolver = ModelResolver(
+            self.services, self._models_to_services, self.availability_fetcher
+        )
+
+    @classmethod
+    def add_model(cls, service_name: str, model_name: str) -> None:
+        if service_name not in cls.added_models:
+            cls.added_models[service_name].append(model_name)
+
+    def service_names_to_classes(self) -> Dict[str, InferenceServiceABC]:
+        return {service._inference_service_: service for service in self.services}
+
+    def available(
+        self,
+        service: Optional[str] = None,
+    ) -> List[Tuple[str, str, int]]:
+        return self.availability_fetcher.available(service)
+
+    def reset_cache(self) -> None:
+        self.availability_fetcher.reset_cache()
+
+    @property
+    def num_cache_entries(self) -> int:
+        return self.availability_fetcher.num_cache_entries
+
+    def register(self, service: InferenceServiceABC) -> None:
+        self.services.append(service)
+
+    def create_model_factory(
+        self, model_name: str, service_name: Optional[InferenceServiceLiteral] = None
+    ) -> "LanguageModel":
+
+        if service_name is None:  # we try to find the right service
+            service = self.resolver.resolve_model(model_name, service_name)
+        else:  # if they passed a service, we'll use that
+            service = self.service_names_to_classes().get(service_name)
+
+        if not service:  # but if we can't find it, we'll raise an error
+            raise InferenceServiceError(f"Service {service_name} not found")
+
+        return service.create_model(model_name)
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()