lionagi 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

lionagi/integrations/groq_/groq_max_output_token_data.yaml

@@ -0,0 +1,21 @@
+# Text Models
+gemma2-9b-it: 8192
+gemma-7b-it: 8192
+llama3-groq-70b-8192-tool-use-preview: 8192
+llama3-groq-8b-8192-tool-use-preview: 8192
+llama-3.1-70b-versatile: 32768
+llama-3.1-70b-specdec: 8192
+llama-3.1-8b-instant: 8192
+llama-3.2-1b-preview: 8192
+llama-3.2-3b-preview: 8192
+llama-3.2-11b-vision-preview: 8192
+llama-3.2-90b-vision-preview: 8192
+llama-guard-3-8b: 8192
+llama3-70b-8192: 8192
+llama3-8b-8192: 8192
+mixtral-8x7b-32768: 32768
+
+# Audio Models (25MB max file size)
+whisper-large-v3: 448
+whisper-large-v3-turbo: 448
+distil-whisper-large-v3-en: 448

lionagi/integrations/groq_/groq_price_data.yaml

@@ -0,0 +1,58 @@
+model:
+  # Text Models
+  gemma2-9b-it:
+    input_tokens: 0
+    output_tokens: 0
+  gemma-7b-it:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-groq-70b-8192-tool-use-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-groq-8b-8192-tool-use-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-70b-versatile:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-70b-specdec:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-8b-instant:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-1b-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-3b-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-11b-vision-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-90b-vision-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-guard-3-8b:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-70b-8192:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-8b-8192:
+    input_tokens: 0
+    output_tokens: 0
+  mixtral-8x7b-32768:
+    input_tokens: 0
+    output_tokens: 0
+
+  # Audio Models
+  whisper-large-v3:
+    input_tokens: 0
+    output_tokens: 0
+  whisper-large-v3-turbo:
+    input_tokens: 0
+    output_tokens: 0
+  distil-whisper-large-v3-en:
+    input_tokens: 0
+    output_tokens: 0

lionagi/integrations/groq_/groq_rate_limits.yaml

@@ -0,0 +1,105 @@
+# Text Models Rate Limits
+text_models:
+  gemma-7b-it:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  gemma2-9b-it:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama-3.1-70b-versatile:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 6000
+    tokens_per_day: 200000
+  llama-3.1-8b-instant:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 20000
+    tokens_per_day: 500000
+  llama-3.2-11b-text-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-11b-vision-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-1b-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-3b-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-90b-text-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-90b-vision-preview:
+    requests_per_minute: 15
+    requests_per_day: 3500
+    tokens_per_minute: 7000
+    tokens_per_day: 250000
+  llama-guard-3-8b:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama3-70b-8192:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 6000
+    tokens_per_day: 500000
+  llama3-8b-8192:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 30000
+    tokens_per_day: 500000
+  llama3-groq-70b-8192-tool-use-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama3-groq-8b-8192-tool-use-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llava-v1.5-7b-4096-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 30000
+    tokens_per_day: null  # No limit
+  mixtral-8x7b-32768:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 5000
+    tokens_per_day: 500000
+
+# Audio Models Rate Limits
+audio_models:
+  distil-whisper-large-v3-en:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800
+  whisper-large-v3:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800
+  whisper-large-v3-turbo:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800

lionagi/integrations/groq_/version.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+__version__ = "1.0.0"

lionagi/integrations/litellm_/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0

lionagi/integrations/litellm_/imodel.py

@@ -0,0 +1,73 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import os
+
+import litellm
+from dotenv import load_dotenv
+
+litellm.drop_params = True
+load_dotenv()
+
+
+RESERVED_PARAMS = [
+    "invoke_action",
+    "instruction",
+    "clear_messages",
+]
+
+
+class LiteiModel:
+
+    def __init__(self, **kwargs):
+        if "api_key" in kwargs:
+            try:
+                api_key1 = os.getenv(kwargs["api_key"], None)
+                if api_key1:
+                    # Store the original env var name as schema
+                    self.api_key_schema = kwargs["api_key"]
+                    # Store the resolved value for actual use
+                    kwargs["api_key"] = api_key1
+            except Exception:
+                pass
+        self.kwargs = kwargs
+        self.acompletion = litellm.acompletion
+
+    def to_dict(self) -> dict:
+        dict_ = {
+            k: v for k, v in self.kwargs.items() if k not in RESERVED_PARAMS
+        }
+        # If we have an api_key_schema, use that instead of the resolved value
+        if hasattr(self, "api_key_schema"):
+            dict_["api_key"] = self.api_key_schema
+        return dict_
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "LiteiModel":
+        return cls(**data)
+
+    async def invoke(self, **kwargs):
+        config = {**self.kwargs, **kwargs}
+        for i in RESERVED_PARAMS:
+            config.pop(i, None)
+
+        return await self.acompletion(**config)
+
+    def __hash__(self):
+        # Convert kwargs to a hashable format by serializing unhashable types
+        hashable_items = []
+        for k, v in self.kwargs.items():
+            if isinstance(v, (list, dict)):
+                # Convert unhashable types to JSON string for hashing
+                v = json.dumps(v, sort_keys=True)
+            elif not isinstance(v, (str, int, float, bool, type(None))):
+                # Convert other unhashable types to string representation
+                v = str(v)
+            hashable_items.append((k, v))
+        return hash(frozenset(hashable_items))
+
+    @property
+    def allowed_roles(self):
+        return ["user", "assistant", "system"]

lionagi/integrations/ollama_/OllamaModel.py

@@ -0,0 +1,244 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import traceback
+
+from dotenv import load_dotenv
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    SerializeAsAny,
+    field_serializer,
+    model_validator,
+)
+
+from lionagi.service.rate_limiter import RateLimiter
+from lionagi.service.service_util import invoke_retry
+from lionagi.service.token_calculator import TokenCalculator
+
+load_dotenv()
+
+from lionagi.integrations.ollama_.api_endpoints.api_request import (
+    OllamaRequest,
+)
+from lionagi.integrations.ollama_.api_endpoints.chat_completion.request_body import (
+    OllamaChatCompletionRequestBody,
+)
+from lionagi.integrations.ollama_.api_endpoints.completion.request_body import (
+    OllamaCompletionRequestBody,
+)
+from lionagi.integrations.ollama_.api_endpoints.data_models import (
+    OllamaEndpointRequestBody,
+)
+from lionagi.integrations.ollama_.api_endpoints.embedding.request_body import (
+    OllamaEmbeddingRequestBody,
+)
+from lionagi.integrations.ollama_.api_endpoints.match_response import (
+    match_response,
+)
+
+
+class OllamaModel(BaseModel):
+    model: str = Field(description="ID of the model to use.")
+
+    request_model: OllamaRequest = Field(description="Making requests")
+
+    rate_limiter: RateLimiter = Field(
+        description="Rate Limiter to track usage"
+    )
+
+    text_token_calculator: SerializeAsAny[TokenCalculator] = Field(
+        default=None, description="Token Calculator"
+    )
+
+    # image_token_calculator: TokenCalculator = Field(default=None, description="Image Token Calculator")
+
+    estimated_output_len: int = Field(
+        default=0, description="Expected output len before making request"
+    )
+
+    model_config = ConfigDict(extra="forbid")
+
+    @model_validator(mode="before")
+    @classmethod
+    def parse_input(cls, data: dict):
+        if not isinstance(data, dict):
+            raise ValueError("Invalid init param")
+
+        # parse request model
+        request_model_params = {
+            "endpoint": data.pop("endpoint", None),
+            "method": data.pop("method", None),
+        }
+
+        data["request_model"] = OllamaRequest(**request_model_params)
+
+        # parse rate limiter
+        rate_limiter_params = {}
+        if limit_tokens := data.pop("limit_tokens", None):
+            rate_limiter_params["limit_tokens"] = limit_tokens
+        if limit_requests := data.pop("limit_requests", None):
+            rate_limiter_params["limit_requests"] = limit_requests
+
+        data["rate_limiter"] = RateLimiter(**rate_limiter_params)
+
+        if not data.get("text_token_calculator"):
+            try:
+                from lionagi.service.token_calculator import TiktokenCalculator
+
+                text_calc = TiktokenCalculator(encoding_name=data.get("model"))
+                data["text_token_calculator"] = text_calc
+            except:
+                pass
+
+        return data
+
+    @field_serializer("request_model")
+    def serialize_request_model(self, value: OllamaRequest):
+        return value.model_dump(exclude_unset=True)
+
+    @invoke_retry(max_retries=3, base_delay=1, max_delay=60)
+    async def invoke(
+        self,
+        request_body: OllamaEndpointRequestBody,
+        estimated_output_len: int = 0,
+        output_file=None,
+        parse_response=True,
+    ):
+        if request_model := getattr(request_body, "model"):
+            if request_model != self.model:
+                raise ValueError(
+                    f"Request model does not match. Model is {self.model}, but request is made for {request_model}."
+                )
+
+        if (
+            isinstance(request_body, OllamaCompletionRequestBody)
+            and not getattr(request_body, "prompt")
+        ) or (
+            isinstance(request_body, OllamaChatCompletionRequestBody)
+            and not getattr(request_body, "messages")
+        ):
+            response_body, response_headers = await self.request_model.invoke(
+                json_data=request_body,
+                output_file=output_file,
+                with_response_header=True,
+                parse_response=False,
+            )
+
+            if parse_response:
+                return match_response(self.request_model, response_body)
+            else:
+                return response_body
+
+        # check remaining rate limit
+        input_token_len = self.get_input_token_len(request_body)
+
+        invoke_viability_result = self.verify_invoke_viability(
+            input_tokens_len=input_token_len,
+            estimated_output_len=estimated_output_len,
+        )
+        if not invoke_viability_result:
+            raise ValueError("Rate limit reached for requests")
+
+        try:
+            if getattr(request_body, "stream", None):
+                return await self.stream(request_body)
+
+            response_body, response_headers = await self.request_model.invoke(
+                json_data=request_body,
+                output_file=output_file,
+                with_response_header=True,
+                parse_response=False,
+            )
+
+            total_token_usage = response_body.get(
+                "prompt_eval_count", 0
+            ) + response_body.get("eval_count", 0)
+            self.rate_limiter.update_rate_limit(
+                response_headers.get("Date"), total_token_usage
+            )
+
+            if parse_response:
+                return match_response(self.request_model, response_body)
+            else:
+                return response_body
+
+        except Exception as e:  # TODO: example
+            return Exception(f"Error: {e}. {traceback.format_exc()}")
+
+    async def stream(
+        self,
+        request_body: OllamaEndpointRequestBody,
+        output_file=None,
+        parse_response=True,
+    ):
+        response_list = []
+        async for chunk in self.request_model.stream(
+            json_data=request_body,
+            output_file=output_file,
+            with_response_header=True,
+        ):
+            response_list.append(chunk)
+
+        response_headers = response_list.pop()
+        total_token_usage = response_list[-1].get(
+            "prompt_eval_count", 0
+        ) + response_list[-1].get("eval_count", 0)
+        self.rate_limiter.update_rate_limit(
+            response_headers.get("Date"), total_token_usage
+        )
+
+        if parse_response:
+            return match_response(self.request_model, response_list)
+        else:
+            return response_list
+
+    def get_input_token_len(self, request_body: OllamaEndpointRequestBody):
+        if request_model := getattr(request_body, "model"):
+            if request_model != self.model:
+                raise ValueError(
+                    f"Request model does not match. Model is {self.model}, but request is made for {request_model}."
+                )
+
+        if isinstance(request_body, OllamaCompletionRequestBody):
+            messages_text = getattr(request_body, "prompt", "")
+            text_tokens = self.text_token_calculator.calculate(messages_text)
+            return text_tokens
+        elif isinstance(request_body, OllamaChatCompletionRequestBody):
+            messages_text = request_body.model_dump_json(
+                include={"messages": {"__all__": {"role", "content"}}},
+                exclude_unset=True,
+            )
+            text_tokens = self.text_token_calculator.calculate(messages_text)
+            return text_tokens
+        elif isinstance(request_body, OllamaEmbeddingRequestBody):
+            messages_text = getattr(request_body, "input")
+            messages_text = (
+                " ".join(messages_text)
+                if isinstance(messages_text, list)
+                else messages_text
+            )
+            text_tokens = self.text_token_calculator.calculate(messages_text)
+            return text_tokens
+        else:
+            return 0
+
+    def verify_invoke_viability(
+        self, input_tokens_len: int = 0, estimated_output_len: int = 0
+    ):
+        self.rate_limiter.release_tokens()
+
+        estimated_output_len = (
+            estimated_output_len
+            if estimated_output_len != 0
+            else self.estimated_output_len
+        )
+
+        if self.rate_limiter.check_availability(
+            input_tokens_len, estimated_output_len
+        ):
+            return True
+        else:
+            return False

lionagi/integrations/ollama_/OllamaService.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import inspect
+
+from lionagi.service import Service, register_service
+
+from .api_endpoints.api_request import OllamaRequest
+from .api_endpoints.match_data_model import match_data_model
+from .OllamaModel import OllamaModel
+
+
+@register_service
+class OllamaService(Service):
+    def __init__(
+        self,
+        name: str = None,
+    ):
+        self.name = name
+        self.rate_limiters = {}  # model: RateLimiter
+
+    def check_rate_limiter(
+        self,
+        ollama_model: OllamaModel,
+        limit_requests: int = None,
+        limit_tokens: int = None,
+    ):
+        model = ollama_model.model
+
+        if model not in self.rate_limiters:
+            self.rate_limiters[model] = ollama_model.rate_limiter
+        else:
+            ollama_model.rate_limiter = self.rate_limiters[model]
+            if limit_requests:
+                ollama_model.rate_limiter.limit_requests = limit_requests
+            if limit_tokens:
+                ollama_model.rate_limiter.limit_tokens = limit_tokens
+
+        return ollama_model
+
+    @staticmethod
+    def match_data_model(task_name):
+        return match_data_model(task_name)
+
+    @classmethod
+    def list_tasks(cls):
+        methods = []
+        for name, member in inspect.getmembers(
+            cls, predicate=inspect.isfunction
+        ):
+            if name not in [
+                "__init__",
+                "check_rate_limiter",
+                "match_data_model",
+            ]:
+                methods.append(name)
+        return methods
+
+    # Generate a completion
+    def generate_completion(
+        self, model: str, limit_tokens: int = None, limit_requests: int = None
+    ):
+        model_obj = OllamaModel(
+            model=model,
+            endpoint="generate",
+            method="POST",
+            limit_tokens=limit_tokens,
+            limit_requests=limit_requests,
+        )
+
+        return self.check_rate_limiter(
+            model_obj, limit_requests=limit_requests, limit_tokens=limit_tokens
+        )
+
+    # Generate a chat completion
+    def generate_chat_completion(
+        self, model: str, limit_tokens: int = None, limit_requests: int = None
+    ):
+        model_obj = OllamaModel(
+            model=model,
+            endpoint="chat",
+            method="POST",
+            limit_tokens=limit_tokens,
+            limit_requests=limit_requests,
+        )
+
+        return self.check_rate_limiter(
+            model_obj, limit_requests=limit_requests, limit_tokens=limit_tokens
+        )
+
+    # Generate Embeddings
+    def generate_embeddings(
+        self, model: str, limit_tokens: int = None, limit_requests: int = None
+    ):
+        model_obj = OllamaModel(
+            model=model,
+            endpoint="embed",
+            method="POST",
+            limit_tokens=limit_tokens,
+            limit_requests=limit_requests,
+        )
+
+        return self.check_rate_limiter(
+            model_obj, limit_requests=limit_requests, limit_tokens=limit_tokens
+        )
+
+    # Create a Model
+    def create_model(self):
+        return OllamaRequest(endpoint="create", method="POST")
+
+    # List Local Models
+    def list_local_models(self):
+        return OllamaRequest(endpoint="tags", method="GET")
+
+    # Show Model Information
+    def show_model_information(self):
+        return OllamaRequest(endpoint="show", method="POST")
+
+    # Copy a Model
+    def copy_model(self):
+        return OllamaRequest(endpoint="copy", method="POST")
+
+    # Delete a Model
+    def delete_model(self):
+        return OllamaRequest(endpoint="delete", method="DELETE")
+
+    # Pull a Model
+    def pull_model(self):
+        return OllamaRequest(endpoint="pull", method="POST")
+
+    # Push a Model
+    def push_model(self):
+        return OllamaRequest(endpoint="push", method="POST")
+
+    # List Running Models
+    def list_running_models(self):
+        return OllamaRequest(endpoint="ps", method="GET")
+
+    @property
+    def allowed_roles(self):
+        return ["user", "assistant", "system"]

lionagi/integrations/ollama_/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0

lionagi/integrations/ollama_/api_endpoints/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0