lionagi 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

lionagi/integrations/groq_/api_endpoints/groq_request.py

@@ -0,0 +1,288 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import re
+from collections.abc import AsyncGenerator
+from os import getenv
+from typing import Any
+
+import aiohttp
+from pydantic import BaseModel, Field, field_validator
+
+from .data_models import (
+    GroqEndpointPathParam,
+    GroqEndpointQueryParam,
+    GroqEndpointRequestBody,
+)
+from .response_utils import match_response
+
+GROQ_BASE_URL = "https://api.groq.com/openai/v1"
+DEFAULT_TIMEOUT = 30  # seconds
+
+
+class GroqRequest(BaseModel):
+    api_key: str = Field(
+        description="API key for authentication", exclude=True
+    )
+    endpoint: str = Field(description="API endpoint")
+    method: str = Field(description="HTTP method")
+    content_type: str | None = Field(
+        default="application/json", description="Content type for the request"
+    )
+    timeout: float = Field(
+        default=DEFAULT_TIMEOUT,
+        description="Request timeout in seconds",
+        ge=0,
+    )
+
+    model_config = {
+        "arbitrary_types_allowed": True,
+        "extra": "allow",  # Allow extra attributes for mocking
+    }
+
+    @field_validator("api_key")
+    @classmethod
+    def validate_api_key(cls, v: str) -> str:
+        """Validate API key format."""
+        # First try to get from env if v looks like an env var
+        if re.match(r"^[A-Z_][A-Z0-9_]*$", v):
+            env_value = getenv(v)
+            if env_value:
+                v = env_value
+
+        # Validate key format
+        if not v.startswith(("groq_", "gsk_")):
+            raise ValueError("Invalid Groq API key format")
+        return v
+
+    def get_headers(self) -> dict[str, str]:
+        """Get headers for the request."""
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+        }
+        if self.content_type:
+            headers["Content-Type"] = self.content_type
+        return headers
+
+    def get_endpoint(
+        self, path_param: GroqEndpointPathParam | None = None
+    ) -> str:
+        """Get the formatted endpoint URL."""
+        endpoint = self.endpoint
+        if path_param:
+            try:
+                endpoint = endpoint.format(**path_param.model_dump())
+            except KeyError as e:
+                raise ValueError(f"Missing path parameter: {e}")
+        return f"{GROQ_BASE_URL}/{endpoint}"
+
+    async def invoke(
+        self,
+        request_body: GroqEndpointRequestBody | None = None,
+        params: GroqEndpointQueryParam | None = None,
+        form_data: dict[str, Any] | None = None,
+        path_param: GroqEndpointPathParam | None = None,
+        output_file: str | None = None,
+        with_response_header: bool = False,
+        parse_response: bool = True,
+        **kwargs,
+    ) -> Any:
+        """Make a request to the Groq API."""
+        url = self.get_endpoint(path_param)
+        headers = self.get_headers()
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+
+        # Convert Pydantic models to dict
+        json_data = (
+            request_body.model_dump(exclude_unset=True)
+            if request_body
+            else None
+        )
+        params = params.model_dump(exclude_unset=True) if params else None
+
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                if form_data:
+                    data = aiohttp.FormData()
+                    for key, value in form_data.items():
+                        data.add_field(key, value)
+                    async with session.request(
+                        self.method,
+                        url,
+                        headers=headers,
+                        data=data,
+                        params=params,
+                    ) as response:
+                        return await self._handle_response(
+                            response,
+                            output_file,
+                            with_response_header,
+                            parse_response,
+                        )
+                else:
+                    async with session.request(
+                        self.method,
+                        url,
+                        headers=headers,
+                        json=json_data,
+                        params=params,
+                    ) as response:
+                        return await self._handle_response(
+                            response,
+                            output_file,
+                            with_response_header,
+                            parse_response,
+                        )
+        except aiohttp.ClientError as e:
+            raise ConnectionError(f"Failed to connect to Groq API: {str(e)}")
+        except TimeoutError:
+            raise TimeoutError(
+                f"Request timed out after {self.timeout} seconds"
+            )
+
+    async def stream(
+        self,
+        request_body: GroqEndpointRequestBody | None = None,
+        output_file: str | None = None,
+        with_response_header: bool = False,
+    ) -> AsyncGenerator[dict[str, Any], None]:
+        """Stream response from the Groq API."""
+        if request_body and not getattr(request_body, "stream", False):
+            raise ValueError(
+                "Request does not support stream. "
+                "Only requests with stream=True are supported"
+            )
+
+        url = self.get_endpoint()
+        headers = self.get_headers()
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+
+        # Convert Pydantic model to dict
+        json_data = (
+            request_body.model_dump(exclude_unset=True)
+            if request_body
+            else None
+        )
+
+        try:
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.request(
+                    self.method, url, headers=headers, json=json_data
+                ) as response:
+                    if response.status != 200:
+                        error_text = await self._get_error_text(response)
+                        raise aiohttp.ClientResponseError(
+                            request_info=response.request_info,
+                            history=response.history,
+                            status=response.status,
+                            message=error_text,
+                            headers=response.headers,
+                        )
+
+                    file_handle = None
+                    if output_file:
+                        try:
+                            file_handle = open(output_file, "w")
+                        except Exception as e:
+                            raise ValueError(
+                                f"Failed to open output file {output_file}: {str(e)}"
+                            )
+
+                    try:
+                        async for line in response.content:
+                            if line:
+                                try:
+                                    line = line.decode("utf-8").strip()
+                                except UnicodeDecodeError:
+                                    import warnings
+
+                                    warnings.warn(
+                                        "Failed to decode response chunk"
+                                    )
+                                    continue
+
+                                if line.startswith("data: "):
+                                    line = line[6:]  # Remove "data: " prefix
+
+                                if line == "[DONE]":
+                                    break
+
+                                try:
+                                    chunk_data = json.loads(line)
+                                    if file_handle:
+                                        file_handle.write(
+                                            json.dumps(chunk_data) + "\n"
+                                        )
+                                    yield chunk_data
+                                except json.JSONDecodeError:
+                                    import warnings
+
+                                    warnings.warn(
+                                        f"Failed to parse response chunk: {line}"
+                                    )
+                                    continue
+
+                        if with_response_header:
+                            yield dict(response.headers)
+                    finally:
+                        if file_handle:
+                            file_handle.close()
+
+        except aiohttp.ClientError as e:
+            raise ConnectionError(f"Failed to connect to Groq API: {str(e)}")
+        except TimeoutError:
+            raise TimeoutError(
+                f"Stream timed out after {self.timeout} seconds"
+            )
+
+    async def _handle_response(
+        self,
+        response: aiohttp.ClientResponse,
+        output_file: str | None = None,
+        with_response_header: bool = False,
+        parse_response: bool = True,
+    ) -> Any:
+        """Handle the API response."""
+        if response.status != 200:
+            error_text = await self._get_error_text(response)
+            raise aiohttp.ClientResponseError(
+                request_info=response.request_info,
+                history=response.history,
+                status=response.status,
+                message=error_text,
+                headers=response.headers,
+            )
+
+        if output_file:
+            with open(output_file, "wb") as f:
+                async for chunk in response.content.iter_chunked(1024):
+                    f.write(chunk)
+            return None
+
+        response_body = await response.json()
+
+        if parse_response:
+            response_body = match_response(self, response_body)
+
+        if with_response_header:
+            return response_body, dict(response.headers)
+        return response_body
+
+    async def _get_error_text(self, response: aiohttp.ClientResponse) -> str:
+        """Extract error text from response."""
+        try:
+            error_json = await response.json()
+            return json.dumps(error_json)
+        except Exception:
+            try:
+                return await response.text()
+            except Exception:
+                return f"HTTP {response.status}"
+
+    def __repr__(self):
+        return (
+            f"GroqRequest(endpoint={self.endpoint}, method={self.method}, "
+            f"content_type={self.content_type})"
+        )

lionagi/integrations/groq_/api_endpoints/match_response.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from .data_models import GroqAudioResponse, GroqChatCompletionResponse
+from .groq_request import GroqRequest
+
+
+def match_response(
+    request_model: GroqRequest,
+    response: dict[str, Any] | list[dict[str, Any]] | None,
+) -> dict[str, Any] | list[dict[str, Any]] | None:
+    """Match response to appropriate model and format."""
+    if response is None:
+        return None
+
+    endpoint = request_model.endpoint.split("/")[0]
+
+    # Chat completions endpoint
+    if endpoint == "chat":
+        if isinstance(response, dict):
+            # Single response
+            try:
+                # Validate and parse through Pydantic model
+                parsed = GroqChatCompletionResponse(**response)
+                # Return in consistent format
+                return {
+                    "choices": parsed.choices,
+                    "model": parsed.model,
+                    "usage": parsed.usage,
+                }
+            except Exception as e:
+                import warnings
+
+                warnings.warn(f"Failed to parse chat response: {str(e)}")
+                return response
+        else:
+            # Stream response list
+            result = []
+            for chunk in response:
+                if not isinstance(chunk, dict):
+                    continue
+
+                try:
+                    if "choices" in chunk:
+                        # Regular chunk with content
+                        result.append(
+                            {
+                                "choices": [
+                                    {
+                                        "delta": {
+                                            "content": choice.get(
+                                                "delta", {}
+                                            ).get("content", ""),
+                                            "role": "assistant",
+                                        }
+                                    }
+                                    for choice in chunk["choices"]
+                                ]
+                            }
+                        )
+
+                        # Add usage if present
+                        if "usage" in chunk:
+                            result[-1]["usage"] = chunk["usage"]
+
+                    elif "usage" in chunk:
+                        # Final chunk with usage stats
+                        result.append(
+                            {
+                                "choices": [
+                                    {
+                                        "delta": {
+                                            "content": "",
+                                            "role": "assistant",
+                                        },
+                                        "finish_reason": "stop",
+                                    }
+                                ],
+                                "usage": chunk["usage"],
+                            }
+                        )
+                except Exception as e:
+                    import warnings
+
+                    warnings.warn(f"Failed to parse stream chunk: {str(e)}")
+                    continue
+
+            return result
+
+    # Audio endpoints
+    elif endpoint == "audio":
+        try:
+            if isinstance(response, dict):
+                parsed = GroqAudioResponse(**response)
+                return {"text": parsed.text, "metadata": parsed.x_groq}
+        except Exception as e:
+            import warnings
+
+            warnings.warn(f"Failed to parse audio response: {str(e)}")
+            return response
+
+    # Default case
+    return response

lionagi/integrations/groq_/api_endpoints/response_utils.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from .data_models import GroqAudioResponse, GroqChatCompletionResponse
+
+
+def match_response(
+    request_model: Any,
+    response: dict[str, Any] | list[dict[str, Any]] | None,
+) -> dict[str, Any] | list[dict[str, Any]] | None:
+    """Match response to appropriate model and format."""
+    if response is None:
+        return None
+
+    endpoint = request_model.endpoint.split("/")[0]
+
+    # Chat completions endpoint
+    if endpoint == "chat":
+        if isinstance(response, dict):
+            # Single response
+            try:
+                # Validate and parse through Pydantic model
+                parsed = GroqChatCompletionResponse(**response)
+                # Return in consistent format
+                return {
+                    "choices": parsed.choices,
+                    "model": parsed.model,
+                    "usage": parsed.usage,
+                }
+            except Exception as e:
+                import warnings
+
+                warnings.warn(f"Failed to parse chat response: {str(e)}")
+                return response
+        else:
+            # Stream response list
+            result = []
+            for chunk in response:
+                if not isinstance(chunk, dict):
+                    continue
+
+                try:
+                    if "choices" in chunk:
+                        # Regular chunk with content
+                        result.append(
+                            {
+                                "choices": [
+                                    {
+                                        "delta": {
+                                            "content": choice.get(
+                                                "delta", {}
+                                            ).get("content", ""),
+                                            "role": "assistant",
+                                        }
+                                    }
+                                    for choice in chunk["choices"]
+                                ]
+                            }
+                        )
+
+                        # Add usage if present
+                        if "usage" in chunk:
+                            result[-1]["usage"] = chunk["usage"]
+
+                    elif "usage" in chunk:
+                        # Final chunk with usage stats
+                        result.append(
+                            {
+                                "choices": [
+                                    {
+                                        "delta": {
+                                            "content": "",
+                                            "role": "assistant",
+                                        },
+                                        "finish_reason": "stop",
+                                    }
+                                ],
+                                "usage": chunk["usage"],
+                            }
+                        )
+                except Exception as e:
+                    import warnings
+
+                    warnings.warn(f"Failed to parse stream chunk: {str(e)}")
+                    continue
+
+            return result
+
+    # Audio endpoints
+    elif endpoint == "audio":
+        try:
+            if isinstance(response, dict):
+                parsed = GroqAudioResponse(**response)
+                return {"text": parsed.text, "metadata": parsed.x_groq}
+        except Exception as e:
+            import warnings
+
+            warnings.warn(f"Failed to parse audio response: {str(e)}")
+            return response
+
+    # Default case
+    return response

lionagi/integrations/groq_/groq_max_output_token_data.yaml

@@ -0,0 +1,21 @@
+# Text Models
+gemma2-9b-it: 8192
+gemma-7b-it: 8192
+llama3-groq-70b-8192-tool-use-preview: 8192
+llama3-groq-8b-8192-tool-use-preview: 8192
+llama-3.1-70b-versatile: 32768
+llama-3.1-70b-specdec: 8192
+llama-3.1-8b-instant: 8192
+llama-3.2-1b-preview: 8192
+llama-3.2-3b-preview: 8192
+llama-3.2-11b-vision-preview: 8192
+llama-3.2-90b-vision-preview: 8192
+llama-guard-3-8b: 8192
+llama3-70b-8192: 8192
+llama3-8b-8192: 8192
+mixtral-8x7b-32768: 32768
+
+# Audio Models (25MB max file size)
+whisper-large-v3: 448
+whisper-large-v3-turbo: 448
+distil-whisper-large-v3-en: 448

lionagi/integrations/groq_/groq_price_data.yaml

@@ -0,0 +1,58 @@
+model:
+  # Text Models
+  gemma2-9b-it:
+    input_tokens: 0
+    output_tokens: 0
+  gemma-7b-it:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-groq-70b-8192-tool-use-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-groq-8b-8192-tool-use-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-70b-versatile:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-70b-specdec:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.1-8b-instant:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-1b-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-3b-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-11b-vision-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-3.2-90b-vision-preview:
+    input_tokens: 0
+    output_tokens: 0
+  llama-guard-3-8b:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-70b-8192:
+    input_tokens: 0
+    output_tokens: 0
+  llama3-8b-8192:
+    input_tokens: 0
+    output_tokens: 0
+  mixtral-8x7b-32768:
+    input_tokens: 0
+    output_tokens: 0
+
+  # Audio Models
+  whisper-large-v3:
+    input_tokens: 0
+    output_tokens: 0
+  whisper-large-v3-turbo:
+    input_tokens: 0
+    output_tokens: 0
+  distil-whisper-large-v3-en:
+    input_tokens: 0
+    output_tokens: 0

lionagi/integrations/groq_/groq_rate_limits.yaml

@@ -0,0 +1,105 @@
+# Text Models Rate Limits
+text_models:
+  gemma-7b-it:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  gemma2-9b-it:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama-3.1-70b-versatile:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 6000
+    tokens_per_day: 200000
+  llama-3.1-8b-instant:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 20000
+    tokens_per_day: 500000
+  llama-3.2-11b-text-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-11b-vision-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-1b-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-3b-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-90b-text-preview:
+    requests_per_minute: 30
+    requests_per_day: 7000
+    tokens_per_minute: 7000
+    tokens_per_day: 500000
+  llama-3.2-90b-vision-preview:
+    requests_per_minute: 15
+    requests_per_day: 3500
+    tokens_per_minute: 7000
+    tokens_per_day: 250000
+  llama-guard-3-8b:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama3-70b-8192:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 6000
+    tokens_per_day: 500000
+  llama3-8b-8192:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 30000
+    tokens_per_day: 500000
+  llama3-groq-70b-8192-tool-use-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llama3-groq-8b-8192-tool-use-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 15000
+    tokens_per_day: 500000
+  llava-v1.5-7b-4096-preview:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 30000
+    tokens_per_day: null  # No limit
+  mixtral-8x7b-32768:
+    requests_per_minute: 30
+    requests_per_day: 14400
+    tokens_per_minute: 5000
+    tokens_per_day: 500000
+
+# Audio Models Rate Limits
+audio_models:
+  distil-whisper-large-v3-en:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800
+  whisper-large-v3:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800
+  whisper-large-v3-turbo:
+    requests_per_minute: 20
+    requests_per_day: 2000
+    audio_seconds_per_hour: 7200
+    audio_seconds_per_day: 28800

lionagi/integrations/groq_/version.py

@@ -0,0 +1,5 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+__version__ = "1.0.0"

lionagi/integrations/litellm_/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) 2023 - 2024, HaiyangLi <quantocean.li at gmail dot com>
+#
+# SPDX-License-Identifier: Apache-2.0