gllm-inference-binary 0.5.55__cp313-cp313-macosx_13_0_arm64.whl

gllm_inference/lm_invoker/batch/batch_operations.pyi

@@ -0,0 +1,127 @@
+from gllm_inference.exceptions import InvokerRuntimeError as InvokerRuntimeError
+from gllm_inference.schema import BatchStatus as BatchStatus, LMInput as LMInput, LMOutput as LMOutput
+from typing import Any
+
+DEFAULT_STATUS_CHECK_INTERVAL: float
+
+class BatchOperations:
+    """Handles batch operations for an LM invoker.
+
+    This class provides a wrapper around the batch operations of an LM invoker.
+    It provides a simple interface to perform batch invocation:
+    ```python
+    results = await lm_invoker.batch.invoke(...)
+    ```
+
+    Additionally, it also supports the following standalone batch operations:
+
+    1. Create a batch job:
+    ```python
+    batch_id = await lm_invoker.batch.create(...)
+    ```
+
+    2. Get the status of a batch job:
+    ```python
+    status = await lm_invoker.batch.status(batch_id)
+    ```
+
+    3. Retrieve the results of a batch job:
+    ```python
+    results = await lm_invoker.batch.retrieve(batch_id)
+    ```
+
+    4. List the batch jobs:
+    ```python
+    batch_jobs = await lm_invoker.batch.list()
+    ```
+
+    5. Cancel a batch job:
+    ```python
+    await lm_invoker.batch.cancel(batch_id)
+    ```
+    """
+    def __init__(self, invoker: BaseLMInvoker) -> None:
+        """Initializes the batch operations.
+
+        Args:
+            invoker (BaseLMInvoker): The LM invoker to use for the batch operations.
+        """
+    async def invoke(self, requests: dict[str, LMInput], hyperparameters: dict[str, Any] | None = None, status_check_interval: float = ..., max_iterations: int | None = None) -> dict[str, LMOutput]:
+        """Invokes the language model in batch mode.
+
+        This method orchestrates the entire batch invocation process, including;
+        1. Creating a batch job.
+        2. Iteratively checking the status of the batch job until it is finished.
+        3. Retrieving the results of the batch job.
+        The method includes retry logic with exponential backoff for transient failures.
+
+        Args:
+            requests (dict[str, LMInput]): The dictionary of requests that maps request ID to the request.
+                Each request must be a valid input for the language model.
+                1. If the request is a list of Message objects, it is used as is.
+                2. If the request is a list of MessageContent or a string, it is converted into a user message.
+            hyperparameters (dict[str, Any] | None, optional): A dictionary of hyperparameters for the language model.
+                Defaults to None, in which case the default hyperparameters are used.
+            status_check_interval (float, optional): The interval in seconds to check the status of the batch job.
+                Defaults to DEFAULT_STATUS_CHECK_INTERVAL.
+            max_iterations (int | None, optional): The maximum number of iterations to check the status of the batch
+                job. Defaults to None, in which case the number of iterations is infinite.
+
+        Returns:
+            dict[str, LMOutput]: The results of the batch job.
+
+        Raises:
+            CancelledError: If the invocation is cancelled.
+            ModelNotFoundError: If the model is not found.
+            ProviderAuthError: If the model authentication fails.
+            ProviderInternalError: If the model internal error occurs.
+            ProviderInvalidArgsError: If the model parameters are invalid.
+            ProviderOverloadedError: If the model is overloaded.
+            ProviderRateLimitError: If the model rate limit is exceeded.
+            TimeoutError: If the invocation times out.
+            ValueError: If the messages are not in the correct format.
+        """
+    async def create(self, requests: dict[str, LMInput], hyperparameters: dict[str, Any] | None = None) -> str:
+        """Creates a new batch job.
+
+        Args:
+            requests (dict[str, LMInput]): The dictionary of requests that maps request ID to the request.
+                Each request must be a valid input for the language model.
+                1. If the request is a list of Message objects, it is used as is.
+                2. If the request is a list of MessageContent or a string, it is converted into a user message.
+            hyperparameters (dict[str, Any] | None, optional): A dictionary of hyperparameters for the language model.
+                Defaults to None, in which case the default hyperparameters are used.
+
+        Returns:
+            str: The ID of the batch job.
+        """
+    async def status(self, batch_id: str) -> BatchStatus:
+        """Gets the status of a batch job.
+
+        Args:
+            batch_id (str): The ID of the batch job to get the status of.
+
+        Returns:
+            BatchStatus: The status of the batch job.
+        """
+    async def retrieve(self, batch_id: str) -> dict[str, LMOutput]:
+        """Retrieves the results of a batch job.
+
+        Args:
+            batch_id (str): The ID of the batch job to get the results of.
+
+        Returns:
+            dict[str, LMOutput]: The results of the batch job.
+        """
+    async def list(self) -> list[dict[str, Any]]:
+        """Lists the batch jobs.
+
+        Returns:
+            list[dict[str, Any]]: The list of batch jobs.
+        """
+    async def cancel(self, batch_id: str) -> None:
+        """Cancels a batch job.
+
+        Args:
+            batch_id (str): The ID of the batch job to cancel.
+        """

gllm_inference/lm_invoker/bedrock_lm_invoker.pyi

@@ -0,0 +1,212 @@
+from _typeshed import Incomplete
+from gllm_core.event import EventEmitter as EventEmitter
+from gllm_core.schema.tool import Tool as Tool
+from gllm_core.utils.retry import RetryConfig as RetryConfig
+from gllm_inference.exceptions import BaseInvokerError as BaseInvokerError, convert_http_status_to_base_invoker_error as convert_http_status_to_base_invoker_error
+from gllm_inference.exceptions.provider_error_map import BEDROCK_ERROR_MAPPING as BEDROCK_ERROR_MAPPING
+from gllm_inference.lm_invoker.lm_invoker import BaseLMInvoker as BaseLMInvoker
+from gllm_inference.lm_invoker.schema.bedrock import InputType as InputType, Key as Key, OutputType as OutputType
+from gllm_inference.schema import Attachment as Attachment, AttachmentType as AttachmentType, LMOutput as LMOutput, Message as Message, ModelId as ModelId, ModelProvider as ModelProvider, ResponseSchema as ResponseSchema, TokenUsage as TokenUsage, ToolCall as ToolCall, ToolResult as ToolResult
+from langchain_core.tools import Tool as LangChainTool
+from typing import Any
+
+FILENAME_SANITIZATION_REGEX: Incomplete
+SUPPORTED_ATTACHMENTS: Incomplete
+
+class BedrockLMInvoker(BaseLMInvoker):
+    '''A language model invoker to interact with AWS Bedrock language models.
+
+    Attributes:
+        model_id (str): The model ID of the language model.
+        model_provider (str): The provider of the language model.
+        model_name (str): The name of the language model.
+        session (Session): The Bedrock client session.
+        client_kwargs (dict[str, Any]): The Bedrock client kwargs.
+        default_hyperparameters (dict[str, Any]): Default hyperparameters for invoking the model.
+        tools (list[Tool]): Tools provided to the model to enable tool calling.
+        response_schema (ResponseSchema | None): The schema of the response. If provided, the model will output a
+            structured response as defined by the schema. Supports both Pydantic BaseModel and JSON schema dictionary.
+        output_analytics (bool): Whether to output the invocation analytics.
+        retry_config (RetryConfig): The retry configuration for the language model.
+
+    Basic usage:
+        The `BedrockLMInvoker` can be used as follows:
+        ```python
+        lm_invoker = BedrockLMInvoker(
+            model_name="us.anthropic.claude-sonnet-4-20250514-v1:0",
+            aws_access_key_id="<your-aws-access-key-id>",
+            aws_secret_access_key="<your-aws-secret-access-key>",
+        )
+        result = await lm_invoker.invoke("Hi there!")
+        ```
+
+    Input types:
+        The `BedrockLMInvoker` supports the following input types: text, document, image, and video.
+        Non-text inputs can be passed as an `Attachment` object with the `user` role.
+
+        Usage example:
+        ```python
+        text = "What animal is in this image?"
+        image = Attachment.from_path("path/to/local/image.png")
+        result = await lm_invoker.invoke([text, image])
+        ```
+
+    Text output:
+        The `BedrockLMInvoker` generates text outputs by default.
+        Text outputs are stored in the `outputs` attribute of the `LMOutput` object and can be accessed
+        via the `texts` (all text outputs) or `text` (first text output) properties.
+
+        Output example:
+        ```python
+        LMOutput(outputs=[LMOutputItem(type="text", output="Hello, there!")])
+        ```
+
+    Structured output:
+        The `BedrockLMInvoker` can be configured to generate structured outputs.
+        This feature can be enabled by providing a schema to the `response_schema` parameter.
+
+        Structured outputs are stored in the `outputs` attribute of the `LMOutput` object and can be accessed
+        via the `structureds` (all structured outputs) or `structured` (first structured output) properties.
+
+        The schema must either be one of the following:
+        1. A Pydantic BaseModel class
+            The structured output will be a Pydantic model.
+        2. A JSON schema dictionary
+            JSON dictionary schema must be compatible with Pydantic\'s JSON schema, especially for complex schemas.
+            Thus, it is recommended to create the JSON schema using Pydantic\'s `model_json_schema` method.
+            The structured output will be a dictionary.
+
+        Usage example:
+        ```python
+        class Animal(BaseModel):
+            name: str
+            color: str
+
+        json_schema = Animal.model_json_schema()
+
+        lm_invoker = BedrockLMInvoker(..., response_schema=Animal)  # Using Pydantic BaseModel class
+        lm_invoker = BedrockLMInvoker(..., response_schema=json_schema)  # Using JSON schema dictionary
+        ```
+
+        Output example:
+        ```python
+        # Using Pydantic BaseModel class outputs a Pydantic model
+        LMOutput(outputs=[LMOutputItem(type="structured", output=Animal(name="dog", color="white"))])
+
+        # Using JSON schema dictionary outputs a dictionary
+        LMOutput(outputs=[LMOutputItem(type="structured", output={"name": "dog", "color": "white"})])
+        ```
+
+        Structured output is not compatible with tool calling.
+        When structured output is enabled, streaming is disabled.
+
+    Tool calling:
+        The `BedrockLMInvoker` can be configured to call tools to perform certain tasks.
+        This feature can be enabled by providing a list of `Tool` objects to the `tools` parameter.
+
+        Tool calls outputs are stored in the `outputs` attribute of the `LMOutput` object and
+        can be accessed via the `tool_calls` property.
+
+        Usage example:
+        ```python
+        lm_invoker = BedrockLMInvoker(..., tools=[tool_1, tool_2])
+        ```
+
+        Output example:
+        ```python
+        LMOutput(
+            outputs=[
+                LMOutputItem(type="text", output="I\'m using tools..."),
+                LMOutputItem(type="tool_call", output=ToolCall(id="123", name="tool_1", args={"key": "value"})),
+                LMOutputItem(type="tool_call", output=ToolCall(id="456", name="tool_2", args={"key": "value"})),
+            ]
+        )
+        ```
+
+    Analytics tracking:
+        The `BedrockLMInvoker` can be configured to output additional information about the invocation.
+        This feature can be enabled by setting the `output_analytics` parameter to `True`.
+
+        When enabled, the following attributes will be stored in the output:
+        1. `token_usage`: The token usage.
+        2. `duration`: The duration in seconds.
+        3. `finish_details`: The details about how the generation finished.
+
+        Output example:
+        ```python
+        LMOutput(
+            outputs=[...],
+            token_usage=TokenUsage(input_tokens=100, output_tokens=50),
+            duration=0.729,
+            finish_details={"stop_reason": "end_turn"},
+        )
+        ```
+
+    Retry and timeout:
+        The `BedrockLMInvoker` supports retry and timeout configuration.
+        By default, the max retries is set to 0 and the timeout is set to 30.0 seconds.
+        They can be customized by providing a custom `RetryConfig` object to the `retry_config` parameter.
+
+        Retry config examples:
+        ```python
+        retry_config = RetryConfig(max_retries=0, timeout=None)  # No retry, no timeout
+        retry_config = RetryConfig(max_retries=5, timeout=10.0)  # 5 max retries, 10.0 seconds timeout
+        ```
+
+        Usage example:
+        ```python
+        lm_invoker = BedrockLMInvoker(..., retry_config=retry_config)
+        ```
+    '''
+    session: Incomplete
+    client_kwargs: Incomplete
+    def __init__(self, model_name: str, access_key_id: str | None = None, secret_access_key: str | None = None, region_name: str = 'us-east-1', model_kwargs: dict[str, Any] | None = None, default_hyperparameters: dict[str, Any] | None = None, tools: list[Tool | LangChainTool] | None = None, response_schema: ResponseSchema | None = None, output_analytics: bool = False, retry_config: RetryConfig | None = None) -> None:
+        '''Initializes the BedrockLMInvoker instance.
+
+        Args:
+            model_name (str): The name of the Bedrock language model.
+            access_key_id (str | None, optional): The AWS access key ID. Defaults to None, in which case
+                the `AWS_ACCESS_KEY_ID` environment variable will be used.
+            secret_access_key (str | None, optional): The AWS secret access key. Defaults to None, in which case
+                the `AWS_SECRET_ACCESS_KEY` environment variable will be used.
+            region_name (str, optional): The AWS region name. Defaults to "us-east-1".
+            model_kwargs (dict[str, Any] | None, optional): Additional keyword arguments for the Bedrock client.
+            default_hyperparameters (dict[str, Any] | None, optional): Default hyperparameters for invoking the model.
+                Defaults to None.
+            tools (list[Tool | LangChainTool] | None, optional): Tools provided to the model to enable tool calling.
+                Defaults to None.
+            response_schema (ResponseSchema | None, optional): The schema of the response. If provided, the model will
+                output a structured response as defined by the schema. Supports both Pydantic BaseModel and JSON schema
+                dictionary. Defaults to None.
+            output_analytics (bool, optional): Whether to output the invocation analytics. Defaults to False.
+            retry_config (RetryConfig | None, optional): The retry configuration for the language model.
+                Defaults to None, in which case a default config with no retry and 30.0 seconds timeout will be used.
+
+        Raises:
+            ValueError: If `response_schema` is provided, but `tools` are also provided.
+            ValueError: If `access_key_id` or `secret_access_key` is neither provided nor set in the
+                `AWS_ACCESS_KEY_ID` or `AWS_SECRET_ACCESS_KEY` environment variables, respectively.
+        '''
+    def set_tools(self, tools: list[Tool | LangChainTool]) -> None:
+        """Sets the tools for the Bedrock language model.
+
+        This method sets the tools for the Bedrock language model. Any existing tools will be replaced.
+
+        Args:
+            tools (list[Tool | LangChainTool]): The list of tools to be used.
+
+        Raises:
+            ValueError: If `response_schema` exists.
+        """
+    def set_response_schema(self, response_schema: ResponseSchema | None) -> None:
+        """Sets the response schema for the Bedrock language model.
+
+        This method sets the response schema for the Bedrock language model. Any existing response schema will be
+        replaced.
+
+        Args:
+            response_schema (ResponseSchema | None): The response schema to be used.
+
+        Raises:
+            ValueError: If `tools` exists.
+        """

gllm_inference/lm_invoker/datasaur_lm_invoker.pyi

@@ -0,0 +1,157 @@
+from _typeshed import Incomplete
+from gllm_core.event import EventEmitter as EventEmitter
+from gllm_core.schema.tool import Tool as Tool
+from gllm_core.utils.retry import RetryConfig as RetryConfig
+from gllm_inference.constants import DOCUMENT_MIME_TYPES as DOCUMENT_MIME_TYPES, INVOKER_PROPAGATED_MAX_RETRIES as INVOKER_PROPAGATED_MAX_RETRIES
+from gllm_inference.lm_invoker.openai_chat_completions_lm_invoker import OpenAIChatCompletionsLMInvoker as OpenAIChatCompletionsLMInvoker
+from gllm_inference.lm_invoker.schema.datasaur import InputType as InputType, Key as Key
+from gllm_inference.schema import Attachment as Attachment, AttachmentType as AttachmentType, LMOutput as LMOutput, Message as Message, ModelId as ModelId, ModelProvider as ModelProvider, ResponseSchema as ResponseSchema, ToolCall as ToolCall, ToolResult as ToolResult
+from langchain_core.tools import Tool as LangChainTool
+from typing import Any
+
+SUPPORTED_ATTACHMENTS: Incomplete
+
+class DatasaurLMInvoker(OpenAIChatCompletionsLMInvoker):
+    '''A language model invoker to interact with Datasaur LLM Projects Deployment API.
+
+    Attributes:
+        model_id (str): The model ID of the language model.
+        model_provider (str): The provider of the language model.
+        model_name (str): The name of the language model.
+        client_kwargs (dict[str, Any]): The keyword arguments for the OpenAI client.
+        default_hyperparameters (dict[str, Any]): Default hyperparameters for invoking the model.
+        tools (list[Any]): The list of tools provided to the model to enable tool calling. Currently not supported.
+        response_schema (ResponseSchema | None): The schema of the response. Currently not supported.
+        output_analytics (bool): Whether to output the invocation analytics.
+        retry_config (RetryConfig | None): The retry configuration for the language model.
+        citations (bool): Whether to output the citations.
+
+    Basic usage:
+        The `DatasaurLMInvoker` can be used as follows:
+        ```python
+        lm_invoker = DatasaurLMInvoker(base_url="https://deployment.datasaur.ai/api/deployment/teamId/deploymentId/")
+        result = await lm_invoker.invoke("Hi there!")
+        ```
+
+    Input types:
+        The `DatasaurLMInvoker` supports the following input types: text, audio, image, and document.
+        Non-text inputs can be passed as an `Attachment` object with the `user` role.
+
+        Usage example:
+        ```python
+        text = "What animal is in this image?"
+        image = Attachment.from_path("path/to/local/image.png")
+        result = await lm_invoker.invoke([text, image])
+        ```
+
+    Text output:
+        The `DatasaurLMInvoker` generates text outputs by default.
+        Text outputs are stored in the `outputs` attribute of the `LMOutput` object and can be accessed
+        via the `texts` (all text outputs) or `text` (first text output) properties.
+
+        Output example:
+        ```python
+        LMOutput(outputs=[LMOutputItem(type="text", output="Hello, there!")])
+        ```
+
+    Citations:
+        The `DatasaurLMInvoker` can be configured to output the citations used to generate the response.
+        This feature can be enabled by setting the `citations` parameter to `True`.
+
+        Citations outputs are stored in the `outputs` attribute of the `LMOutput` object and
+        can be accessed via the `citations` property.
+
+        Usage example:
+        ```python
+        lm_invoker = DatasaurLMInvoker(..., citations=True)
+        ```
+
+        Output example:
+        ```python
+        LMOutput(
+            outputs=[
+                LMOutputItem(type="citation", output=Chunk(id="123", content="...", metadata={...}, score=0.95)),
+                LMOutputItem(type="text", output="According to recent reports... ([Source](https://www.example.com))."),
+            ],
+        )
+        ```
+
+    Analytics tracking:
+        The `DatasaurLMInvoker` can be configured to output additional information about the invocation.
+        This feature can be enabled by setting the `output_analytics` parameter to `True`.
+
+        When enabled, the following attributes will be stored in the output:
+        1. `token_usage`: The token usage.
+        2. `duration`: The duration in seconds.
+        3. `finish_details`: The details about how the generation finished.
+
+        Output example:
+        ```python
+        LMOutput(
+            outputs=[...],
+            token_usage=TokenUsage(input_tokens=100, output_tokens=50),
+            duration=0.729,
+            finish_details={"stop_reason": "end_turn"},
+        )
+        ```
+
+    Retry and timeout:
+        The `DatasaurLMInvoker` supports retry and timeout configuration.
+        By default, the max retries is set to 0 and the timeout is set to 30.0 seconds.
+        They can be customized by providing a custom `RetryConfig` object to the `retry_config` parameter.
+
+        Retry config examples:
+        ```python
+        retry_config = RetryConfig(max_retries=0, timeout=None)  # No retry, no timeout
+        retry_config = RetryConfig(max_retries=5, timeout=10.0)  # 5 max retries, 10.0 seconds timeout
+        ```
+
+        Usage example:
+        ```python
+        lm_invoker = DatasaurLMInvoker(..., retry_config=retry_config)
+        ```
+    '''
+    client_kwargs: Incomplete
+    citations: Incomplete
+    def __init__(self, base_url: str, api_key: str | None = None, model_kwargs: dict[str, Any] | None = None, default_hyperparameters: dict[str, Any] | None = None, output_analytics: bool = False, retry_config: RetryConfig | None = None, citations: bool = False) -> None:
+        """Initializes a new instance of the DatasaurLMInvoker class.
+
+        Args:
+            base_url (str): The base URL of the Datasaur LLM Projects Deployment API.
+            api_key (str | None, optional): The API key for authenticating with Datasaur LLM Projects Deployment API.
+                Defaults to None, in which case the `DATASAUR_API_KEY` environment variable will be used.
+            model_kwargs (dict[str, Any] | None, optional): Additional model parameters. Defaults to None.
+            default_hyperparameters (dict[str, Any] | None, optional): Default hyperparameters for invoking the model.
+                Defaults to None.
+            output_analytics (bool, optional): Whether to output the invocation analytics. Defaults to False.
+            retry_config (RetryConfig | None, optional): The retry configuration for the language model.
+                Defaults to None, in which case a default config with no retry and 30.0 seconds timeout will be used.
+            citations (bool, optional): Whether to output the citations. Defaults to False.
+
+        Raises:
+            ValueError: If the `api_key` is not provided and the `DATASAUR_API_KEY` environment variable is not set.
+        """
+    def set_tools(self, tools: list[Tool | LangChainTool]) -> None:
+        """Sets the tools for the Datasaur LLM Projects Deployment API.
+
+        This method is raises a `NotImplementedError` because the Datasaur LLM Projects Deployment API does not
+        support tools.
+
+        Args:
+            tools (list[Tool | LangChainTool]): The list of tools to be used.
+
+        Raises:
+            NotImplementedError: This method is not supported for the Datasaur LLM Projects Deployment API.
+        """
+    def set_response_schema(self, response_schema: ResponseSchema | None) -> None:
+        """Sets the response schema for the Datasaur LLM Projects Deployment API.
+
+        This method is raises a `NotImplementedError` because the Datasaur LLM Projects Deployment API does not
+        support response schema.
+
+        Args:
+            response_schema (ResponseSchema | None): The response schema to be used.
+
+        Raises:
+            NotImplementedError: This method is not supported for the Datasaur LLM Projects Deployment API.
+        """