ag2 0.10.2__py3-none-any.whl

autogen/oai/together.py

@@ -0,0 +1,387 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# Portions derived from  https://github.com/microsoft/autogen are under the MIT License.
+# SPDX-License-Identifier: MIT
+"""Create an OpenAI-compatible client using Together.AI's API.
+
+Example:
+    ```python
+    llm_config = {
+        "config_list": [
+            {
+                "api_type": "together",
+                "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
+                "api_key": os.environ.get("TOGETHER_API_KEY"),
+            }
+        ]
+    }
+
+    agent = autogen.AssistantAgent("my_agent", llm_config=llm_config)
+    ```
+
+Install Together.AI python library using: pip install --upgrade together
+
+Resources:
+- https://docs.together.ai/docs/inference-python
+"""
+
+from __future__ import annotations
+
+import copy
+import os
+import time
+import warnings
+from typing import Any, Literal
+
+from pydantic import Field
+from typing_extensions import Unpack
+
+from ..import_utils import optional_import_block, require_optional_import
+from ..llm_config.entry import LLMConfigEntry, LLMConfigEntryDict
+from .client_utils import should_hide_tools, validate_parameter
+from .oai_models import ChatCompletion, ChatCompletionMessage, ChatCompletionMessageToolCall, Choice, CompletionUsage
+
+with optional_import_block():
+    from together import Together
+
+
+class TogetherEntryDict(LLMConfigEntryDict, total=False):
+    api_type: Literal["together"]
+
+    stream: bool
+    top_k: int | None
+    repetition_penalty: float | None
+    presence_penalty: float | None
+    frequency_penalty: float | None
+    min_p: float | None
+    safety_model: str | None
+    hide_tools: Literal["if_all_run", "if_any_run", "never"]
+    price: list[float] | None
+    tool_choice: str | dict[str, str | dict[str, str]] | None
+
+
+class TogetherLLMConfigEntry(LLMConfigEntry):
+    api_type: Literal["together"] = "together"
+
+    max_tokens: int = Field(default=512, ge=0)
+
+    stream: bool = False
+    top_k: int | None = Field(default=None)
+    repetition_penalty: float | None = Field(default=None)
+    presence_penalty: float | None = Field(default=None, ge=-2, le=2)
+    frequency_penalty: float | None = Field(default=None, ge=-2, le=2)
+    min_p: float | None = Field(default=None, ge=0, le=1)
+    safety_model: str | None = None
+    hide_tools: Literal["if_all_run", "if_any_run", "never"] = "never"
+    price: list[float] | None = Field(default=None, min_length=2, max_length=2)
+    tool_choice: str | dict[str, str | dict[str, str]] | None = (
+        None  # dict is the tool to call: {"type": "function", "function": {"name": "my_function"}}
+    )
+
+    def create_client(self):
+        raise NotImplementedError("TogetherLLMConfigEntry.create_client is not implemented.")
+
+
+class TogetherClient:
+    """Client for Together.AI's API."""
+
+    RESPONSE_USAGE_KEYS: list[str] = ["prompt_tokens", "completion_tokens", "total_tokens", "cost", "model"]
+
+    def __init__(self, **kwargs: Unpack[TogetherEntryDict]):
+        """Requires api_key or environment variable to be set
+
+        Args:
+            **kwargs: Additional keyword arguments to pass to the client.
+        """
+        # Ensure we have the api_key upon instantiation
+        self.api_key = kwargs.get("api_key")
+        if not self.api_key:
+            self.api_key = os.getenv("TOGETHER_API_KEY")
+
+        if "response_format" in kwargs and kwargs["response_format"] is not None:
+            warnings.warn("response_format is not supported for Together.AI, it will be ignored.", UserWarning)
+
+        assert self.api_key, (
+            "Please include the api_key in your config list entry for Together.AI or set the TOGETHER_API_KEY env variable."
+        )
+
+    def message_retrieval(self, response) -> list:
+        """Retrieve and return a list of strings or a list of Choice.Message from the response.
+
+        NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object,
+        since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used.
+        """
+        return [choice.message for choice in response.choices]
+
+    def cost(self, response) -> float:
+        return response.cost
+
+    @staticmethod
+    def get_usage(response) -> dict:
+        """Return usage summary of the response using RESPONSE_USAGE_KEYS."""
+        # ...  # pragma: no cover
+        return {
+            "prompt_tokens": response.usage.prompt_tokens,
+            "completion_tokens": response.usage.completion_tokens,
+            "total_tokens": response.usage.total_tokens,
+            "cost": response.cost,
+            "model": response.model,
+        }
+
+    def parse_params(self, params: dict[str, Any]) -> dict[str, Any]:
+        """Loads the parameters for Together.AI API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults"""
+        together_params = {}
+
+        # Check that we have what we need to use Together.AI's API
+        together_params["model"] = params.get("model")
+        assert together_params["model"], (
+            "Please specify the 'model' in your config list entry to nominate the Together.AI model to use."
+        )
+
+        # Validate allowed Together.AI parameters
+        # https://github.com/togethercomputer/together-python/blob/94ffb30daf0ac3e078be986af7228f85f79bde99/src/together/resources/completions.py#L44
+        together_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, 512, (0, None), None)
+        together_params["stream"] = validate_parameter(params, "stream", bool, False, False, None, None)
+        together_params["temperature"] = validate_parameter(params, "temperature", (int, float), True, None, None, None)
+        together_params["top_p"] = validate_parameter(params, "top_p", (int, float), True, None, None, None)
+        together_params["top_k"] = validate_parameter(params, "top_k", int, True, None, None, None)
+        together_params["repetition_penalty"] = validate_parameter(
+            params, "repetition_penalty", float, True, None, None, None
+        )
+        together_params["presence_penalty"] = validate_parameter(
+            params, "presence_penalty", (int, float), True, None, (-2, 2), None
+        )
+        together_params["frequency_penalty"] = validate_parameter(
+            params, "frequency_penalty", (int, float), True, None, (-2, 2), None
+        )
+        together_params["min_p"] = validate_parameter(params, "min_p", (int, float), True, None, (0, 1), None)
+        together_params["safety_model"] = validate_parameter(
+            params, "safety_model", str, True, None, None, None
+        )  # We won't enforce the available models as they are likely to change
+
+        # Check if they want to stream and use tools, which isn't currently supported (TODO)
+        if together_params["stream"] and "tools" in params:
+            warnings.warn(
+                "Streaming is not supported when using tools, streaming will be disabled.",
+                UserWarning,
+            )
+
+            together_params["stream"] = False
+
+        if "tool_choice" in params:
+            together_params["tool_choice"] = params["tool_choice"]
+
+        return together_params
+
+    @require_optional_import("together", "together")
+    def create(self, params: dict) -> ChatCompletion:
+        messages = params.get("messages", [])
+
+        # Convert AG2 messages to Together.AI messages
+        together_messages = oai_messages_to_together_messages(messages)
+
+        # Parse parameters to Together.AI API's parameters
+        together_params = self.parse_params(params)
+
+        # Add tools to the call if we have them and aren't hiding them
+        if "tools" in params:
+            hide_tools = validate_parameter(
+                params, "hide_tools", str, False, "never", None, ["if_all_run", "if_any_run", "never"]
+            )
+            if not should_hide_tools(together_messages, params["tools"], hide_tools):
+                together_params["tools"] = params["tools"]
+
+        together_params["messages"] = together_messages
+
+        # We use chat model by default
+        client = Together(api_key=self.api_key)
+
+        # Token counts will be returned
+        prompt_tokens = 0
+        completion_tokens = 0
+        total_tokens = 0
+
+        response = client.chat.completions.create(**together_params)
+        if together_params["stream"]:
+            # Read in the chunks as they stream
+            ans = ""
+            for chunk in response:
+                ans = ans + (chunk.choices[0].delta.content or "")
+
+            prompt_tokens = chunk.usage.prompt_tokens
+            completion_tokens = chunk.usage.completion_tokens
+            total_tokens = chunk.usage.total_tokens
+        else:
+            ans: str = response.choices[0].message.content
+
+            prompt_tokens = response.usage.prompt_tokens
+            completion_tokens = response.usage.completion_tokens
+            total_tokens = response.usage.total_tokens
+
+        if response.choices[0].finish_reason == "tool_calls":
+            together_finish = "tool_calls"
+            tool_calls = []
+            for tool_call in response.choices[0].message.tool_calls:
+                tool_calls.append(
+                    ChatCompletionMessageToolCall(
+                        id=tool_call.id,
+                        function={"name": tool_call.function.name, "arguments": tool_call.function.arguments},
+                        type="function",
+                    )
+                )
+        else:
+            together_finish = "stop"
+            tool_calls = None
+
+        # 3. convert output
+        message = ChatCompletionMessage(
+            role="assistant",
+            content=response.choices[0].message.content,
+            function_call=None,
+            tool_calls=tool_calls,
+        )
+        choices = [Choice(finish_reason=together_finish, index=0, message=message)]
+
+        response_oai = ChatCompletion(
+            id=response.id,
+            model=together_params["model"],
+            created=int(time.time()),
+            object="chat.completion",
+            choices=choices,
+            usage=CompletionUsage(
+                prompt_tokens=prompt_tokens,
+                completion_tokens=completion_tokens,
+                total_tokens=total_tokens,
+            ),
+            cost=calculate_together_cost(prompt_tokens, completion_tokens, together_params["model"]),
+        )
+
+        return response_oai
+
+
+def oai_messages_to_together_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Convert messages from OAI format to Together.AI format.
+    We correct for any specific role orders and types.
+    """
+    together_messages = copy.deepcopy(messages)
+
+    # If we have a message with role='tool', which occurs when a function is executed, change it to 'user'
+    for msg in together_messages:
+        if "role" in msg and msg["role"] == "tool":
+            msg["role"] = "user"
+
+    return together_messages
+
+
+# MODELS AND COSTS
+chat_lang_code_model_sizes = {
+    "zero-one-ai/Yi-34B-Chat": 34,
+    "allenai/OLMo-7B-Instruct": 7,
+    "allenai/OLMo-7B-Twin-2T": 7,
+    "allenai/OLMo-7B": 7,
+    "Austism/chronos-hermes-13b": 13,
+    "deepseek-ai/deepseek-coder-33b-instruct": 33,
+    "deepseek-ai/deepseek-llm-67b-chat": 67,
+    "garage-bAInd/Platypus2-70B-instruct": 70,
+    "google/gemma-2b-it": 2,
+    "google/gemma-7b-it": 7,
+    "Gryphe/MythoMax-L2-13b": 13,
+    "lmsys/vicuna-13b-v1.5": 13,
+    "lmsys/vicuna-7b-v1.5": 7,
+    "codellama/CodeLlama-13b-Instruct-hf": 13,
+    "codellama/CodeLlama-34b-Instruct-hf": 34,
+    "codellama/CodeLlama-70b-Instruct-hf": 70,
+    "codellama/CodeLlama-7b-Instruct-hf": 7,
+    "meta-llama/Llama-2-70b-chat-hf": 70,
+    "meta-llama/Llama-2-13b-chat-hf": 13,
+    "meta-llama/Llama-2-7b-chat-hf": 7,
+    "meta-llama/Llama-3-8b-chat-hf": 8,
+    "meta-llama/Llama-3-70b-chat-hf": 70,
+    "mistralai/Mistral-7B-Instruct-v0.1": 7,
+    "mistralai/Mistral-7B-Instruct-v0.2": 7,
+    "mistralai/Mistral-7B-Instruct-v0.3": 7,
+    "NousResearch/Nous-Capybara-7B-V1p9": 7,
+    "NousResearch/Nous-Hermes-llama-2-7b": 7,
+    "NousResearch/Nous-Hermes-Llama2-13b": 13,
+    "NousResearch/Nous-Hermes-2-Yi-34B": 34,
+    "openchat/openchat-3.5-1210": 7,
+    "Open-Orca/Mistral-7B-OpenOrca": 7,
+    "Qwen/Qwen1.5-0.5B-Chat": 0.5,
+    "Qwen/Qwen1.5-1.8B-Chat": 1.8,
+    "Qwen/Qwen1.5-4B-Chat": 4,
+    "Qwen/Qwen1.5-7B-Chat": 7,
+    "Qwen/Qwen1.5-14B-Chat": 14,
+    "Qwen/Qwen1.5-32B-Chat": 32,
+    "Qwen/Qwen1.5-72B-Chat": 72,
+    "Qwen/Qwen1.5-110B-Chat": 110,
+    "Qwen/Qwen2-72B-Instruct": 72,
+    "snorkelai/Snorkel-Mistral-PairRM-DPO": 7,
+    "togethercomputer/alpaca-7b": 7,
+    "teknium/OpenHermes-2-Mistral-7B": 7,
+    "teknium/OpenHermes-2p5-Mistral-7B": 7,
+    "togethercomputer/Llama-2-7B-32K-Instruct": 7,
+    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": 3,
+    "togethercomputer/RedPajama-INCITE-7B-Chat": 7,
+    "togethercomputer/StripedHyena-Nous-7B": 7,
+    "Undi95/ReMM-SLERP-L2-13B": 13,
+    "Undi95/Toppy-M-7B": 7,
+    "WizardLM/WizardLM-13B-V1.2": 13,
+    "upstage/SOLAR-10.7B-Instruct-v1.0": 11,
+}
+
+# Cost per million tokens based on up to X Billion parameters, e.g. up 4B is $0.1/million
+chat_lang_code_model_costs = {4: 0.1, 8: 0.2, 21: 0.3, 41: 0.8, 80: 0.9, 110: 1.8}
+
+mixture_model_sizes = {
+    "cognitivecomputations/dolphin-2.5-mixtral-8x7b": 56,
+    "databricks/dbrx-instruct": 132,
+    "mistralai/Mixtral-8x7B-Instruct-v0.1": 47,
+    "mistralai/Mixtral-8x22B-Instruct-v0.1": 141,
+    "NousResearch/Nous-Hermes-2-Mistral-7B-DPO": 7,
+    "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO": 47,
+    "NousResearch/Nous-Hermes-2-Mixtral-8x7B-SFT": 47,
+    "Snowflake/snowflake-arctic-instruct": 480,
+}
+
+# Cost per million tokens based on up to X Billion parameters, e.g. up 56B is $0.6/million
+mixture_costs = {56: 0.6, 176: 1.2, 480: 2.4}
+
+
+def calculate_together_cost(input_tokens: int, output_tokens: int, model_name: str) -> float:
+    """Cost calculation for inference"""
+    if model_name in chat_lang_code_model_sizes or model_name in mixture_model_sizes:
+        cost_per_mil = 0
+
+        # Chat, Language, Code models
+        if model_name in chat_lang_code_model_sizes:
+            size_in_b = chat_lang_code_model_sizes[model_name]
+
+            for top_size in chat_lang_code_model_costs:
+                if size_in_b <= top_size:
+                    cost_per_mil = chat_lang_code_model_costs[top_size]
+                    break
+
+        else:
+            # Mixture-of-experts
+            size_in_b = mixture_model_sizes[model_name]
+
+            for top_size in mixture_costs:
+                if size_in_b <= top_size:
+                    cost_per_mil = mixture_costs[top_size]
+                    break
+
+        if cost_per_mil == 0:
+            warnings.warn("Model size doesn't align with cost structure.", UserWarning)
+
+        return cost_per_mil * ((input_tokens + output_tokens) / 1e6)
+
+    else:
+        # Model is not in our list of models, can't determine the cost
+        warnings.warn(
+            "The model isn't catered for costing, to apply costs you can use the 'price' key on your config_list.",
+            UserWarning,
+        )
+
+        return 0

autogen/remote/__init__.py

@@ -0,0 +1,18 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+
+try:
+    import httpx  # noqa: F401
+except ImportError as e:
+    raise ImportError("httpx is not installed. Please install it with:\npip install httpx") from e
+
+from .agent import HTTPRemoteAgent
+from .httpx_client_factory import HttpxClientFactory
+from .runtime import HTTPAgentBus
+
+__all__ = (
+    "HTTPAgentBus",
+    "HTTPRemoteAgent",
+    "HttpxClientFactory",
+)

autogen/remote/agent.py

@@ -0,0 +1,199 @@
+# Copyright (c) 2023 - 2025, AG2ai, Inc., AG2ai open-source projects maintainers and core contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+from typing import Any
+
+import httpx
+
+from autogen import ConversableAgent
+from autogen.agentchat.group import ContextVariables
+from autogen.oai.client import OpenAIWrapper
+
+from .errors import RemoteAgentError, RemoteAgentNotFoundError
+from .httpx_client_factory import ClientFactory, EmptyClientFactory
+from .protocol import RequestMessage, ResponseMessage
+from .retry import NoRetryPolicy, RetryPolicy
+
+
+class HTTPRemoteAgent(ConversableAgent):
+    """A remote agent that communicates with other agents via HTTP long-polling.
+
+    This agent forwards messages to a remote endpoint and handles the response
+    through HTTP requests. It supports both synchronous and asynchronous operations.
+
+    Example:
+        >>> remote_agent = HTTPRemoteAgent(url="http://api.example.com/agents", name="my_remote_agent")
+    """
+
+    def __init__(
+        self,
+        url: str,
+        name: str,
+        *,
+        silent: bool = False,
+        client: ClientFactory | None = None,
+        retry_policy: RetryPolicy | None = None,
+    ) -> None:
+        """Initialize the HTTPRemoteAgent.
+
+        Args:
+            url (str): The base URL of the remote agent service.
+            name (str): The name of this agent.
+            silent (bool): If True, suppresses logging output.
+            client (ClientFactory | None): HTTP client factory. If None, uses EmptyClientFactory.
+            retry_policy (RetryPolicy | None): Retry policy for HTTP requests. If None, uses NoRetryPolicy.
+        """
+
+        self.url = url
+        self.retry_policy: RetryPolicy = retry_policy or NoRetryPolicy
+
+        self._httpx_client_factory = client or EmptyClientFactory()
+
+        super().__init__(name, silent=silent)
+
+        self.__llm_config: dict[str, Any] = {}
+
+        self.replace_reply_func(
+            ConversableAgent.generate_oai_reply,
+            HTTPRemoteAgent.generate_remote_reply,
+        )
+        self.replace_reply_func(
+            ConversableAgent.a_generate_oai_reply,
+            HTTPRemoteAgent.a_generate_remote_reply,
+        )
+
+    def generate_remote_reply(
+        self,
+        messages: list[dict[str, Any]] | None = None,
+        sender: ConversableAgent | None = None,
+        config: OpenAIWrapper | None = None,
+    ) -> tuple[bool, dict[str, Any] | None]:
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        retry_policy = self.retry_policy()
+
+        task_id: Any = None
+        with self._httpx_client_factory.make_sync() as client:
+            while True:
+                with retry_policy:
+                    if task_id is None:
+                        # initiate remote procedure
+                        task_id = self._process_create_remote_task_response(
+                            client.post(
+                                f"{self.url}/{self.name}",
+                                content=RequestMessage(
+                                    messages=messages,
+                                    context=self.context_variables.data,
+                                    client_tools=self.__llm_config.get("tools", []),
+                                ).model_dump_json(),
+                            )
+                        )
+
+                    reply_response = client.get(f"{self.url}/{self.name}/{task_id}")
+
+                    if reply_response.status_code in (200, 204):  # valid answer codes
+                        break
+
+                    if reply_response.status_code == 425:  # task still in progress
+                        continue
+
+                    if reply_response.status_code == 404:
+                        task_id = None  # recreate task due remote agent lost it
+                        continue
+
+                    raise RemoteAgentError(f"Remote client error: {reply_response}, {reply_response.content!r}")
+
+        if reply := self._process_remote_reply(reply_response):
+            if sender:
+                context_variables = ContextVariables(reply.context)
+                sender.context_variables.update(context_variables.to_dict())
+            # TODO: support multiple messages response for remote chat history
+            return True, reply.messages[-1]
+
+        return True, None
+
+    async def a_generate_remote_reply(
+        self,
+        messages: list[dict[str, Any]] | None = None,
+        sender: ConversableAgent | None = None,
+        config: OpenAIWrapper | None = None,
+    ) -> tuple[bool, dict[str, Any] | None]:
+        if messages is None:
+            messages = self._oai_messages[sender]
+
+        retry_policy = self.retry_policy()
+
+        task_id: Any = None
+        async with self._httpx_client_factory() as client:
+            while True:
+                with retry_policy:
+                    if task_id is None:
+                        # initiate remote procedure
+                        task_id = self._process_create_remote_task_response(
+                            await client.post(
+                                f"{self.url}/{self.name}",
+                                content=RequestMessage(
+                                    messages=messages,
+                                    context=self.context_variables.data,
+                                    client_tools=self.__llm_config.get("tools", []),
+                                ).model_dump_json(),
+                            )
+                        )
+
+                    reply_response = await client.get(f"{self.url}/{self.name}/{task_id}")
+
+                    if reply_response.status_code in (200, 204):  # valid answer codes
+                        break
+
+                    if reply_response.status_code == 425:  # task still in progress
+                        continue
+
+                    if reply_response.status_code == 404:
+                        task_id = None  # recreate task due remote agent lost it
+                        continue
+
+                    raise RemoteAgentError(f"Remote client error: {reply_response}, {reply_response.content!r}")
+
+        if reply := self._process_remote_reply(reply_response):
+            if sender:
+                context_variables = ContextVariables(reply.context)
+                sender.context_variables.update(context_variables.to_dict())
+            # TODO: support multiple messages response for remote chat history
+            return True, reply.messages[-1]
+
+        return True, None
+
+    def _process_create_remote_task_response(self, response: httpx.Response) -> Any:
+        if response.status_code == 404:
+            raise RemoteAgentNotFoundError(self.name)
+
+        if response.status_code != 202:
+            raise RemoteAgentError(f"Remote client error: {response}, {response.content!r}")
+
+        return response.json()
+
+    def _process_remote_reply(self, reply_response: httpx.Response) -> ResponseMessage | None:
+        if reply_response.status_code == 204:
+            return None
+
+        try:
+            serialized_message = ResponseMessage.model_validate_json(reply_response.content)
+
+        except Exception as e:
+            raise RemoteAgentError(f"Remote client error: {reply_response}, {reply_response.content!r}") from e
+
+        return serialized_message
+
+    def update_tool_signature(
+        self,
+        tool_sig: str | dict[str, Any],
+        is_remove: bool,
+        silent_override: bool = False,
+    ) -> None:
+        self.__llm_config = self._update_tool_config(
+            self.__llm_config,
+            tool_sig=tool_sig,
+            is_remove=is_remove,
+            silent_override=silent_override,
+        )