vision-agents-plugins-xai 0.0.17__tar.gz

vision_agents_plugins_xai-0.0.17/.gitignore

@@ -0,0 +1,32 @@
+*/__pycache__
+*/chat/__pycache__
+*/video/__pycache__
+*/chat/sync/__pycache__
+*/chat/async_/__pycache__
+*/sync/__pycache__
+*/async_/__pycache__
+*/video/sync/__pycache__
+*/model/__pycache__/
+*/cli/__pycache__
+*/cli/__pycache__
+.env
+.venv
+.vscode/settings.json
+*.pyc
+dist/*
+dist/*
+*.log
+.python-version
+pyvenv.cfg
+.idea*
+bin/*
+lib/*
+shell.nix
+pyrightconfig.json
+.DS_Store
+
+*.egg-info/
+*.egg
+*.pt
+*.kef
+.env.bak

vision_agents_plugins_xai-0.0.17/PKG-INFO

@@ -0,0 +1,166 @@
+Metadata-Version: 2.4
+Name: vision-agents-plugins-xai
+Version: 0.0.17
+Summary: XAI for stream agents
+Project-URL: Documentation, https://visionagents.ai/
+Project-URL: Website, https://visionagents.ai/
+Project-URL: Source, https://github.com/GetStream/Vision-Agents
+License-Expression: Apache-2.0
+Requires-Python: >=3.10.0
+Requires-Dist: xai-sdk
+Description-Content-Type: text/markdown
+
+# XAI Plugin for Stream Agents
+
+This package provides xAI (Grok) integration for the Stream Agents ecosystem, enabling you to use xAI's powerful language models in your conversational AI applications.
+
+## Features
+
+- **Native xAI SDK Integration**: Full access to xAI's chat completion and streaming APIs
+- **Conversation Memory**: Automatic conversation history management
+- **Streaming Support**: Real-time response streaming with standardized events
+- **Multimodal Support**: Handle text and image inputs
+- **Event System**: Subscribe to response events for custom handling
+- **Easy Integration**: Drop-in replacement for other LLM providers
+
+## Installation
+
+```bash
+pip install vision-agents-plugins-xai
+```
+
+## Quick Start
+
+```python
+import asyncio
+from vision_agents.plugins import xai
+
+async def main():
+    # Initialize with your xAI API key
+    llm = xai.LLM(
+        model="grok-4",
+        api_key="your_xai_api_key"  # or set XAI_API_KEY environment variable
+    )
+    
+    # Simple response
+    response = await llm.simple_response("Explain quantum computing in simple terms")
+    
+    print(f"\n\nComplete response: {response.text}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Advanced Usage
+
+### Conversation with Memory
+
+```python
+from vision_agents.plugins import xai
+
+llm = xai.LLM(model="grok-4", api_key="your_api_key")
+
+# First message
+await llm.simple_response("My name is Alice and I have 2 cats")
+
+# Second message - the LLM remembers the context
+response = await llm.simple_response("How many pets do I have?")
+print(response.text)  # Will mention the 2 cats
+```
+
+### Using Instructions
+
+```python
+llm = LLM(
+    model="grok-4", 
+    api_key="your_api_key"
+)
+
+# Create a response with system instructions
+response = await llm.create_response(
+    input="Tell me about the weather",
+    instructions="You are a helpful weather assistant. Always be cheerful and optimistic.",
+    stream=True
+)
+```
+
+### Multimodal Input
+
+```python
+# Handle complex multimodal messages
+advanced_message = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "input_text", "text": "What do you see in this image?"},
+            {"type": "input_image", "image_url": "https://example.com/image.jpg"},
+        ],
+    }
+]
+
+messages = LLM._normalize_message(advanced_message)
+# Use with your conversation system
+```
+
+
+## API Reference
+
+### XAILLM Class
+
+#### Constructor
+
+```python
+LLM(
+    model: str = "grok-4",
+    api_key: Optional[str] = None,
+    client: Optional[AsyncClient] = None
+)
+```
+
+**Parameters:**
+- `model`: xAI model to use (default: "grok-4")
+- `api_key`: Your xAI API key (default: reads from `XAI_API_KEY` environment variable)
+- `client`: Optional pre-configured xAI AsyncClient
+
+#### Methods
+
+##### `async simple_response(text: str, processors=None, participant=None)`
+
+Generate a simple response to text input.
+
+**Parameters:**
+- `text`: Input text to respond to
+- `processors`: Optional list of processors for video/voice AI context
+- `participant`: Optional participant object
+
+**Returns:** `LLMResponseEvent[Response]` with the generated text
+
+##### `async create_response(input: str, instructions: str = "", model: str = None, stream: bool = True)`
+
+Create a response with full control over parameters.
+
+**Parameters:**
+- `input`: Input text
+- `instructions`: System instructions for the model
+- `model`: Override the default model
+- `stream`: Whether to stream the response (default: True)
+
+**Returns:** `LLMResponseEvent[Response]` with the generated text
+
+
+## Configuration
+
+### Environment Variables
+
+- `XAI_API_KEY`: Your xAI API key (required if not provided in constructor)
+
+
+## Requirements
+
+- Python 3.10+
+- `xai-sdk`
+- `vision-agents-core`
+
+## License
+
+Apache-2.0

vision_agents_plugins_xai-0.0.17/README.md

@@ -0,0 +1,154 @@
+# XAI Plugin for Stream Agents
+
+This package provides xAI (Grok) integration for the Stream Agents ecosystem, enabling you to use xAI's powerful language models in your conversational AI applications.
+
+## Features
+
+- **Native xAI SDK Integration**: Full access to xAI's chat completion and streaming APIs
+- **Conversation Memory**: Automatic conversation history management
+- **Streaming Support**: Real-time response streaming with standardized events
+- **Multimodal Support**: Handle text and image inputs
+- **Event System**: Subscribe to response events for custom handling
+- **Easy Integration**: Drop-in replacement for other LLM providers
+
+## Installation
+
+```bash
+pip install vision-agents-plugins-xai
+```
+
+## Quick Start
+
+```python
+import asyncio
+from vision_agents.plugins import xai
+
+async def main():
+    # Initialize with your xAI API key
+    llm = xai.LLM(
+        model="grok-4",
+        api_key="your_xai_api_key"  # or set XAI_API_KEY environment variable
+    )
+    
+    # Simple response
+    response = await llm.simple_response("Explain quantum computing in simple terms")
+    
+    print(f"\n\nComplete response: {response.text}")
+
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+## Advanced Usage
+
+### Conversation with Memory
+
+```python
+from vision_agents.plugins import xai
+
+llm = xai.LLM(model="grok-4", api_key="your_api_key")
+
+# First message
+await llm.simple_response("My name is Alice and I have 2 cats")
+
+# Second message - the LLM remembers the context
+response = await llm.simple_response("How many pets do I have?")
+print(response.text)  # Will mention the 2 cats
+```
+
+### Using Instructions
+
+```python
+llm = LLM(
+    model="grok-4", 
+    api_key="your_api_key"
+)
+
+# Create a response with system instructions
+response = await llm.create_response(
+    input="Tell me about the weather",
+    instructions="You are a helpful weather assistant. Always be cheerful and optimistic.",
+    stream=True
+)
+```
+
+### Multimodal Input
+
+```python
+# Handle complex multimodal messages
+advanced_message = [
+    {
+        "role": "user",
+        "content": [
+            {"type": "input_text", "text": "What do you see in this image?"},
+            {"type": "input_image", "image_url": "https://example.com/image.jpg"},
+        ],
+    }
+]
+
+messages = LLM._normalize_message(advanced_message)
+# Use with your conversation system
+```
+
+
+## API Reference
+
+### XAILLM Class
+
+#### Constructor
+
+```python
+LLM(
+    model: str = "grok-4",
+    api_key: Optional[str] = None,
+    client: Optional[AsyncClient] = None
+)
+```
+
+**Parameters:**
+- `model`: xAI model to use (default: "grok-4")
+- `api_key`: Your xAI API key (default: reads from `XAI_API_KEY` environment variable)
+- `client`: Optional pre-configured xAI AsyncClient
+
+#### Methods
+
+##### `async simple_response(text: str, processors=None, participant=None)`
+
+Generate a simple response to text input.
+
+**Parameters:**
+- `text`: Input text to respond to
+- `processors`: Optional list of processors for video/voice AI context
+- `participant`: Optional participant object
+
+**Returns:** `LLMResponseEvent[Response]` with the generated text
+
+##### `async create_response(input: str, instructions: str = "", model: str = None, stream: bool = True)`
+
+Create a response with full control over parameters.
+
+**Parameters:**
+- `input`: Input text
+- `instructions`: System instructions for the model
+- `model`: Override the default model
+- `stream`: Whether to stream the response (default: True)
+
+**Returns:** `LLMResponseEvent[Response]` with the generated text
+
+
+## Configuration
+
+### Environment Variables
+
+- `XAI_API_KEY`: Your xAI API key (required if not provided in constructor)
+
+
+## Requirements
+
+- Python 3.10+
+- `xai-sdk`
+- `vision-agents-core`
+
+## License
+
+Apache-2.0

vision_agents_plugins_xai-0.0.17/pyproject.toml

@@ -0,0 +1,35 @@
+[build-system]
+requires = ["hatchling", "hatch-vcs"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-xai"
+dynamic = ["version"]
+description = "XAI for stream agents"
+readme = "README.md"
+requires-python = ">=3.10.0"
+license = "Apache-2.0"
+dependencies = [
+    "xai-sdk",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]

vision_agents_plugins_xai-0.0.17/tests/test_xai_llm.py

@@ -0,0 +1,98 @@
+import pytest
+from dotenv import load_dotenv
+import os
+
+from vision_agents.core.agents.conversation import Message
+from vision_agents.plugins.xai.llm import XAILLM
+from vision_agents.core.llm.events import LLMResponseChunkEvent
+
+load_dotenv()
+
+
+class TestXAILLM:
+    """Test suite for XAILLM class with live API calls."""
+
+    def test_message(self):
+        messages = XAILLM._normalize_message("say hi")
+        assert isinstance(messages[0], Message)
+        message = messages[0]
+        assert message.original is not None
+        assert message.content == "say hi"
+
+    async def test_advanced_message(self):
+        img_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/d5/2023_06_08_Raccoon1.jpg/1599px-2023_06_08_Raccoon1.jpg"
+
+        advanced = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "input_text", "text": "what do you see in this image?"},
+                    {"type": "input_image", "image_url": f"{img_url}"},
+                ],
+            }
+        ]
+        messages2 = XAILLM._normalize_message(advanced)
+        assert messages2[0].original is not None
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(not os.getenv("XAI_API_KEY"), reason="XAI_API_KEY not set")
+    async def test_simple(self):
+        llm = XAILLM(model="grok-4", api_key=os.getenv("XAI_API_KEY"))
+        response = await llm.simple_response(
+            "Explain quantum computing in 1 paragraph",
+        )
+        assert response.text
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(not os.getenv("XAI_API_KEY"), reason="XAI_API_KEY not set")
+    async def test_native_api(self):
+        llm = XAILLM(model="grok-4", api_key=os.getenv("XAI_API_KEY"))
+        response = await llm.create_response(
+            input="say hi", instructions="You are a helpful assistant."
+        )
+        assert response.text
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(not os.getenv("XAI_API_KEY"), reason="XAI_API_KEY not set")
+    async def test_streaming(self):
+        llm = XAILLM(model="grok-4", api_key=os.getenv("XAI_API_KEY"))
+        streaming_works = False
+
+        @llm.events.subscribe
+        async def passed(event: LLMResponseChunkEvent):
+            nonlocal streaming_works
+            streaming_works = True
+
+        response = await llm.simple_response(
+            "Explain quantum computing in 1 paragraph",
+        )
+        await llm.events.wait()
+
+        assert response.text
+        assert streaming_works
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(not os.getenv("XAI_API_KEY"), reason="XAI_API_KEY not set")
+    async def test_memory(self):
+        llm = XAILLM(model="grok-4", api_key=os.getenv("XAI_API_KEY"))
+        await llm.simple_response(
+            text="There are 2 dogs in the room",
+        )
+        await llm.events.wait()
+        response = await llm.simple_response(
+            text="How many paws are there in the room?",
+        )
+        assert "8" in response.text or "eight" in response.text
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(not os.getenv("XAI_API_KEY"), reason="XAI_API_KEY not set")
+    async def test_native_memory(self):
+        llm = XAILLM(model="grok-4", api_key=os.getenv("XAI_API_KEY"))
+        await llm.create_response(
+            input="There are 2 dogs in the room",
+        )
+        await llm.events.wait()
+        response = await llm.create_response(
+            input="How many paws are there in the room?",
+        )
+        assert "8" in response.text or "eight" in response.text

vision_agents_plugins_xai-0.0.17/vision_agents/plugins/xai/__init__.py

@@ -0,0 +1,4 @@
+from .llm import XAILLM as LLM
+from .version import __version__
+
+__all__ = ["LLM", "__version__"]

vision_agents_plugins_xai-0.0.17/vision_agents/plugins/xai/events.py

@@ -0,0 +1,10 @@
+from dataclasses import dataclass, field
+from vision_agents.core.events import PluginBaseEvent
+from typing import Optional, Any
+
+
+@dataclass
+class XAIChunkEvent(PluginBaseEvent):
+    """Event emitted when xAI provides a chunk."""
+    type: str = field(default='plugin.xai.chunk', init=False)
+    chunk: Optional[Any] = None

vision_agents_plugins_xai-0.0.17/vision_agents/plugins/xai/llm.py

@@ -0,0 +1,200 @@
+from typing import Optional, List, Any, TYPE_CHECKING
+from xai_sdk import AsyncClient
+from xai_sdk.chat import system, user, Response, Chunk
+from xai_sdk.proto import chat_pb2
+
+from vision_agents.core.llm.llm import LLM, LLMResponseEvent
+from vision_agents.core.processors import Processor
+from vision_agents.core.llm.events import LLMResponseChunkEvent, LLMResponseCompletedEvent
+from . import events
+
+if TYPE_CHECKING:
+    from vision_agents.core.agents.conversation import Message
+    from getstream.video.rtc.pb.stream.video.sfu.models.models_pb2 import Participant
+    from xai_sdk.aio.chat import Chat
+else:
+    from getstream.video.rtc.pb.stream.video.sfu.models.models_pb2 import Participant
+
+
+class XAILLM(LLM):
+    """
+    The XAILLM class provides full/native access to the xAI SDK methods.
+    It only standardizes the minimal feature set that's needed for the agent integration.
+
+    The agent requires that we standardize:
+    - sharing instructions
+    - keeping conversation history
+    - response normalization
+
+    Notes on the xAI integration
+    - the native method is called create_response (maps to xAI chat.sample())
+    - history is maintained using the chat object's append method
+
+    Examples:
+
+        from vision_agents.plugins import xai
+        llm = xai.LLM(model="grok-beta")
+
+    """
+
+    def __init__(
+        self,
+        model: str = "grok-4",
+        api_key: Optional[str] = None,
+        client: Optional[AsyncClient] = None,
+    ):
+        """
+        Initialize the XAILLM class.
+
+        Args:
+            model (str): The xAI model to use. Defaults to "grok-4"
+            api_key: optional API key. by default loads from XAI_API_KEY
+            client: optional xAI client. by default creates a new client object.
+        """
+        super().__init__()
+        self.events.register_events_from_module(events)
+        self.model = model
+        self.xai_chat: Optional["Chat"] = None
+        self.conversation = None
+
+        if client is not None:
+            self.client = client
+        elif api_key is not None and api_key != "":
+            self.client = AsyncClient(api_key=api_key)
+        else:
+            self.client = AsyncClient()
+
+    async def simple_response(
+        self,
+        text: str,
+        processors: Optional[List[Processor]] = None,
+        participant: Optional[Participant] = None,
+    ):
+        """
+        simple_response is a standardized way (across openai, claude, gemini etc.) to create a response.
+
+        Args:
+            text: The text to respond to
+            processors: list of processors (which contain state) about the video/voice AI
+            participant: optionally the participant object
+
+        Examples:
+
+            llm.simple_response("say hi to the user, be mean")
+        """
+        instructions = None
+        if self.conversation is not None:
+            instructions = self.conversation.instructions
+
+        return await self.create_response(
+            input=text,
+            instructions=instructions,
+        )
+
+    async def create_response(self, *args: Any, **kwargs: Any) -> LLMResponseEvent[Response]:
+        """
+        create_response gives you full support/access to the native xAI chat.sample() and chat.stream() methods
+        this method wraps the xAI method and ensures we broadcast an event which the agent class hooks into
+        """
+        input_text = kwargs.get("input", "")
+        instructions = kwargs.get("instructions", "")
+        model = kwargs.get("model", self.model)
+        stream = kwargs.get("stream", True)
+
+        # Create or reuse chat session
+        if not self.xai_chat:
+            messages = []
+            if instructions:
+                messages.append(system(instructions))
+            self.xai_chat = self.client.chat.create(model=model, messages=messages)
+
+        # Add user message
+        assert self.xai_chat is not None
+        self.xai_chat.append(user(input_text))
+
+        # Get response based on streaming preference
+        if stream:
+            # Handle streaming response
+            llm_response: Optional[LLMResponseEvent[Response]] = None
+            assert self.xai_chat is not None
+            async for response, chunk in self.xai_chat.stream():
+                llm_response_optional = self._standardize_and_emit_chunk(
+                    chunk, response
+                )
+                if llm_response_optional is not None:
+                    llm_response = llm_response_optional
+
+            # Add response to chat history
+            if llm_response and llm_response.original:
+                assert self.xai_chat is not None
+                self.xai_chat.append(llm_response.original)
+        else:
+            # Handle non-streaming response
+            assert self.xai_chat is not None
+            response = await self.xai_chat.sample()
+            llm_response = LLMResponseEvent[Response](response, response.content)
+
+            # Add response to chat history
+            assert self.xai_chat is not None
+            self.xai_chat.append(response)
+
+        if llm_response is not None:
+            self.events.send(LLMResponseCompletedEvent(
+                original=llm_response.original,
+                text=llm_response.text
+            ))
+
+        return llm_response or LLMResponseEvent[Response](
+            Response(chat_pb2.GetChatCompletionResponse(), 0), ""
+        )
+
+    @staticmethod
+    def _normalize_message(input_text: str) -> List["Message"]:
+        """
+        Takes the input text and standardizes it so we can store it in chat
+        """
+        from vision_agents.core.agents.conversation import Message
+
+        # Create a standardized message from input text
+        message = Message(
+            original={"content": input_text, "role": "user", "type": "message"},
+            content=input_text,
+        )
+
+        return [message]
+
+    def _standardize_and_emit_chunk(
+        self, chunk: Chunk, response: Response
+    ) -> Optional[LLMResponseEvent[Response]]:
+        """
+        Forwards the chunk events and also send out a standardized version (the agent class hooks into that)
+        """
+        # Emit the raw chunk event
+        self.events.send(events.XAIChunkEvent(
+            plugin_name="xai",
+            chunk=chunk
+        ))
+
+        # Emit standardized delta events for content
+        if chunk.content:
+            self.events.send(LLMResponseChunkEvent(
+                content_index=0,  # xAI doesn't have content_index
+                item_id=chunk.proto.id if hasattr(chunk.proto, "id") else "",
+                output_index=0,  # xAI doesn't have output_index
+                sequence_number=0,  # xAI doesn't have sequence_number
+                delta=chunk.content,
+                plugin_name="xai",
+            ))
+
+        # Check if this is the final chunk (finish_reason indicates completion)
+        if chunk.choices and chunk.choices[0].finish_reason:
+            # This is the final chunk, return the complete response
+            llm_response = LLMResponseEvent[Response](response, response.content)
+            self.events.send(LLMResponseCompletedEvent(
+                plugin_name="xai",
+                text=llm_response.text,
+                original=llm_response.original
+            ))
+            return llm_response
+
+        return None

vision_agents_plugins_xai-0.0.17/vision_agents/plugins/xai/version.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"