solana-agent 20.1.2__py3-none-any.whl → 31.4.0__py3-none-any.whl

solana_agent/interfaces/providers/audio.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+
+
+class AudioTranscoder(ABC):
+    """Abstract audio transcoder for converting between compressed audio and PCM16.
+
+    Implementations may rely on external tools (e.g., ffmpeg).
+    """
+
+    @abstractmethod
+    async def to_pcm16(
+        self, audio_bytes: bytes, input_mime: str, rate_hz: int
+    ) -> bytes:
+        """Transcode arbitrary audio to mono PCM16LE at rate_hz.
+
+        Args:
+            audio_bytes: Source audio bytes (e.g., MP4/AAC)
+            input_mime: Source mime-type (e.g., 'audio/mp4', 'audio/aac')
+            rate_hz: Target sample rate
+        Returns:
+            Raw PCM16LE mono bytes at the given rate
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    async def from_pcm16(
+        self, pcm16_bytes: bytes, output_mime: str, rate_hz: int
+    ) -> bytes:
+        """Transcode mono PCM16LE at rate_hz to the desired output mime.
+
+        Args:
+            pcm16_bytes: Raw PCM16LE bytes
+            output_mime: Target mime-type (e.g., 'audio/aac')
+            rate_hz: Sample rate of the PCM
+        Returns:
+            Encoded audio bytes
+        """
+        raise NotImplementedError

solana_agent/interfaces/providers/data_storage.py

@@ -27,13 +27,20 @@ class DataStorageProvider(ABC):
 
     @abstractmethod
     def find(
-        self, collection: str, query: Dict, sort: Optional[List] = None, limit: int = 0, skip: int = 0
+        self,
+        collection: str,
+        query: Dict,
+        sort: Optional[List] = None,
+        limit: int = 0,
+        skip: int = 0,
     ) -> List[Dict]:
         """Find documents matching query."""
         pass
 
     @abstractmethod
-    def update_one(self, collection: str, query: Dict, update: Dict, upsert: bool = False) -> bool:
+    def update_one(
+        self, collection: str, query: Dict, update: Dict, upsert: bool = False
+    ) -> bool:
         """Update a document."""
         pass
 

solana_agent/interfaces/providers/llm.py

@@ -1,10 +1,20 @@
 from abc import ABC, abstractmethod
-from typing import AsyncGenerator, List, Literal, Type, TypeVar, Union
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Type,
+    TypeVar,
+    Union,
+)
 
 from pydantic import BaseModel
 
 
-T = TypeVar('T', bound=BaseModel)
+T = TypeVar("T", bound=BaseModel)
 
 
 class LLMProvider(ABC):
@@ -15,13 +25,43 @@ class LLMProvider(ABC):
         self,
         prompt: str,
         system_prompt: str = "",
-    ) -> AsyncGenerator[str, None]:
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        model: Optional[str] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> Any:
         """Generate text from the language model."""
         pass
 
+    @abstractmethod
+    async def chat_stream(
+        self,
+        messages: List[Dict[str, Any]],
+        model: Optional[str] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+    ) -> AsyncGenerator[Dict[str, Any], None]:
+        """Stream chat completion deltas and tool call deltas.
+
+            Yields normalized events:
+            - {"type": "content", "delta": str}
+        - {"type": "tool_call_delta", "id": Optional[str], "index": Optional[int], "name": Optional[str], "arguments_delta": str}
+            - {"type": "message_end", "finish_reason": str}
+            - {"type": "error", "error": str}
+        """
+        pass
+
     @abstractmethod
     async def parse_structured_output(
-        self, prompt: str, system_prompt: str, model_class: Type[T],
+        self,
+        prompt: str,
+        system_prompt: str,
+        model_class: Type[T],
+        api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        model: Optional[str] = None,
+        tools: Optional[List[Dict[str, Any]]] = None,
     ) -> T:
         """Generate structured output using a specific model class."""
         pass
@@ -30,11 +70,19 @@ class LLMProvider(ABC):
     async def tts(
         self,
         text: str,
-        instructions: str = "",
-        voice: Literal["alloy", "ash", "ballad", "coral", "echo",
-                       "fable", "onyx", "nova", "sage", "shimmer"] = "nova",
-        response_format: Literal['mp3', 'opus',
-                                 'aac', 'flac', 'wav', 'pcm'] = "aac",
+        voice: Literal[
+            "alloy",
+            "ash",
+            "ballad",
+            "coral",
+            "echo",
+            "fable",
+            "onyx",
+            "nova",
+            "sage",
+            "shimmer",
+        ] = "nova",
+        response_format: Literal["mp3", "opus", "aac", "flac", "wav", "pcm"] = "aac",
     ) -> AsyncGenerator[bytes, None]:
         """Stream text-to-speech audio from the language model."""
         pass
@@ -49,3 +97,32 @@ class LLMProvider(ABC):
     ) -> AsyncGenerator[str, None]:
         """Transcribe audio from the language model."""
         pass
+
+    @abstractmethod
+    async def embed_text(
+        self, text: str, model: Optional[str] = None, dimensions: Optional[int] = None
+    ) -> List[float]:
+        """
+        Generate an embedding for the given text.
+
+        Args:
+            text: The text to embed.
+            model: The embedding model to use.
+            dimensions: Optional desired output dimensions for the embedding.
+
+        Returns:
+            A list of floats representing the embedding vector.
+        """
+        pass
+
+    @abstractmethod
+    async def generate_text_with_images(
+        self,
+        prompt: str,
+        images: List[Union[str, bytes]],
+        system_prompt: str = "",
+        detail: Literal["low", "high", "auto"] = "auto",
+        tools: Optional[List[Dict[str, Any]]] = None,
+    ) -> str:
+        """Generate text from the language model using images."""
+        pass

solana_agent/interfaces/providers/memory.py

@@ -27,7 +27,7 @@ class MemoryProvider(ABC):
         query: Dict,
         sort: Optional[List[Tuple]] = None,
         limit: int = 0,
-        skip: int = 0
+        skip: int = 0,
     ) -> List[Dict]:
         """Find documents matching query."""
         pass
@@ -36,3 +36,15 @@ class MemoryProvider(ABC):
     def count_documents(self, collection: str, query: Dict) -> int:
         """Count documents matching query."""
         pass
+
+    @abstractmethod
+    async def save_capture(
+        self,
+        user_id: str,
+        capture_name: str,
+        agent_name: Optional[str],
+        data: Dict[str, Any],
+        schema: Optional[Dict[str, Any]] = None,
+    ) -> Optional[str]:
+        """Persist a structured capture for a user and return its ID if available."""
+        pass

solana_agent/interfaces/providers/realtime.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import (
+    Any,
+    AsyncGenerator,
+    Dict,
+    Literal,
+    Optional,
+    Awaitable,
+    Callable,
+    List,
+    Union,
+)
+
+
+@dataclass
+class RealtimeSessionOptions:
+    model: Optional[str] = None
+    voice: Literal[
+        "alloy",
+        "ash",
+        "ballad",
+        "cedar",
+        "coral",
+        "echo",
+        "marin",
+        "sage",
+        "shimmer",
+        "verse",
+    ] = "marin"
+    vad_enabled: bool = True
+    input_rate_hz: int = 24000
+    output_rate_hz: int = 24000
+    input_mime: str = "audio/pcm"  # 16-bit PCM
+    output_mime: str = "audio/pcm"  # 16-bit PCM
+    output_modalities: List[Literal["audio", "text"]] = None  # None means auto-detect
+    instructions: Optional[str] = None
+    # Optional: tools payload compatible with OpenAI Realtime session.update
+    tools: Optional[list[dict[str, Any]]] = None
+    tool_choice: str = "auto"
+    # Tool execution behavior
+    # Max time to allow a tool to run before timing out (seconds)
+    tool_timeout_s: float = 300.0
+    # Optional guard: if a tool takes longer than this to complete, skip sending
+    # function_call_output to avoid stale/expired call_id issues. Set to None to always send.
+    tool_result_max_age_s: Optional[float] = None
+    # --- Realtime transcription configuration (optional) ---
+    # When transcription_model is set, QueryService should skip the HTTP STT path and rely on
+    # realtime websocket transcription events. Other fields customize that behavior.
+    transcription_model: Optional[str] = None
+    transcription_language: Optional[str] = None  # e.g. 'en'
+    transcription_prompt: Optional[str] = None
+    transcription_noise_reduction: Optional[bool] = None
+    transcription_include_logprobs: bool = False
+
+
+@dataclass
+class RealtimeChunk:
+    """Represents a chunk of data from a realtime session with its modality type."""
+
+    modality: Literal["audio", "text"]
+    data: Union[str, bytes]
+    timestamp: Optional[float] = None  # Optional timestamp for ordering
+    metadata: Optional[Dict[str, Any]] = None  # Optional additional metadata
+
+    @property
+    def is_audio(self) -> bool:
+        """Check if this is an audio chunk."""
+        return self.modality == "audio"
+
+    @property
+    def is_text(self) -> bool:
+        """Check if this is a text chunk."""
+        return self.modality == "text"
+
+    @property
+    def text_data(self) -> Optional[str]:
+        """Get text data if this is a text chunk."""
+        return self.data if isinstance(self.data, str) else None
+
+    @property
+    def audio_data(self) -> Optional[bytes]:
+        """Get audio data if this is an audio chunk."""
+        return self.data if isinstance(self.data, bytes) else None
+
+
+async def separate_audio_chunks(
+    chunks: AsyncGenerator[RealtimeChunk, None],
+) -> AsyncGenerator[bytes, None]:
+    """Extract only audio chunks from a stream of RealtimeChunk objects.
+
+    Args:
+        chunks: Stream of RealtimeChunk objects
+
+    Yields:
+        Audio data bytes from audio chunks only
+    """
+    async for chunk in chunks:
+        if chunk.is_audio and chunk.audio_data:
+            yield chunk.audio_data
+
+
+async def separate_text_chunks(
+    chunks: AsyncGenerator[RealtimeChunk, None],
+) -> AsyncGenerator[str, None]:
+    """Extract only text chunks from a stream of RealtimeChunk objects.
+
+    Args:
+        chunks: Stream of RealtimeChunk objects
+
+    Yields:
+        Text data from text chunks only
+    """
+    async for chunk in chunks:
+        if chunk.is_text and chunk.text_data:
+            yield chunk.text_data
+
+
+async def demux_realtime_chunks(
+    chunks: AsyncGenerator[RealtimeChunk, None],
+) -> tuple[AsyncGenerator[bytes, None], AsyncGenerator[str, None]]:
+    """Demux a stream of RealtimeChunk objects into separate audio and text streams.
+
+    Note: This function consumes the input generator, so each output stream can only be consumed once.
+
+    Args:
+        chunks: Stream of RealtimeChunk objects
+
+    Returns:
+        Tuple of (audio_stream, text_stream) async generators
+    """
+    # Collect all chunks first since we can't consume the generator twice
+    collected_chunks = []
+    async for chunk in chunks:
+        collected_chunks.append(chunk)
+
+    async def audio_stream():
+        for chunk in collected_chunks:
+            if chunk.is_audio and chunk.audio_data:
+                yield chunk.audio_data
+
+    async def text_stream():
+        for chunk in collected_chunks:
+            if chunk.is_text and chunk.text_data:
+                yield chunk.text_data
+
+    return audio_stream(), text_stream()
+
+
+class BaseRealtimeSession(ABC):
+    """Abstract realtime session supporting bidirectional audio/text over WebSocket."""
+
+    @abstractmethod
+    async def connect(self) -> None:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    async def close(self) -> None:  # pragma: no cover
+        pass
+
+    # --- Client events ---
+    @abstractmethod
+    async def update_session(
+        self, session_patch: Dict[str, Any]
+    ) -> None:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    async def append_audio(self, pcm16_bytes: bytes) -> None:  # pragma: no cover
+        """Append 16-bit PCM audio bytes (matching configured input rate/mime)."""
+        pass
+
+    @abstractmethod
+    async def commit_input(self) -> None:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    async def clear_input(self) -> None:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    async def create_response(
+        self, response_patch: Optional[Dict[str, Any]] = None
+    ) -> None:  # pragma: no cover
+        pass
+
+    # --- Server events (demuxed) ---
+    @abstractmethod
+    def iter_events(self) -> AsyncGenerator[Dict[str, Any], None]:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    def iter_output_audio(self) -> AsyncGenerator[bytes, None]:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    def iter_input_transcript(self) -> AsyncGenerator[str, None]:  # pragma: no cover
+        pass
+
+    @abstractmethod
+    def iter_output_transcript(self) -> AsyncGenerator[str, None]:  # pragma: no cover
+        pass
+
+    # --- Optional tool execution hook ---
+    @abstractmethod
+    def set_tool_executor(
+        self,
+        executor: Callable[[str, Dict[str, Any]], Awaitable[Dict[str, Any]]],
+    ) -> None:  # pragma: no cover
+        """Register a coroutine that executes a tool by name with arguments and returns a result dict."""
+        pass

solana_agent/interfaces/providers/vector_storage.py

@@ -0,0 +1,53 @@
+from typing import List, Dict, Any, Optional
+
+
+class VectorStorageProvider:
+    """Interface for Vector Storage Providers."""
+
+    async def upsert(
+        self, vectors: List[Dict[str, Any]], namespace: Optional[str] = None
+    ) -> None:
+        """Upsert vectors into the storage."""
+        pass
+
+    async def upsert_text(
+        self,
+        texts: List[str],
+        ids: List[str],
+        metadatas: Optional[List[Dict[str, Any]]] = None,
+        namespace: Optional[str] = None,
+    ) -> None:
+        """Embeds texts and upserts them into the storage."""
+        pass
+
+    async def query(
+        self,
+        vector: List[float],
+        top_k: int = 5,
+        namespace: Optional[str] = None,
+        filter: Optional[Dict[str, Any]] = None,
+        include_values: bool = False,
+        include_metadata: bool = True,
+    ) -> List[Dict[str, Any]]:
+        """Query for similar vectors."""
+        pass
+
+    async def query_text(
+        self,
+        query_text: str,
+        top_k: Optional[int] = None,
+        namespace: Optional[str] = None,
+        filter: Optional[Dict[str, Any]] = None,
+        include_values: bool = False,
+        include_metadata: bool = True,
+    ) -> List[Dict[str, Any]]:
+        """Embeds query text and queries the storage."""
+        pass
+
+    async def delete(self, ids: List[str], namespace: Optional[str] = None) -> None:
+        """Delete vectors by IDs."""
+        pass
+
+    async def describe_index_stats(self) -> Dict[str, Any]:
+        """Get statistics about the index/collection."""
+        pass

solana_agent/interfaces/services/agent.py

@@ -1,5 +1,7 @@
 from abc import ABC, abstractmethod
-from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Union
+from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Type, Union
+
+from pydantic import BaseModel
 
 from solana_agent.domains.agent import AIAgent
 
@@ -8,7 +10,9 @@ class AgentService(ABC):
     """Interface for agent management and response generation."""
 
     @abstractmethod
-    def register_ai_agent(self, name: str, instructions: str, specialization: str) -> None:
+    def register_ai_agent(
+        self, name: str, instructions: str, specialization: str
+    ) -> None:
         """Register an AI agent with its specialization."""
         pass
 
@@ -25,16 +29,25 @@ class AgentService(ABC):
         query: Union[str, bytes],
         memory_context: str = "",
         output_format: Literal["text", "audio"] = "text",
-        audio_voice: Literal["alloy", "ash", "ballad", "coral", "echo",
-                             "fable", "onyx", "nova", "sage", "shimmer"] = "nova",
-        audio_instructions: Optional[str] = None,
-        audio_output_format: Literal['mp3', 'opus',
-                                     'aac', 'flac', 'wav', 'pcm'] = "aac",
-        audio_input_format: Literal[
-            "flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"
-        ] = "mp4",
+        audio_voice: Literal[
+            "alloy",
+            "ash",
+            "ballad",
+            "coral",
+            "echo",
+            "fable",
+            "onyx",
+            "nova",
+            "sage",
+            "shimmer",
+        ] = "nova",
+        audio_output_format: Literal[
+            "mp3", "opus", "aac", "flac", "wav", "pcm"
+        ] = "aac",
         prompt: Optional[str] = None,
-    ) -> AsyncGenerator[Union[str, bytes], None]:
+        images: Optional[List[Union[str, bytes]]] = None,
+        output_model: Optional[Type[BaseModel]] = None,
+    ) -> AsyncGenerator[Union[str, bytes, BaseModel], None]:
         """Generate a response from an agent."""
         pass
 
@@ -49,7 +62,9 @@ class AgentService(ABC):
         pass
 
     @abstractmethod
-    async def execute_tool(self, agent_name: str, tool_name: str, parameters: Dict[str, Any]) -> Dict[str, Any]:
+    async def execute_tool(
+        self, agent_name: str, tool_name: str, parameters: Dict[str, Any]
+    ) -> Dict[str, Any]:
         """Execute a tool on behalf of an agent."""
         pass
 

solana_agent/interfaces/services/knowledge_base.py

@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List, Any, Optional, Union
+
+
+class KnowledgeBaseService(ABC):
+    """
+    Interface for a Knowledge Base service.
+    """
+
+    @abstractmethod
+    async def add_document(
+        self,
+        text: str,
+        metadata: Dict[str, Any],
+        document_id: Optional[str] = None,
+        namespace: Optional[str] = None,
+    ) -> str:
+        """
+        Add a document to the knowledge base.
+        """
+        pass
+
+    @abstractmethod
+    async def query(
+        self,
+        query_text: str,
+        filter: Optional[Dict[str, Any]] = None,
+        top_k: int = 5,
+        namespace: Optional[str] = None,
+        include_content: bool = True,
+        include_metadata: bool = True,
+    ) -> List[Dict[str, Any]]:
+        """
+        Query the knowledge base with semantic search.
+        """
+        pass
+
+    @abstractmethod
+    async def delete_document(
+        self, document_id: str, namespace: Optional[str] = None
+    ) -> bool:
+        """
+        Delete a document from the knowledge base.
+        """
+        pass
+
+    @abstractmethod
+    async def add_pdf_document(
+        self,
+        pdf_data: Union[bytes, str],  # PDF bytes or file path
+        metadata: Dict[str, Any],
+        document_id: Optional[str] = None,
+        namespace: Optional[str] = None,
+        chunk_batch_size: int = 50,
+    ) -> str:
+        """
+        Add a PDF document to the knowledge base.
+        """
+        pass

solana_agent/interfaces/services/query.py

@@ -1,5 +1,10 @@
 from abc import ABC, abstractmethod
-from typing import Any, AsyncGenerator, Dict, Literal, Optional, Union
+from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Type, Union
+
+from pydantic import BaseModel
+
+from solana_agent.interfaces.services.routing import RoutingService as RoutingInterface
+from solana_agent.interfaces.providers.realtime import RealtimeChunk
 
 
 class QueryService(ABC):
@@ -11,16 +16,44 @@ class QueryService(ABC):
         user_id: str,
         query: Union[str, bytes],
         output_format: Literal["text", "audio"] = "text",
-        audio_voice: Literal["alloy", "ash", "ballad", "coral", "echo",
-                             "fable", "onyx", "nova", "sage", "shimmer"] = "nova",
-        audio_instructions: Optional[str] = None,
-        audio_output_format: Literal['mp3', 'opus',
-                                     'aac', 'flac', 'wav', 'pcm'] = "aac",
+        rt_output_modalities: Optional[List[Literal["audio", "text"]]] = None,
+        rt_voice: Literal[
+            "alloy",
+            "ash",
+            "ballad",
+            "cedar",
+            "coral",
+            "echo",
+            "marin",
+            "sage",
+            "shimmer",
+            "verse",
+        ] = "marin",
+        audio_voice: Literal[
+            "alloy",
+            "ash",
+            "ballad",
+            "coral",
+            "echo",
+            "fable",
+            "onyx",
+            "nova",
+            "sage",
+            "shimmer",
+        ] = "nova",
+        audio_output_format: Literal[
+            "mp3", "opus", "aac", "flac", "wav", "pcm"
+        ] = "aac",
         audio_input_format: Literal[
             "flac", "mp3", "mp4", "mpeg", "mpga", "m4a", "ogg", "wav", "webm"
         ] = "mp4",
         prompt: Optional[str] = None,
-    ) -> AsyncGenerator[Union[str, bytes], None]:
+        router: Optional[RoutingInterface] = None,
+        images: Optional[List[Union[str, bytes]]] = None,
+        output_model: Optional[Type[BaseModel]] = None,
+        capture_schema: Optional[Dict[str, Any]] = None,
+        capture_name: Optional[str] = None,
+    ) -> AsyncGenerator[Union[str, bytes, BaseModel, RealtimeChunk], None]:
         """Process the user request and generate a response."""
         pass
 
@@ -30,7 +63,7 @@ class QueryService(ABC):
         user_id: str,
         page_num: int = 1,
         page_size: int = 20,
-        sort_order: str = "desc"  # "asc" for oldest-first, "desc" for newest-first
+        sort_order: str = "desc",  # "asc" for oldest-first, "desc" for newest-first
     ) -> Dict[str, Any]:
         """Get paginated message history for a user."""
         pass

solana_agent/interfaces/services/routing.py

@@ -1,5 +1,4 @@
 from abc import ABC, abstractmethod
-from typing import Any, Tuple
 
 
 class RoutingService(ABC):