spaik-sdk 0.6.2__py3-none-any.whl

spaik_sdk/llm/cost/builtin_cost_provider.py

@@ -0,0 +1,83 @@
+from spaik_sdk.llm.consumption.token_usage import TokenUsage
+from spaik_sdk.llm.cost.cost_provider import CostProvider
+from spaik_sdk.models.llm_model import LLMModel
+
+
+class BuiltinCostProvider(CostProvider):
+    def get_token_pricing(self, model: LLMModel) -> TokenUsage:
+        """Get token pricing in USD cents per million tokens."""
+        name = model.name
+
+        # Anthropic Claude models
+        if name.startswith("claude-3-7-sonnet"):
+            # Claude 3.7 Sonnet: $3.00 input, $15.00 output per 1M tokens
+            return TokenUsage(
+                input_tokens=300,  # $3.00 in cents per 1M tokens
+                output_tokens=1500,  # $15.00 in cents per 1M tokens
+                reasoning_tokens=0,
+                cache_creation_tokens=375,  # 25% markup on input for cache creation
+                cache_read_tokens=30,  # 10% of input cost for cache reads
+            )
+        elif name.startswith("claude-sonnet-4") or name.startswith("claude-4-sonnet"):
+            # Claude 4 Sonnet: $3.00 input, $15.00 output per 1M tokens
+            return TokenUsage(input_tokens=300, output_tokens=1500, reasoning_tokens=0, cache_creation_tokens=375, cache_read_tokens=30)
+        elif name.startswith("claude-opus-4") or name.startswith("claude-4-opus"):
+            # Claude 4 Opus: $15.00 input, $75.00 output per 1M tokens
+            return TokenUsage(input_tokens=1500, output_tokens=7500, reasoning_tokens=0, cache_creation_tokens=1875, cache_read_tokens=150)
+
+        # OpenAI models
+        elif name.startswith("gpt-4.1"):
+            # GPT-4.1: $2.00 input, $8.00 output per 1M tokens
+            return TokenUsage(input_tokens=200, output_tokens=800, reasoning_tokens=0, cache_creation_tokens=250, cache_read_tokens=20)
+        elif name.startswith("gpt-4o"):
+            # GPT-4o: $2.50 input, $10.00 output per 1M tokens
+            return TokenUsage(input_tokens=250, output_tokens=1000, reasoning_tokens=0, cache_creation_tokens=312, cache_read_tokens=25)
+        elif name.startswith("o4-mini"):
+            # O4-mini: $0.40 input, $1.60 output per 1M tokens (based on GPT-4.1-mini pricing)
+            return TokenUsage(
+                input_tokens=40,
+                output_tokens=160,
+                reasoning_tokens=440,  # 110% markup for reasoning tokens
+                cache_creation_tokens=50,
+                cache_read_tokens=4,
+            )
+        elif name.startswith("gpt-5"):
+            if "nano" in name:
+                # GPT-5 Nano: $0.05 input, $0.40 output per 1M tokens
+                return TokenUsage(
+                    input_tokens=5,
+                    output_tokens=40,
+                    reasoning_tokens=44,  # 10% markup for reasoning
+                    cache_creation_tokens=6,  # 25% markup on input for cache creation
+                    cache_read_tokens=0,  # 90% discount: $0.005 per 1M tokens
+                )
+            elif "mini" in name:
+                # GPT-5 Mini: $0.25 input, $2.00 output per 1M tokens
+                return TokenUsage(
+                    input_tokens=25,
+                    output_tokens=200,
+                    reasoning_tokens=220,  # 10% markup for reasoning
+                    cache_creation_tokens=31,  # 25% markup on input for cache creation
+                    cache_read_tokens=2,  # 90% discount: $0.025 per 1M tokens
+                )
+            else:
+                # GPT-5: $1.25 input, $10.00 output per 1M tokens
+                return TokenUsage(
+                    input_tokens=125,
+                    output_tokens=1000,
+                    reasoning_tokens=1100,  # 10% markup for reasoning
+                    cache_creation_tokens=156,  # 25% markup on input for cache creation
+                    cache_read_tokens=12,  # 90% discount: $0.125 per 1M tokens
+                )
+
+        # Google Gemini models
+        elif name.startswith("gemini-2.5-flash"):
+            # Gemini 2.5 Flash: $0.15 input, $0.60 output per 1M tokens
+            return TokenUsage(input_tokens=15, output_tokens=60, reasoning_tokens=0, cache_creation_tokens=19, cache_read_tokens=1)
+        elif name.startswith("gemini-2.5-pro"):
+            # Gemini 2.5 Pro: $1.25 input, $10.00 output per 1M tokens
+            return TokenUsage(input_tokens=125, output_tokens=1000, reasoning_tokens=0, cache_creation_tokens=156, cache_read_tokens=12)
+
+        # Default fallback for unknown models
+        else:
+            return TokenUsage(input_tokens=0, output_tokens=0, reasoning_tokens=0, cache_creation_tokens=0, cache_read_tokens=0)

spaik_sdk/llm/cost/cost_estimate.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class CostEstimate:
+    cost: float
+    currency: str
+    is_estimate: bool

spaik_sdk/llm/cost/cost_provider.py

@@ -0,0 +1,28 @@
+from abc import ABC, abstractmethod
+
+from spaik_sdk.llm.consumption.token_usage import TokenUsage
+from spaik_sdk.llm.cost.cost_estimate import CostEstimate
+from spaik_sdk.models.llm_model import LLMModel
+
+
+class CostProvider(ABC):
+    def get_cost_estimate(self, model: LLMModel, token_usage: TokenUsage) -> CostEstimate:
+        token_pricing: TokenUsage = self.get_token_pricing(model)
+
+        total = 0
+
+        total += token_usage.input_tokens * token_pricing.input_tokens
+        total += token_usage.output_tokens * token_pricing.output_tokens
+        total += token_usage.reasoning_tokens * token_pricing.reasoning_tokens
+        total += token_usage.cache_creation_tokens * token_pricing.cache_creation_tokens
+        total += token_usage.cache_read_tokens * token_pricing.cache_read_tokens
+
+        return CostEstimate(
+            cost=(total) / 100000000.0,
+            currency="USD",
+            is_estimate=False,
+        )
+
+    @abstractmethod
+    def get_token_pricing(self, model: LLMModel) -> TokenUsage:
+        pass

spaik_sdk/llm/extract_error_message.py

@@ -0,0 +1,37 @@
+import json
+
+
+def extract_error_message(exception: Exception) -> str:
+    """Extract a meaningful error message from various exception types."""
+    error_str = str(exception)
+
+    # Try to parse as JSON if it looks like a structured error
+    if "Error code:" in error_str and "{" in error_str:
+        try:
+            # Extract JSON part from the error string
+            json_start = error_str.find("{")
+            json_part = error_str[json_start:]
+            error_data = json.loads(json_part)
+
+            # Handle Azure OpenAI content filter errors
+            if "error" in error_data:
+                error_info = error_data["error"]
+                if error_info.get("code") == "content_filter":
+                    return f"Content filtered: {error_info.get('message', 'Content policy violation')}"
+                else:
+                    return error_info.get("message", error_str)
+        except (json.JSONDecodeError, KeyError):
+            pass
+
+    # Handle other common error patterns
+    if "content management policy" in error_str.lower():
+        return "Content was filtered due to content management policy"
+    elif "rate limit" in error_str.lower():
+        return "Rate limit exceeded"
+    elif "authentication" in error_str.lower():
+        return "Authentication failed"
+    elif "quota" in error_str.lower():
+        return "Quota exceeded"
+
+    # Return the original error message if no specific pattern matches
+    return error_str

spaik_sdk/llm/langchain_loop_manager.py

@@ -0,0 +1,270 @@
+"""
+LangChain Loop Manager - Event Loop Isolation for Models with Event Loop Issues
+
+This module exists to work around a fundamental incompatibility between certain
+model providers (Google/Gemini, Ollama) and the way asyncio.run() manages event loops.
+
+THE PROBLEM:
+============
+Some model providers (Google's gRPC-based clients, Ollama's async HTTP client, etc.)
+create internal connections and async state that get bound to the specific
+event loop instance they're created in. When that event loop closes, these
+internal connections become unusable and raise "Event loop is closed" errors.
+
+This manifests in two scenarios:
+
+1. STANDALONE SCRIPTS with multiple asyncio.run() calls:
+   ```python
+   asyncio.run(main())  # Creates event loop, Google client binds to it
+   # Event loop closes here
+   asyncio.run(main())  # Creates NEW event loop, but Google client still references old one
+   # → RuntimeError: Event loop is closed
+   ```
+
+2. WEB SERVERS (FastAPI, etc.) with persistent event loops:
+   ```python
+   # Web server starts one event loop and keeps it running
+   # All requests use the SAME loop, so Google client works fine
+   ```
+
+THE WORKAROUND:
+===============
+We detect the execution context and apply different strategies:
+
+1. **Standalone Context** (detected by stack frame inspection):
+   - Use a persistent background event loop in a separate thread
+   - All affected model operations run in this persistent loop
+   - The background loop never closes, so the clients stay happy
+
+2. **Web Server Context** (detected by uvicorn/fastapi in call stack):
+   - Use normal execution (no loop manager)
+   - The web server's persistent loop handles everything naturally
+
+WHY WEB SERVERS CAN'T USE THE LOOP MANAGER:
+===========================================
+Web servers MUST NOT use the external event loop approach because:
+
+1. **Streaming breaks**: When operations run in a separate thread's event loop,
+   you lose the ability to stream results back to the web server's event loop
+   in real-time. The thread boundary kills the streaming semantics.
+
+2. **Request context isolation**: Web frameworks expect all operations for a
+   request to happen in the same event loop to maintain proper async context,
+   request isolation, and cancellation semantics.
+
+3. **Performance overhead**: Cross-thread async communication adds significant
+   latency and complexity that's unnecessary when the web server already
+   provides a persistent event loop.
+
+The key insight: Web servers naturally solve these client issues by having
+persistent event loops, so they don't need (and can't use) the workaround.
+
+DETECTION STRATEGY:
+===================
+We use multiple heuristics to detect execution context:
+- Thread names (uvicorn, fastapi, etc.)
+- Call stack inspection (looking for web framework files)
+- Event loop state (persistent vs transient)
+
+This is admittedly hacky, but it's the only way to transparently handle
+both contexts without requiring users to explicitly configure the behavior.
+
+AFFECTED MODELS:
+================
+- All Google/Gemini models (provider_type == ProviderType.GOOGLE)
+- All Ollama models (provider_type == ProviderType.OLLAMA)
+- Other providers (Anthropic, OpenAI) are unaffected
+
+EXAMPLES:
+=========
+```python
+# This would fail with Gemini without the loop manager:
+asyncio.run(agent.get_response("hello"))
+asyncio.run(agent.get_response("world"))  # ← RuntimeError
+
+# This works fine with web servers (no loop manager needed):
+@app.post("/chat")
+async def chat():
+    return await agent.get_response("hello")  # Same persistent loop
+```
+
+ALTERNATIVES CONSIDERED:
+========================
+1. Process isolation - Too heavy, breaks streaming
+2. Raw Google API - Bypasses LangChain ecosystem
+3. Client recreation - Google's state is deeper than we can reach
+4. Thread-per-call - Breaks async/await semantics
+5. User configuration - Poor DX, easy to get wrong
+
+WHY THIS IS NECESSARY:
+======================
+Some client designs assume a long-lived event loop (like in web servers).
+The asyncio.run() pattern creates short-lived loops that violate this assumption.
+Other providers (Anthropic, OpenAI) handle this gracefully by recreating connections
+or using stateless clients.
+
+This is a known limitation that's unlikely to be fixed in these clients
+since it would require significant architectural changes on their end.
+"""
+
+import asyncio
+import threading
+import time
+from typing import Optional
+
+from spaik_sdk.models.llm_config import LLMConfig
+from spaik_sdk.models.providers.provider_type import ProviderType
+from spaik_sdk.utils.init_logger import init_logger
+
+logger = init_logger(__name__)
+
+
+class LangChainLoopManager:
+    """Manages a persistent event loop for langchain operations"""
+
+    def __init__(self):
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        self._loop_thread: Optional[threading.Thread] = None
+        self._lock = threading.Lock()
+
+    def get_loop(self) -> asyncio.AbstractEventLoop:
+        """Get or create the persistent event loop for langchain operations"""
+        with self._lock:
+            if self._loop is None or self._loop.is_closed():
+                self._loop = None
+
+                def run_loop():
+                    self._loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(self._loop)
+                    self._loop.run_forever()
+
+                self._loop_thread = threading.Thread(target=run_loop, daemon=True)
+                self._loop_thread.start()
+
+                # Wait for loop to be created
+                while self._loop is None:
+                    time.sleep(0.01)
+
+            return self._loop
+
+    async def run_in_loop(self, coro):
+        """Run a coroutine in the langchain loop and return the result"""
+        loop = self.get_loop()
+
+        try:
+            current_loop = asyncio.get_running_loop()
+            if current_loop != loop:
+                # We're in a different loop context, run in langchain loop
+                future = asyncio.run_coroutine_threadsafe(coro, loop)
+                return future.result()
+            else:
+                # We're already in the langchain loop, run directly
+                return await coro
+        except RuntimeError:
+            # No running loop, run in langchain loop
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            return future.result()
+
+    async def stream_in_loop(self, async_generator):
+        """Stream results from an async generator running in the langchain loop"""
+        loop = self.get_loop()
+
+        try:
+            current_loop = asyncio.get_running_loop()
+            if current_loop != loop:
+                # We're in a different loop context, collect all results first
+                future = asyncio.run_coroutine_threadsafe(self._collect_from_async_generator(async_generator), loop)
+                results = future.result()
+                for result in results:
+                    yield result
+            else:
+                # We're already in the langchain loop, stream directly
+                async for result in async_generator:
+                    yield result
+        except RuntimeError:
+            # No running loop, collect all results first
+            future = asyncio.run_coroutine_threadsafe(self._collect_from_async_generator(async_generator), loop)
+            results = future.result()
+            for result in results:
+                yield result
+
+    async def _collect_from_async_generator(self, async_generator):
+        """Collect all items from an async generator"""
+        results = []
+        async for item in async_generator:
+            results.append(item)
+        return results
+
+
+def _is_in_web_server_context() -> bool:
+    """Detect if we're running in a web server/FastAPI context vs standalone asyncio.run()."""
+    try:
+        # Check if we're in an event loop
+        loop = asyncio.get_running_loop()
+
+        # FastAPI/web servers typically run event loops indefinitely
+        # Check for common web server indicators in the call stack
+        import inspect
+        import threading
+
+        # Get current thread name - web servers often have descriptive thread names
+        thread_name = threading.current_thread().name
+        if any(name in thread_name.lower() for name in ["uvicorn", "fastapi", "starlette", "asgi", "wsgi"]):
+            return True
+
+        # Check the call stack for web framework indicators
+        frame = inspect.currentframe()
+        try:
+            while frame:
+                frame_info = inspect.getframeinfo(frame)
+                filename = frame_info.filename.lower()
+
+                # Look for web framework files in the call stack
+                if any(
+                    indicator in filename
+                    for indicator in ["uvicorn", "fastapi", "starlette", "asgi", "wsgi", "tornado", "aiohttp", "sanic", "quart"]
+                ):
+                    return True
+
+                frame = frame.f_back
+        finally:
+            del frame
+
+        # Check if the event loop has been running for a while (web servers)
+        # vs just started (asyncio.run())
+        if hasattr(loop, "_ready") and hasattr(loop._ready, "__len__") and len(loop._ready) > 0:  # type: ignore[arg-type]
+            # This is a heuristic - web servers tend to have more pending tasks
+            return True
+
+        return False
+
+    except RuntimeError:
+        # No running event loop - definitely not in a web server
+        return False
+    except Exception as e:
+        logger.debug(f"Error detecting web server context: {e}")
+        return False
+
+
+def _needs_loop_manager(llm_config: LLMConfig) -> bool:
+    """Check if this model provider might have event loop issues with multiple asyncio.run() calls."""
+    return llm_config.provider_type in (ProviderType.GOOGLE, ProviderType.OLLAMA)
+
+
+def should_use_loop_manager(llm_config: LLMConfig) -> bool:
+    """Determine if we should use the loop manager for models with event loop issues."""
+    if not _needs_loop_manager(llm_config):
+        return False
+
+    # Only use loop manager if NOT in a web server context
+    # Web servers have persistent event loops, so the issue doesn't occur
+    return not _is_in_web_server_context()
+
+
+# Global instance
+_loop_manager = LangChainLoopManager()
+
+
+def get_langchain_loop_manager() -> LangChainLoopManager:
+    """Get the global langchain loop manager instance"""
+    return _loop_manager

spaik_sdk/llm/langchain_service.py

@@ -0,0 +1,196 @@
+import json
+import logging
+import time
+import uuid
+from typing import Dict, List, Optional, Type, TypeVar, cast
+
+from langchain_core.runnables import RunnableConfig
+from langchain_core.tools import BaseTool
+
+# Using create_react_agent because create_agent from langchain.agents
+# uses invoke() internally and does NOT emit on_chat_model_stream events,
+# which breaks token-level streaming. See: https://github.com/langchain-ai/langchain/issues/34017
+from langgraph.prebuilt import create_react_agent
+from pydantic import BaseModel
+
+from spaik_sdk.attachments.file_storage_provider import get_file_storage
+from spaik_sdk.attachments.models import Attachment
+from spaik_sdk.config.env import env_config
+from spaik_sdk.llm.cancellation_handle import CancellationHandle
+from spaik_sdk.llm.extract_error_message import extract_error_message
+from spaik_sdk.llm.langchain_loop_manager import get_langchain_loop_manager, should_use_loop_manager
+from spaik_sdk.llm.message_handler import MessageHandler
+from spaik_sdk.models.llm_config import LLMConfig
+from spaik_sdk.recording.base_playback import BasePlayback
+from spaik_sdk.recording.base_recorder import BaseRecorder
+from spaik_sdk.thread.models import MessageBlock, MessageBlockType, ThreadMessage
+from spaik_sdk.thread.thread_container import ThreadContainer
+from spaik_sdk.utils.init_logger import init_logger
+
+DEBUG = env_config.is_debug_mode("langchain")
+logger = init_logger(__name__)
+
+# Suppress noisy HTTP request logs from anthropic and httpx
+logging.getLogger("anthropic._base_client").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+if DEBUG:
+    from langchain_core.globals import set_debug
+
+    set_debug(True)
+
+config = RunnableConfig(recursion_limit=100)
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class LangChainService:
+    def __init__(
+        self,
+        llm_config: LLMConfig,
+        thread_container: ThreadContainer,
+        assistant_name: str,
+        assistant_id: str,
+        recorder: Optional[BaseRecorder] = None,
+        playback: Optional[BasePlayback] = None,
+        cancellation_handle: Optional[CancellationHandle] = None,
+    ):
+        self.llm_config = llm_config
+
+        self.thread_container = thread_container
+        self.message_handler = MessageHandler(self.thread_container, assistant_name, assistant_id, recorder)
+        self.is_used = False
+        self.recorder = recorder
+        self.playback = playback
+        self.cancellation_handle = cancellation_handle
+
+    def create_executor(self, tools: list[BaseTool]):
+        return create_react_agent(self._get_model(), tools)
+
+    def _get_model(self):
+        return self.llm_config.get_model_wrapper().get_langchain_model()
+
+    def get_structured_response(self, input: str, output_schema: Type[T]) -> T:
+        # Handle playback mode
+        if self.playback is not None:
+            ret = output_schema.model_validate(next(self.playback))
+            self._on_request_completed()
+            return ret
+
+        self.thread_container.add_message(
+            ThreadMessage(
+                id=str(uuid.uuid4()),
+                ai=False,
+                author_id="structured_response  ",
+                author_name="structured_response",
+                timestamp=int(time.time() * 1000),
+                blocks=[MessageBlock(id=str(uuid.uuid4()), streaming=False, type=MessageBlockType.PLAIN, content=input)],
+            )
+        )
+        model_with_tools = self._get_model().with_structured_output(output_schema)
+        ret = cast(T, model_with_tools.invoke(input))
+
+        # Record structured response if recorder is present
+        if self.recorder is not None:
+            self.recorder.record_structured(ret.model_dump())
+
+        as_json_block = "```json\n" + json.dumps(ret.model_dump()) + "\n```"
+        self.thread_container.add_message(
+            ThreadMessage(
+                id=str(uuid.uuid4()),
+                ai=True,
+                author_id=self.message_handler.assistant_id,
+                author_name=self.message_handler.assistant_name,
+                timestamp=int(time.time() * 1000),
+                blocks=[MessageBlock(id=str(uuid.uuid4()), streaming=False, type=MessageBlockType.PLAIN, content=as_json_block)],
+            )
+        )
+        self._on_request_completed()
+        return ret
+
+    async def execute_stream_tokens(
+        self,
+        user_input: Optional[str] = None,
+        tools: List[BaseTool] = [],
+        attachments: Optional[List[Attachment]] = None,
+    ):
+        """Execute agent and yield individual tokens as they arrive.
+
+        Gemini models have weird hickups regarding event loops and require a hack.
+
+        See documentation of LangChainLoopManager for more details.
+        """
+        if self.is_used:
+            raise ValueError("LangChainService is single use because of reasons")
+        self.is_used = True
+
+        try:
+            if should_use_loop_manager(self.llm_config):
+                logger.debug("Using loop manager for Google model in standalone context")
+                async for token_data in get_langchain_loop_manager().stream_in_loop(
+                    self._execute_stream_tokens_direct(user_input, tools, attachments)
+                ):
+                    yield token_data
+            else:
+                async for token_data in self._execute_stream_tokens_direct(user_input, tools, attachments):
+                    yield token_data
+
+        except Exception as e:
+            yield {"type": "error", "error": self._handle_error(e)}
+        finally:
+            self._on_request_completed()
+
+    async def _execute_stream_tokens_direct(
+        self,
+        user_input: Optional[str] = None,
+        tools: List[BaseTool] = [],
+        attachments: Optional[List[Attachment]] = None,
+    ):
+        """Direct execution of stream tokens (core logic)"""
+        if self.playback is not None:
+            # Playback mode - yield recorded tokens
+            async for token_data in self.message_handler.process_agent_token_stream(self.playback):
+                # Check for cancellation even in playback mode
+                if self.cancellation_handle and await self.cancellation_handle.is_cancelled():
+                    self.message_handler.handle_cancellation()
+                    return
+                yield token_data
+            return
+
+        agent = self.create_executor(tools)
+        if user_input is not None:
+            self.message_handler.add_user_message(user_input, "user", "user", attachments)
+
+        # Get messages - use multimodal converter if file_storage is available
+        file_storage = get_file_storage()
+        if file_storage is not None:
+            provider_family = self.llm_config.model.family
+            messages = await self.thread_container.get_langchain_messages_multimodal(file_storage, provider_family)
+        else:
+            messages = self.thread_container.get_langchain_messages()
+
+        # Use astream_events to get individual token events
+        agent_stream = agent.astream_events({"messages": messages}, version="v2", config=config)
+
+        # Let MessageHandler handle the token stream processing
+        async for token_data in self.message_handler.process_agent_token_stream(agent_stream):
+            if self.cancellation_handle and await self.cancellation_handle.is_cancelled():
+                logger.info("Cancellation detected, stopping stream")
+                self.message_handler.handle_cancellation()
+                return
+            yield token_data
+
+    def _handle_error(self, error: Exception) -> Dict[str, str]:
+        """Handle and format errors consistently."""
+        error_message = extract_error_message(error)
+        logger.error(f"Error executing agent: {error_message}")
+
+        # Add error to thread container
+        self.message_handler.add_error(error_message, "system")
+
+        return {"error": error_message}
+
+    def _on_request_completed(self):
+        if self.recorder is not None:
+            self.recorder.request_completed()