janito 3.12.1__py3-none-any.whl → 3.12.3__py3-none-any.whl

janito/llm/driver.py

@@ -1,254 +1,290 @@
-import threading
-from abc import ABC, abstractmethod
-from queue import Queue
-from janito.llm.driver_input import DriverInput
-from janito.driver_events import (
-    RequestStarted,
-    RequestFinished,
-    ResponseReceived,
-    RequestStatus,
-)
-
-
-class LLMDriver(ABC):
-    def clear_output_queue(self):
-        """Remove all items from the output queue."""
-        try:
-            while True:
-                self.output_queue.get_nowait()
-        except Exception:
-            pass
-
-    def clear_input_queue(self):
-        """Remove all items from the input queue."""
-        try:
-            while True:
-                self.input_queue.get_nowait()
-        except Exception:
-            pass
-
-    """
-    Abstract base class for LLM drivers (threaded, queue-based).
-    Subclasses must implement:
-      - _call_api: Call provider API with DriverInput.
-      - _convert_completion_message_to_parts: Convert provider message to MessagePart objects.
-      - convert_history_to_api_messages: Convert LLMConversationHistory to provider-specific messages format for API calls.
-    Workflow:
-      - Accept DriverInput via input_queue.
-      - Put DriverEvents on output_queue.
-      - Use start() to launch worker loop in a thread.
-    The driver automatically creates its own input/output queues, accessible via .input_queue and .output_queue.
-    """
-
-    available = True
-    unavailable_reason = None
-
-    def __init__(self, tools_adapter=None, provider_name=None):
-        self.input_queue = Queue()
-        self.output_queue = Queue()
-        self._thread = None
-        self.tools_adapter = tools_adapter
-        self.provider_name = provider_name
-
-    def start(self):
-        """Validate tool schemas (if any) and launch the driver's background thread to process DriverInput objects."""
-        # Validate all tool schemas before starting the thread
-        if self.tools_adapter is not None:
-            from janito.tools.tools_schema import ToolSchemaBase
-
-            validator = ToolSchemaBase()
-            for tool in self.tools_adapter.get_tools():
-                # Validate the tool's class (not instance)
-                validator.validate_tool_class(tool.__class__)
-        self._thread = threading.Thread(target=self._run, daemon=True)
-        self._thread.start()
-
-    def _run(self):
-        while True:
-            driver_input = self.input_queue.get()
-            if driver_input is None:
-                break  # Sentinel received, exit thread
-            try:
-                # Only process if driver_input is a DriverInput instance
-                if isinstance(driver_input, DriverInput):
-                    self.process_driver_input(driver_input)
-                else:
-                    # Optionally log or handle unexpected input types
-                    pass
-            except Exception as e:
-                import traceback
-
-                self.output_queue.put(
-                    RequestFinished(
-                        driver_name=self.__class__.__name__,
-                        request_id=getattr(driver_input.config, "request_id", None),
-                        status=RequestStatus.ERROR,
-                        error=str(e),
-                        exception=e,
-                        traceback=traceback.format_exc(),
-                    )
-                )
-
-    def handle_driver_unavailable(self, request_id):
-        self.output_queue.put(
-            RequestFinished(
-                driver_name=self.__class__.__name__,
-                request_id=request_id,
-                status=RequestStatus.ERROR,
-                error=self.unavailable_reason,
-                exception=ImportError(self.unavailable_reason),
-                traceback=None,
-            )
-        )
-
-    def emit_response_received(
-        self, driver_name, request_id, result, parts, timestamp=None, metadata=None
-    ):
-        self.output_queue.put(
-            ResponseReceived(
-                driver_name=driver_name,
-                request_id=request_id,
-                parts=parts,
-                tool_results=[],
-                timestamp=timestamp,
-                metadata=metadata or {},
-            )
-        )
-        # Debug: print summary of parts by type
-        if hasattr(self, "config") and getattr(self.config, "verbose_api", False):
-            from collections import Counter
-
-            type_counts = Counter(type(p).__name__ for p in parts)
-            print(
-                f"[verbose-api] Emitting ResponseReceived with parts: {dict(type_counts)}",
-                flush=True,
-            )
-
-    def process_driver_input(self, driver_input: DriverInput):
-
-        config = driver_input.config
-        request_id = getattr(config, "request_id", None)
-        if not self.available:
-            self.handle_driver_unavailable(request_id)
-            return
-        # Prepare payload for RequestStarted event
-        payload = {"provider_name": self.provider_name}
-        if hasattr(config, "model") and getattr(config, "model", None):
-            payload["model"] = getattr(config, "model")
-        elif hasattr(config, "model_name") and getattr(config, "model_name", None):
-            payload["model"] = getattr(config, "model_name")
-        self.output_queue.put(
-            RequestStarted(
-                driver_name=self.__class__.__name__,
-                request_id=request_id,
-                payload=payload,
-            )
-        )
-        # Check for cancel_event before starting
-        if (
-            hasattr(driver_input, "cancel_event")
-            and driver_input.cancel_event is not None
-            and driver_input.cancel_event.is_set()
-        ):
-            self.output_queue.put(
-                RequestFinished(
-                    driver_name=self.__class__.__name__,
-                    request_id=request_id,
-                    status=RequestStatus.CANCELLED,
-                    reason="Canceled before start",
-                )
-            )
-            return
-        try:
-            result = self._call_api(driver_input)
-            # If result is None and cancel_event is set, treat as cancelled
-            if (
-                hasattr(driver_input, "cancel_event")
-                and driver_input.cancel_event is not None
-                and driver_input.cancel_event.is_set()
-            ):
-                self.output_queue.put(
-                    RequestFinished(
-                        driver_name=self.__class__.__name__,
-                        request_id=request_id,
-                        status=RequestStatus.CANCELLED,
-                        reason="Cancelled during processing (post-API)",
-                    )
-                )
-                return
-            if (
-                result is None
-                and hasattr(driver_input, "cancel_event")
-                and driver_input.cancel_event is not None
-                and driver_input.cancel_event.is_set()
-            ):
-                # Already handled by driver
-                return
-            # Check for cancel_event after API call (subclasses should also check during long calls)
-            if (
-                hasattr(driver_input, "cancel_event")
-                and driver_input.cancel_event is not None
-                and driver_input.cancel_event.is_set()
-            ):
-                self.output_queue.put(
-                    RequestFinished(
-                        driver_name=self.__class__.__name__,
-                        request_id=request_id,
-                        status=RequestStatus.CANCELLED,
-                        reason="Canceled during processing",
-                    )
-                )
-                return
-            message = self._get_message_from_result(result)
-            parts = (
-                self._convert_completion_message_to_parts(message) if message else []
-            )
-            timestamp = getattr(result, "created", None)
-            metadata = {"usage": getattr(result, "usage", None), "raw_response": result}
-            self.emit_response_received(
-                self.__class__.__name__, request_id, result, parts, timestamp, metadata
-            )
-        except Exception as ex:
-            import traceback
-
-            self.output_queue.put(
-                RequestFinished(
-                    driver_name=self.__class__.__name__,
-                    request_id=request_id,
-                    status=RequestStatus.ERROR,
-                    error=str(ex),
-                    exception=ex,
-                    traceback=traceback.format_exc(),
-                )
-            )
-
-    @abstractmethod
-    def _prepare_api_kwargs(self, config, conversation):
-        """
-        Subclasses must implement: Prepare API kwargs for the provider, including any tool schemas if needed.
-        """
-        pass
-
-    @abstractmethod
-    def _call_api(self, driver_input: DriverInput):
-        """Subclasses implement: Use driver_input to call provider and return result object."""
-        pass
-
-    @abstractmethod
-    def _convert_completion_message_to_parts(self, message):
-        """Subclasses implement: Convert provider message to list of MessagePart objects."""
-        pass
-
-    @abstractmethod
-    def convert_history_to_api_messages(self, conversation_history):
-        """
-        Subclasses implement: Convert LLMConversationHistory to the messages object required by their provider API.
-        :param conversation_history: LLMConversationHistory instance
-        :return: Provider-specific messages object (e.g., list of dicts for OpenAI)
-        """
-        pass
-
-    @abstractmethod
-    def _get_message_from_result(self, result):
-        """Extract the message object from the provider result. Subclasses must implement this."""
-        raise NotImplementedError("Subclasses must implement _get_message_from_result.")
+import threading
+from abc import ABC, abstractmethod
+from queue import Queue
+from janito.llm.driver_input import DriverInput
+from janito.driver_events import (
+    RequestStarted,
+    RequestFinished,
+    ResponseReceived,
+    RequestStatus,
+)
+from janito.llm.response_cache import ResponseCache
+
+
+class LLMDriver(ABC):
+    def clear_output_queue(self):
+        """Remove all items from the output queue."""
+        try:
+            while True:
+                self.output_queue.get_nowait()
+        except Exception:
+            pass
+
+    def clear_input_queue(self):
+        """Remove all items from the input queue."""
+        try:
+            while True:
+                self.input_queue.get_nowait()
+        except Exception:
+            pass
+
+    """
+    Abstract base class for LLM drivers (threaded, queue-based).
+    Subclasses must implement:
+      - _call_api: Call provider API with DriverInput.
+      - _convert_completion_message_to_parts: Convert provider message to MessagePart objects.
+      - convert_history_to_api_messages: Convert LLMConversationHistory to provider-specific messages format for API calls.
+    Workflow:
+      - Accept DriverInput via input_queue.
+      - Put DriverEvents on output_queue.
+      - Use start() to launch worker loop in a thread.
+    The driver automatically creates its own input/output queues, accessible via .input_queue and .output_queue.
+    """
+
+    available = True
+    unavailable_reason = None
+
+    def __init__(self, tools_adapter=None, provider_name=None, enable_cache=True):
+        self.input_queue = Queue()
+        self.output_queue = Queue()
+        self._thread = None
+        self.tools_adapter = tools_adapter
+        self.provider_name = provider_name
+        self.enable_cache = enable_cache
+        self.response_cache = ResponseCache() if enable_cache else None
+
+    def start(self):
+        """Validate tool schemas (if any) and launch the driver's background thread to process DriverInput objects."""
+        # Validate all tool schemas before starting the thread
+        if self.tools_adapter is not None:
+            from janito.tools.tools_schema import ToolSchemaBase
+
+            validator = ToolSchemaBase()
+            for tool in self.tools_adapter.get_tools():
+                # Validate the tool's class (not instance)
+                validator.validate_tool_class(tool.__class__)
+        self._thread = threading.Thread(target=self._run, daemon=True)
+        self._thread.start()
+
+    def _run(self):
+        while True:
+            driver_input = self.input_queue.get()
+            if driver_input is None:
+                break  # Sentinel received, exit thread
+            try:
+                # Only process if driver_input is a DriverInput instance
+                if isinstance(driver_input, DriverInput):
+                    self.process_driver_input(driver_input)
+                else:
+                    # Optionally log or handle unexpected input types
+                    pass
+            except Exception as e:
+                import traceback
+
+                self.output_queue.put(
+                    RequestFinished(
+                        driver_name=self.__class__.__name__,
+                        request_id=getattr(driver_input.config, "request_id", None),
+                        status=RequestStatus.ERROR,
+                        error=str(e),
+                        exception=e,
+                        traceback=traceback.format_exc(),
+                    )
+                )
+
+    def handle_driver_unavailable(self, request_id):
+        self.output_queue.put(
+            RequestFinished(
+                driver_name=self.__class__.__name__,
+                request_id=request_id,
+                status=RequestStatus.ERROR,
+                error=self.unavailable_reason,
+                exception=ImportError(self.unavailable_reason),
+                traceback=None,
+            )
+        )
+
+    def emit_response_received(
+        self, driver_name, request_id, result, parts, timestamp=None, metadata=None
+    ):
+        self.output_queue.put(
+            ResponseReceived(
+                driver_name=driver_name,
+                request_id=request_id,
+                parts=parts,
+                tool_results=[],
+                timestamp=timestamp,
+                metadata=metadata or {},
+            )
+        )
+        # Debug: print summary of parts by type
+        if hasattr(self, "config") and getattr(self.config, "verbose_api", False):
+            from collections import Counter
+
+            type_counts = Counter(type(p).__name__ for p in parts)
+            print(
+                f"[verbose-api] Emitting ResponseReceived with parts: {dict(type_counts)}",
+                flush=True,
+            )
+
+    def process_driver_input(self, driver_input: DriverInput):
+
+        config = driver_input.config
+        request_id = getattr(config, "request_id", None)
+        if not self.available:
+            self.handle_driver_unavailable(request_id)
+            return
+        
+        # Check cache first if enabled
+        if self.response_cache:
+            cached_response = self.response_cache.get(driver_input)
+            if cached_response is not None:
+                # Use cached response
+                message = self._get_message_from_result(cached_response)
+                parts = (
+                    self._convert_completion_message_to_parts(message) if message else []
+                )
+                timestamp = getattr(cached_response, "created", None)
+                metadata = {"usage": getattr(cached_response, "usage", None), "raw_response": cached_response, "cached": True}
+                self.emit_response_received(
+                    self.__class__.__name__, request_id, cached_response, parts, timestamp, metadata
+                )
+                return
+        
+        # Prepare payload for RequestStarted event
+        payload = {"provider_name": self.provider_name}
+        if hasattr(config, "model") and getattr(config, "model", None):
+            payload["model"] = getattr(config, "model")
+        elif hasattr(config, "model_name") and getattr(config, "model_name", None):
+            payload["model"] = getattr(config, "model_name")
+        self.output_queue.put(
+            RequestStarted(
+                driver_name=self.__class__.__name__,
+                request_id=request_id,
+                payload=payload,
+            )
+        )
+        # Check for cancel_event before starting
+        if (
+            hasattr(driver_input, "cancel_event")
+            and driver_input.cancel_event is not None
+            and driver_input.cancel_event.is_set()
+        ):
+            self.output_queue.put(
+                RequestFinished(
+                    driver_name=self.__class__.__name__,
+                    request_id=request_id,
+                    status=RequestStatus.CANCELLED,
+                    reason="Canceled before start",
+                )
+            )
+            return
+        try:
+            result = self._call_api(driver_input)
+            # If result is None and cancel_event is set, treat as cancelled
+            if (
+                hasattr(driver_input, "cancel_event")
+                and driver_input.cancel_event is not None
+                and driver_input.cancel_event.is_set()
+            ):
+                self.output_queue.put(
+                    RequestFinished(
+                        driver_name=self.__class__.__name__,
+                        request_id=request_id,
+                        status=RequestStatus.CANCELLED,
+                        reason="Cancelled during processing (post-API)",
+                    )
+                )
+                return
+            if (
+                result is None
+                and hasattr(driver_input, "cancel_event")
+                and driver_input.cancel_event is not None
+                and driver_input.cancel_event.is_set()
+            ):
+                # Already handled by driver
+                return
+            # Check for cancel_event after API call (subclasses should also check during long calls)
+            if (
+                hasattr(driver_input, "cancel_event")
+                and driver_input.cancel_event is not None
+                and driver_input.cancel_event.is_set()
+            ):
+                self.output_queue.put(
+                    RequestFinished(
+                        driver_name=self.__class__.__name__,
+                        request_id=request_id,
+                        status=RequestStatus.CANCELLED,
+                        reason="Canceled during processing",
+                    )
+                )
+                return
+            message = self._get_message_from_result(result)
+            parts = (
+                self._convert_completion_message_to_parts(message) if message else []
+            )
+            timestamp = getattr(result, "created", None)
+            metadata = {"usage": getattr(result, "usage", None), "raw_response": result}
+            
+            # Cache the response if caching is enabled
+            if self.response_cache:
+                self.response_cache.set(driver_input, result)
+            
+            self.emit_response_received(
+                self.__class__.__name__, request_id, result, parts, timestamp, metadata
+            )
+        except Exception as ex:
+            import traceback
+
+            self.output_queue.put(
+                RequestFinished(
+                    driver_name=self.__class__.__name__,
+                    request_id=request_id,
+                    status=RequestStatus.ERROR,
+                    error=str(ex),
+                    exception=ex,
+                    traceback=traceback.format_exc(),
+                )
+            )
+
+    def clear_cache(self):
+        """Clear the response cache if caching is enabled."""
+        if self.response_cache:
+            self.response_cache.clear()
+    
+    def get_cache_stats(self):
+        """Get cache statistics if caching is enabled."""
+        if self.response_cache:
+            return self.response_cache.get_stats()
+        return {"total_entries": 0, "total_size": 0}
+
+    @abstractmethod
+    def _prepare_api_kwargs(self, config, conversation):
+        """
+        Subclasses must implement: Prepare API kwargs for the provider, including any tool schemas if needed.
+        """
+        pass
+
+    @abstractmethod
+    def _call_api(self, driver_input: DriverInput):
+        """Subclasses implement: Use driver_input to call provider and return result object."""
+        pass
+
+    @abstractmethod
+    def _convert_completion_message_to_parts(self, message):
+        """Subclasses implement: Convert provider message to list of MessagePart objects."""
+        pass
+
+    @abstractmethod
+    def convert_history_to_api_messages(self, conversation_history):
+        """
+        Subclasses implement: Convert LLMConversationHistory to the messages object required by their provider API.
+        :param conversation_history: LLMConversationHistory instance
+        :return: Provider-specific messages object (e.g., list of dicts for OpenAI)
+        """
+        pass
+
+    @abstractmethod
+    def _get_message_from_result(self, result):
+        """Extract the message object from the provider result. Subclasses must implement this."""
+        raise NotImplementedError("Subclasses must implement _get_message_from_result.")

janito/llm/response_cache.py

@@ -0,0 +1,57 @@
+"""
+Simple in-memory cache for LLM responses based on input hash.
+No expiration - cache lives for the duration of the process.
+"""
+
+import hashlib
+import json
+from typing import Any, Dict, Optional
+from janito.llm.driver_input import DriverInput
+
+
+class ResponseCache:
+    """Simple in-memory cache for LLM responses with no expiration."""
+    
+    def __init__(self):
+        self._cache: Dict[str, Any] = {}
+    
+    def _generate_key(self, driver_input: DriverInput) -> str:
+        """Generate a cache key from driver input."""
+        # Create a deterministic representation of the input
+        cache_data = {
+            "conversation_history": driver_input.conversation_history.get_history(),
+            "config": {
+                "model": getattr(driver_input.config, "model", None),
+                "temperature": getattr(driver_input.config, "temperature", None),
+                "max_tokens": getattr(driver_input.config, "max_tokens", None),
+                "top_p": getattr(driver_input.config, "top_p", None),
+                "presence_penalty": getattr(driver_input.config, "presence_penalty", None),
+                "frequency_penalty": getattr(driver_input.config, "frequency_penalty", None),
+                "stop": getattr(driver_input.config, "stop", None),
+            }
+        }
+        
+        # Create hash from JSON representation
+        cache_str = json.dumps(cache_data, sort_keys=True, separators=(',', ':'))
+        return hashlib.sha256(cache_str.encode('utf-8')).hexdigest()
+    
+    def get(self, driver_input: DriverInput) -> Optional[Any]:
+        """Get cached response for the given input."""
+        key = self._generate_key(driver_input)
+        return self._cache.get(key)
+    
+    def set(self, driver_input: DriverInput, response: Any) -> None:
+        """Cache the response for the given input."""
+        key = self._generate_key(driver_input)
+        self._cache[key] = response
+    
+    def clear(self) -> None:
+        """Clear all cached responses."""
+        self._cache.clear()
+    
+    def get_stats(self) -> Dict[str, int]:
+        """Get cache statistics."""
+        return {
+            "total_entries": len(self._cache),
+            "total_size": sum(len(str(v)) for v in self._cache.values())
+        }