lollms-client 0.24.2__py3-none-any.whl → 0.27.0__py3-none-any.whl

lollms_client/llm_bindings/litellm/__init__.py

@@ -0,0 +1,201 @@
+# bindings/LiteLLM/binding.py
+import requests
+import json
+from lollms_client.lollms_llm_binding import LollmsLLMBinding
+from lollms_client.lollms_types import MSG_TYPE
+from lollms_client.lollms_discussion import LollmsDiscussion
+from lollms_client.lollms_utilities import encode_image
+from typing import Optional, Callable, List, Union, Dict
+from ascii_colors import ASCIIColors, trace_exception
+
+# Use pipmaster to ensure required packages are installed
+try:
+    import pipmaster as pm
+except ImportError:
+    print("Pipmaster not found. Please install it using 'pip install pipmaster'")
+    raise
+
+# Ensure requests and tiktoken are installed
+pm.ensure_packages(["requests", "tiktoken"])
+
+import tiktoken
+
+BindingName = "LiteLLMBinding"
+
+def get_icon_path(model_name: str) -> str:
+    model_name = model_name.lower()
+    if 'gpt' in model_name: return '/bindings/openai/logo.png'
+    if 'mistral' in model_name or 'mixtral' in model_name: return '/bindings/mistral/logo.png'
+    if 'claude' in model_name: return '/bindings/anthropic/logo.png'
+    return '/bindings/litellm/logo.png'
+
+class LiteLLMBinding(LollmsLLMBinding):
+    """
+    A binding for the LiteLLM proxy using direct HTTP requests.
+    This version includes detailed logging, a fallback for listing models,
+    and correct payload formatting for both streaming and non-streaming modes.
+    """
+    
+    def __init__(self, host_address: str, model_name: str, service_key: str = "anything", verify_ssl_certificate: bool = True, **kwargs):
+        super().__init__(binding_name="litellm")
+        self.host_address = host_address.rstrip('/')
+        self.model_name = model_name
+        self.service_key = service_key
+        self.verify_ssl_certificate = verify_ssl_certificate
+
+    def _perform_generation(self, messages: List[Dict], n_predict: Optional[int], stream: bool, temperature: float, top_p: float, repeat_penalty: float, seed: Optional[int], streaming_callback: Optional[Callable[[str, MSG_TYPE], None]]) -> Union[str, dict]:
+        url = f'{self.host_address}/v1/chat/completions'
+        headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.service_key}'}
+        payload = {
+            "model": self.model_name, "messages": messages, "max_tokens": n_predict,
+            "temperature": temperature, "top_p": top_p, "frequency_penalty": repeat_penalty,
+            "stream": stream
+        }
+        if seed is not None: payload["seed"] = seed
+        
+        payload = {k: v for k, v in payload.items() if v is not None}
+        output = ""
+        try:
+            response = requests.post(url, headers=headers, data=json.dumps(payload), stream=stream, verify=self.verify_ssl_certificate)
+            response.raise_for_status()
+
+            if stream:
+                for line in response.iter_lines():
+                    if line:
+                        decoded_line = line.decode('utf-8')
+                        if decoded_line.startswith('data: '):
+                            if '[DONE]' in decoded_line: break
+                            json_data_string = decoded_line[6:]
+                            try:
+                                chunk_data = json.loads(json_data_string)
+                                delta = chunk_data.get('choices', [{}])[0].get('delta', {})
+                                if 'content' in delta and delta['content'] is not None:
+                                    word = delta['content']
+                                    if streaming_callback and not streaming_callback(word, MSG_TYPE.MSG_TYPE_CHUNK):
+                                        return output
+                                    output += word
+                            except json.JSONDecodeError: continue
+            else:
+                full_response = response.json()
+                output = full_response['choices'][0]['message']['content']
+                if streaming_callback:
+                    streaming_callback(output, MSG_TYPE.MSG_TYPE_CHUNK)
+        except Exception as e:
+            error_message = f"An error occurred: {e}\nResponse: {response.text if 'response' in locals() else 'No response'}"
+            trace_exception(e)
+            if streaming_callback: streaming_callback(error_message, MSG_TYPE.MSG_TYPE_EXCEPTION)
+            return {"status": "error", "message": error_message}
+        return output
+
+    def generate_text(self, prompt: str, images: Optional[List[str]] = None, system_prompt: str = "", n_predict: Optional[int] = None, stream: Optional[bool] = None, temperature: float = 0.7, top_p: float = 0.9, repeat_penalty: float = 1.1, seed: Optional[int] = None, streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None, **kwargs) -> Union[str, dict]:
+        """Generates text from a prompt, correctly formatting for text-only and multi-modal cases."""
+        is_streaming = stream if stream is not None else (streaming_callback is not None)
+        
+        messages = []
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+        
+        # --- THIS IS THE CRITICAL FIX ---
+        if images:
+            # If images are present, use the multi-modal list format for content
+            user_content = [{"type": "text", "text": prompt}]
+            for image_path in images:
+                base64_image = encode_image(image_path)
+                user_content.append({"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"}})
+            messages.append({"role": "user", "content": user_content})
+        else:
+            # If no images, use a simple string for content to avoid the API error
+            messages.append({"role": "user", "content": prompt})
+        # --- END OF FIX ---
+
+        return self._perform_generation(messages, n_predict, is_streaming, temperature, top_p, repeat_penalty, seed, streaming_callback)
+
+    def chat(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None, n_predict: Optional[int] = None, stream: Optional[bool] = None, temperature: float = 0.7, top_p: float = 0.9, repeat_penalty: float = 1.1, seed: Optional[int] = None, streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None, **kwargs) -> Union[str, dict]:
+        is_streaming = stream if stream is not None else (streaming_callback is not None)
+        messages = discussion.export("openai_chat", branch_tip_id)
+        return self._perform_generation(messages, n_predict, is_streaming, temperature, top_p, repeat_penalty, seed, streaming_callback)
+
+    def embed(self, text: str, **kwargs) -> List[float]:
+        url = f'{self.host_address}/v1/embeddings'
+        headers = {'Content-Type': 'application/json', 'Authorization': f'Bearer {self.service_key}'}
+        payload = {"model": self.model_name, "input": text}
+        try:
+            response = requests.post(url, headers=headers, data=json.dumps(payload), verify=self.verify_ssl_certificate)
+            response.raise_for_status()
+            return response.json()['data'][0]['embedding']
+        except Exception as e:
+            trace_exception(e)
+            return []
+
+    def tokenize(self, text: str) -> list:
+        return tiktoken.model.encoding_for_model("gpt-3.5-turbo").encode(text)
+            
+    def detokenize(self, tokens: list) -> str:
+        return tiktoken.model.encoding_for_model("gpt-3.5-turbo").decode(tokens)
+
+    def count_tokens(self, text: str) -> int:
+        return len(self.tokenize(text))
+    
+    def _list_models_openai_fallback(self) -> List[Dict]:
+        ASCIIColors.warning("--- [LiteLLM Binding] Falling back to /v1/models endpoint. Rich metadata will be unavailable.")
+        url = f'{self.host_address}/v1/models'
+        headers = {'Authorization': f'Bearer {self.service_key}'}
+        entries = []
+        try:
+            response = requests.get(url, headers=headers, verify=self.verify_ssl_certificate)
+            response.raise_for_status()
+            models_data = response.json().get('data', [])
+            for model in models_data:
+                model_name = model.get('id')
+                entries.append({
+                    "category": "api", "datasets": "unknown", "icon": get_icon_path(model_name),
+                    "license": "unknown", "model_creator": model.get('owned_by', 'unknown'),
+                    "name": model_name, "provider": "litellm", "rank": "1.0", "type": "api",
+                    "variants": [{"name": model_name, "size": -1}]
+                })
+        except Exception as e:
+            ASCIIColors.error(f"--- [LiteLLM Binding] Fallback method failed: {e}")
+        return entries
+
+    def listModels(self) -> List[Dict]:
+        url = f'{self.host_address}/model/info'
+        headers = {'Authorization': f'Bearer {self.service_key}'}
+        entries = []
+        ASCIIColors.yellow(f"--- [LiteLLM Binding] Attempting to list models from: {url}")
+        try:
+            response = requests.get(url, headers=headers, verify=self.verify_ssl_certificate)
+            if response.status_code == 404:
+                ASCIIColors.warning("--- [LiteLLM Binding] /model/info endpoint not found (404).")
+                return self._list_models_openai_fallback()
+            response.raise_for_status()
+            models_data = response.json().get('data', [])
+            ASCIIColors.info(f"--- [LiteLLM Binding] Successfully parsed {len(models_data)} models from primary endpoint.")
+            for model in models_data:
+                model_name = model.get('model_name')
+                if not model_name: continue
+                model_info = model.get('model_info', {})
+                context_size = model_info.get('max_tokens', model_info.get('max_input_tokens', 4096))
+                entries.append({
+                    "category": "api", "datasets": "unknown", "icon": get_icon_path(model_name),
+                    "license": "unknown", "model_creator": model_info.get('owned_by', 'unknown'),
+                    "name": model_name, "provider": "litellm", "rank": "1.0", "type": "api",
+                    "variants": [{
+                        "name": model_name, "size": context_size,
+                        "input_cost_per_token": model_info.get('input_cost_per_token', 0),
+                        "output_cost_per_token": model_info.get('output_cost_per_token', 0),
+                        "max_output_tokens": model_info.get('max_output_tokens', 0),
+                    }]
+                })
+        except requests.exceptions.RequestException as e:
+            ASCIIColors.error(f"--- [LiteLLM Binding] Network error when trying to list models: {e}")
+            if "404" in str(e): return self._list_models_openai_fallback()
+        except Exception as e:
+            ASCIIColors.error(f"--- [LiteLLM Binding] An unexpected error occurred while listing models: {e}")
+        return entries
+
+    def get_model_info(self) -> dict:
+        return {"name": "LiteLLM", "host_address": self.host_address, "model_name": self.model_name}
+
+    def load_model(self, model_name: str) -> bool:
+        self.model_name = model_name
+        return True

lollms_client/llm_bindings/lollms/__init__.py

@@ -4,6 +4,7 @@ from lollms_client.lollms_llm_binding import LollmsLLMBinding
 from lollms_client.lollms_types import MSG_TYPE
 from lollms_client.lollms_utilities import encode_image
 from lollms_client.lollms_types import ELF_COMPLETION_FORMAT
+from lollms_client.lollms_discussion import LollmsDiscussion
 from ascii_colors import ASCIIColors, trace_exception
 from typing import Optional, Callable, List, Union
 import json
@@ -280,6 +281,7 @@ class LollmsLLMBinding(LollmsLLMBinding):
         Returns:
             list: List of tokens.
         """
+        response=None
         try:
             # Prepare the request payload
             payload = {

lollms_client/llm_bindings/mistral/__init__.py

@@ -0,0 +1,298 @@
+import os
+from typing import Optional, Callable, List, Union, Dict
+
+from lollms_client.lollms_discussion import LollmsDiscussion, LollmsMessage
+from lollms_client.lollms_llm_binding import LollmsLLMBinding
+from lollms_client.lollms_types import MSG_TYPE
+from ascii_colors import ASCIIColors, trace_exception
+
+import pipmaster as pm
+
+# Ensure the required packages are installed
+pm.ensure_packages(["mistralai", "pillow", "tiktoken"])
+
+from mistralai.client import MistralClient
+from mistralai.models.chat_completion import ChatMessage
+from PIL import Image, ImageDraw
+import tiktoken
+
+BindingName = "MistralBinding"
+
+class MistralBinding(LollmsLLMBinding):
+    """
+    Mistral AI API binding implementation.
+
+    This binding allows communication with Mistral's API for both their
+    open-weight and proprietary models.
+    """
+
+    def __init__(self,
+                 model_name: str = "mistral-large-latest",
+                 mistral_api_key: str = None,
+                 **kwargs
+                 ):
+        """
+        Initialize the MistralBinding.
+
+        Args:
+            model_name (str): The name of the Mistral model to use.
+            mistral_api_key (str): The API key for the Mistral service.
+        """
+        super().__init__(binding_name=BindingName)
+        self.model_name = model_name
+        self.mistral_api_key = mistral_api_key or os.getenv("MISTRAL_API_KEY")
+
+        if not self.mistral_api_key:
+            raise ValueError("Mistral API key is required. Set it via 'mistral_api_key' or MISTRAL_API_KEY env var.")
+
+        try:
+            self.client = MistralClient(api_key=self.mistral_api_key)
+        except Exception as e:
+            ASCIIColors.error(f"Failed to configure Mistral client: {e}")
+            self.client = None
+            raise ConnectionError(f"Could not configure Mistral client: {e}") from e
+
+    def _construct_parameters(self,
+                              temperature: float,
+                              top_p: float,
+                              n_predict: int,
+                              seed: Optional[int]) -> Dict[str, any]:
+        """Builds a parameters dictionary for the Mistral API."""
+        params = {}
+        if temperature is not None: params['temperature'] = float(temperature)
+        if top_p is not None: params['top_p'] = top_p
+        if n_predict is not None: params['max_tokens'] = n_predict
+        if seed is not None: params['random_seed'] = seed # Mistral uses 'random_seed'
+        return params
+
+    def _prepare_messages(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None) -> List[ChatMessage]:
+        """Prepares the message list for the Mistral API from a LollmsDiscussion."""
+        history = []
+        if discussion.system_prompt:
+            # Mistral prefers the system prompt as the first message with a user/assistant turn.
+            # A lone system message is not ideal. We will prepend it to the first user message.
+            # However, for API consistency, we will treat it as a separate message if it exists.
+            # The official client will likely handle this.
+            history.append(ChatMessage(role="system", content=discussion.system_prompt))
+
+        for msg in discussion.get_messages(branch_tip_id):
+            role = 'user' if msg.sender_type == "user" else 'assistant'
+            # Note: Mistral API currently does not support image inputs via the chat endpoint.
+            if msg.content:
+                history.append(ChatMessage(role=role, content=msg.content))
+        return history
+
+    def generate_text(self, prompt: str, **kwargs) -> Union[str, dict]:
+        """
+        Generate text using Mistral. This is a wrapper around the chat method.
+        """
+        temp_discussion = LollmsDiscussion.from_messages([
+            LollmsMessage.new_message(sender_type="user", content=prompt)
+        ])
+        if kwargs.get("system_prompt"):
+            temp_discussion.system_prompt = kwargs.get("system_prompt")
+        
+        return self.chat(temp_discussion, **kwargs)
+
+    def chat(self,
+             discussion: LollmsDiscussion,
+             branch_tip_id: Optional[str] = None,
+             n_predict: Optional[int] = 2048,
+             stream: Optional[bool] = False,
+             temperature: float = 0.7,
+             top_p: float = 0.9,
+             seed: Optional[int] = None,
+             streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+             **kwargs
+             ) -> Union[str, dict]:
+        """
+        Conduct a chat session with a Mistral model.
+        """
+        if not self.client:
+            return {"status": "error", "message": "Mistral client not initialized."}
+
+        messages = self._prepare_messages(discussion, branch_tip_id)
+        api_params = self._construct_parameters(temperature, top_p, n_predict, seed)
+        full_response_text = ""
+
+        try:
+            if stream:
+                response = self.client.chat_stream(
+                    model=self.model_name,
+                    messages=messages,
+                    **api_params
+                )
+                for chunk in response:
+                    delta = chunk.choices[0].delta.content
+                    if delta:
+                        full_response_text += delta
+                        if streaming_callback:
+                            if not streaming_callback(delta, MSG_TYPE.MSG_TYPE_CHUNK):
+                                break
+                return full_response_text
+            else:
+                response = self.client.chat(
+                    model=self.model_name,
+                    messages=messages,
+                    **api_params
+                )
+                return response.choices[0].message.content
+
+        except Exception as ex:
+            error_message = f"An unexpected error occurred with Mistral API: {str(ex)}"
+            trace_exception(ex)
+            return {"status": "error", "message": error_message}
+
+    def tokenize(self, text: str) -> list:
+        """Tokenize text using tiktoken as a fallback."""
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.encode(text)
+        except Exception:
+            return list(text.encode('utf-8'))
+
+    def detokenize(self, tokens: list) -> str:
+        """Detokenize tokens using tiktoken."""
+        try:
+            encoding = tiktoken.get_encoding("cl100k_base")
+            return encoding.decode(tokens)
+        except Exception:
+            return bytes(tokens).decode('utf-8', errors='ignore')
+
+    def count_tokens(self, text: str) -> int:
+        """Count tokens in a text using the fallback tokenizer."""
+        return len(self.tokenize(text))
+
+    def embed(self, text: str, **kwargs) -> List[float]:
+        """
+        Get embeddings for the input text using the Mistral embedding API.
+        """
+        if not self.client:
+            raise Exception("Mistral client not initialized.")
+        
+        # Default to the recommended embedding model
+        model_to_use = kwargs.get("model", "mistral-embed")
+
+        try:
+            response = self.client.embeddings(
+                model=model_to_use,
+                input=[text] # API expects a list of strings
+            )
+            return response.data[0].embedding
+        except Exception as ex:
+            trace_exception(ex)
+            raise Exception(f"Mistral embedding failed: {str(ex)}") from ex
+
+    def get_model_info(self) -> dict:
+        """Return information about the current Mistral setup."""
+        return {
+            "name": self.binding_name,
+            "version": "unknown", # mistralai library doesn't expose a version attribute easily
+            "host_address": "https://api.mistral.ai",
+            "model_name": self.model_name,
+            "supports_structured_output": False,
+            "supports_vision": False, # Mistral API does not currently support vision
+        }
+
+    def listModels(self) -> List[Dict[str, str]]:
+        """Lists available models from the Mistral service."""
+        if not self.client:
+            ASCIIColors.error("Mistral client not initialized. Cannot list models.")
+            return []
+        try:
+            ASCIIColors.debug("Listing Mistral models...")
+            models = self.client.list_models()
+            model_info_list = []
+            for m in models.data:
+                model_info_list.append({
+                    'model_name': m.id,
+                    'display_name': m.id.replace('-', ' ').title(),
+                    'description': f"Owned by: {m.owned_by}",
+                    'owned_by': m.owned_by
+                })
+            return model_info_list
+        except Exception as ex:
+            trace_exception(ex)
+            return []
+
+    def load_model(self, model_name: str) -> bool:
+        """Sets the model name for subsequent operations."""
+        self.model_name = model_name
+        ASCIIColors.info(f"Mistral model set to: {model_name}. It will be used on the next API call.")
+        return True
+
+if __name__ == '__main__':
+    # Environment variable to set for testing:
+    # MISTRAL_API_KEY: Your Mistral API key
+
+    if "MISTRAL_API_KEY" not in os.environ:
+        ASCIIColors.red("Error: MISTRAL_API_KEY environment variable not set.")
+        print("Please get your key from https://console.mistral.ai/api-keys/ and set it.")
+        exit(1)
+
+    ASCIIColors.yellow("--- Testing MistralBinding ---")
+
+    test_model_name = "mistral-small-latest" # Use a smaller, faster model for testing
+    test_embedding_model = "mistral-embed"
+    
+    try:
+        # --- Initialization ---
+        ASCIIColors.cyan("\n--- Initializing Binding ---")
+        binding = MistralBinding(model_name=test_model_name)
+        ASCIIColors.green("Binding initialized successfully.")
+
+        # --- List Models ---
+        ASCIIColors.cyan("\n--- Listing Models ---")
+        models = binding.listModels()
+        if models:
+            ASCIIColors.green(f"Found {len(models)} models on Mistral. Available models:")
+            for m in models:
+                print(f"- {m['model_name']}")
+        else:
+            ASCIIColors.warning("No models found or failed to list models.")
+        
+        # --- Text Generation (Non-Streaming) ---
+        ASCIIColors.cyan("\n--- Text Generation (Non-Streaming) ---")
+        prompt_text = "Who developed the transformer architecture and in what paper?"
+        generated_text = binding.generate_text(prompt_text, n_predict=100, stream=False)
+        if isinstance(generated_text, str):
+            ASCIIColors.green(f"Generated text:\n{generated_text}")
+        else:
+            ASCIIColors.error(f"Generation failed: {generated_text}")
+
+        # --- Text Generation (Streaming) ---
+        ASCIIColors.cyan("\n--- Text Generation (Streaming) ---")
+        full_streamed_text = ""
+        def stream_callback(chunk: str, msg_type: int):
+            nonlocal full_streamed_text
+            ASCIIColors.green(chunk, end="", flush=True)
+            full_streamed_text += chunk
+            return True
+        
+        result = binding.generate_text(prompt_text, n_predict=150, stream=True, streaming_callback=stream_callback)
+        print("\n--- End of Stream ---")
+        ASCIIColors.green(f"Full streamed text (for verification): {result}")
+
+        # --- Embeddings Test ---
+        ASCIIColors.cyan("\n--- Embeddings ---")
+        try:
+            embedding_text = "Mistral AI is based in Paris."
+            embedding_vector = binding.embed(embedding_text, model=test_embedding_model)
+            ASCIIColors.green(f"Embedding for '{embedding_text}' (first 5 dims): {embedding_vector[:5]}...")
+            ASCIIColors.info(f"Embedding vector dimension: {len(embedding_vector)}")
+        except Exception as e:
+            ASCIIColors.error(f"Embedding test failed: {e}")
+
+        # --- Vision Test (should be unsupported) ---
+        ASCIIColors.cyan("\n--- Vision Test (Expecting No Support) ---")
+        model_info = binding.get_model_info()
+        if not model_info.get("supports_vision"):
+            ASCIIColors.green("Binding correctly reports no support for vision.")
+        else:
+            ASCIIColors.warning("Binding reports support for vision, which is unexpected for Mistral.")
+
+    except Exception as e:
+        ASCIIColors.error(f"An error occurred during testing: {e}")
+        trace_exception(e)
+
+    ASCIIColors.yellow("\nMistralBinding test finished.")