lollms-client 0.32.0__py3-none-any.whl → 0.33.0__py3-none-any.whl

lollms_client/llm_bindings/mistral/__init__.py

@@ -11,11 +11,11 @@ import pipmaster as pm
 # Ensure the required packages are installed
 pm.ensure_packages(["mistralai", "pillow", "tiktoken"])
 
-from mistralai.client import MistralClient
-from mistralai.models.chat_completion import ChatMessage
+from mistralai import Mistral
 from PIL import Image, ImageDraw
 import tiktoken
 
+
 BindingName = "MistralBinding"
 
 class MistralBinding(LollmsLLMBinding):
@@ -28,7 +28,7 @@ class MistralBinding(LollmsLLMBinding):
 
     def __init__(self,
                  model_name: str = "mistral-large-latest",
-                 mistral_api_key: str = None,
+                 service_key: str|None = None,
                  **kwargs
                  ):
         """
@@ -40,13 +40,13 @@ class MistralBinding(LollmsLLMBinding):
         """
         super().__init__(binding_name=BindingName)
         self.model_name = model_name
-        self.mistral_api_key = mistral_api_key or os.getenv("MISTRAL_API_KEY")
+        self.mistral_api_key = service_key or os.getenv("MISTRAL_API_KEY")
 
         if not self.mistral_api_key:
             raise ValueError("Mistral API key is required. Set it via 'mistral_api_key' or MISTRAL_API_KEY env var.")
 
         try:
-            self.client = MistralClient(api_key=self.mistral_api_key)
+            self.client = Mistral(api_key=self.mistral_api_key)
         except Exception as e:
             ASCIIColors.error(f"Failed to configure Mistral client: {e}")
             self.client = None
@@ -64,35 +64,67 @@ class MistralBinding(LollmsLLMBinding):
         if n_predict is not None: params['max_tokens'] = n_predict
         if seed is not None: params['random_seed'] = seed # Mistral uses 'random_seed'
         return params
+    
 
-    def _prepare_messages(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None) -> List[ChatMessage]:
-        """Prepares the message list for the Mistral API from a LollmsDiscussion."""
+    def _prepare_messages(self, discussion: LollmsDiscussion, branch_tip_id: Optional[str] = None) -> List[Dict[str, any]]:
+        """Prepares the message list for the API from a LollmsDiscussion."""
         history = []
         if discussion.system_prompt:
-            # Mistral prefers the system prompt as the first message with a user/assistant turn.
-            # A lone system message is not ideal. We will prepend it to the first user message.
-            # However, for API consistency, we will treat it as a separate message if it exists.
-            # The official client will likely handle this.
-            history.append(ChatMessage(role="system", content=discussion.system_prompt))
+            history.append({"role": "system", "content": discussion.system_prompt})
 
         for msg in discussion.get_messages(branch_tip_id):
             role = 'user' if msg.sender_type == "user" else 'assistant'
-            # Note: Mistral API currently does not support image inputs via the chat endpoint.
+            # Note: Vision support depends on the specific model being called via OpenRouter.
+            # We will not implement it in this generic binding to avoid complexity,
+            # as different models might expect different formats.
             if msg.content:
-                history.append(ChatMessage(role=role, content=msg.content))
+                history.append({'role': role, 'content': msg.content})
         return history
 
-    def generate_text(self, prompt: str, **kwargs) -> Union[str, dict]:
+    def generate_text(self,
+                    prompt: str,
+                    images: Optional[List[str]] = None,
+                    system_prompt: str = "",
+                    n_predict: Optional[int] = None,
+                    stream: Optional[bool] = None,
+                    temperature: float = 0.7,  # Ollama default is 0.8, common default 0.7
+                    top_k: int = 40,          # Ollama default is 40
+                    top_p: float = 0.9,       # Ollama default is 0.9
+                    repeat_penalty: float = 1.1,  # Ollama default is 1.1
+                    repeat_last_n: int = 64,  # Ollama default is 64
+                    seed: Optional[int] = None,
+                    n_threads: Optional[int] = None,
+                    ctx_size: int | None = None,
+                    streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+                    split: Optional[bool] = False,  # put to true if the prompt is a discussion
+                    user_keyword: Optional[str] = "!@>user:",
+                    ai_keyword: Optional[str] = "!@>assistant:",
+                    **kwargs
+                    ) -> Union[str, dict]:
         """
-        Generate text using Mistral. This is a wrapper around the chat method.
+        Generate text using OpenRouter. This is a wrapper around the chat method.
         """
-        temp_discussion = LollmsDiscussion.from_messages([
-            LollmsMessage.new_message(sender_type="user", content=prompt)
-        ])
-        if kwargs.get("system_prompt"):
-            temp_discussion.system_prompt = kwargs.get("system_prompt")
+        temp_discussion = LollmsDiscussion(None)
+        temp_discussion.add_message(sender="user", content=prompt, images=images or [])
+        if system_prompt:
+            temp_discussion.system_prompt = system_prompt
         
-        return self.chat(temp_discussion, **kwargs)
+        return self.chat(temp_discussion, 
+                        n_predict=n_predict,
+                        stream=stream,
+                        temperature=temperature,
+                        top_k=top_k,
+                        top_p=top_p,
+                        repeat_penalty=repeat_penalty,
+                        repeat_last_n=repeat_last_n,
+                        seed=seed,
+                        n_threads=n_threads,
+                        ctx_size=ctx_size,
+                        streaming_callback=streaming_callback,
+                        split=split,
+                        user_keyword=user_keyword,
+                        ai_keyword=ai_keyword,
+                        **kwargs)
 
     def chat(self,
              discussion: LollmsDiscussion,
@@ -117,7 +149,7 @@ class MistralBinding(LollmsLLMBinding):
 
         try:
             if stream:
-                response = self.client.chat_stream(
+                response = self.client.chat.stream(
                     model=self.model_name,
                     messages=messages,
                     **api_params
@@ -131,7 +163,7 @@ class MistralBinding(LollmsLLMBinding):
                                 break
                 return full_response_text
             else:
-                response = self.client.chat(
+                response = self.client.chat.complete(
                     model=self.model_name,
                     messages=messages,
                     **api_params
@@ -201,7 +233,7 @@ class MistralBinding(LollmsLLMBinding):
             return []
         try:
             ASCIIColors.debug("Listing Mistral models...")
-            models = self.client.list_models()
+            models = self.client.models.list()
             model_info_list = []
             for m in models.data:
                 model_info_list.append({
@@ -264,7 +296,6 @@ if __name__ == '__main__':
         ASCIIColors.cyan("\n--- Text Generation (Streaming) ---")
         full_streamed_text = ""
         def stream_callback(chunk: str, msg_type: int):
-            nonlocal full_streamed_text
             ASCIIColors.green(chunk, end="", flush=True)
             full_streamed_text += chunk
             return True

lollms_client/llm_bindings/ollama/__init__.py

@@ -598,6 +598,94 @@ class OllamaBinding(LollmsLLMBinding):
         ASCIIColors.info(f"Ollama model set to: {model_name}. It will be loaded by the server on first use.")
         return True
 
+    def get_ctx_size(self, model_name: Optional[str] = None) -> Optional[int]:
+        """
+        Retrieves the context size for an Ollama model.
+        
+        The effective context size is the `num_ctx` parameter if overridden in the Modelfile,
+        otherwise it falls back to the model's default context length from its architecture details.
+        As a final failsafe, uses a hardcoded list of known popular models' context lengths.
+        """
+        if model_name is None:
+            model_name = self.model_name
+            
+        try:
+            info = ollama.show(model_name)
+            
+            # Parse num_ctx from the 'parameters' string (e.g., "PARAMETER num_ctx 4096")
+            parameters = info.get('parameters', '')
+            num_ctx = None
+            for param in parameters.split('\n'):
+                if param.strip().startswith('num_ctx'):
+                    num_ctx = int(param.split()[1])
+                    break
+            
+            if num_ctx is not None:
+                return num_ctx
+            
+            # Fall back to model_info context_length (e.g., 'llama.context_length')
+            model_info = info.get('model_info', {})
+            arch = model_info.get('general.architecture', '')
+            context_key = f'{arch}.context_length' if arch else 'general.context_length'
+            context_length = model_info.get(context_key)
+            
+            if context_length is not None:
+                return int(context_length)
+            
+        except Exception as e:
+            ASCIIColors.warning(f"Error fetching model info: {str(e)}")
+        
+        # Failsafe: Hardcoded context sizes for popular Ollama models
+        known_contexts = {
+            'llama2': 4096,       # Llama 2 default
+            'llama3': 8192,       # Llama 3 default
+            'llama3.1': 131072,   # Llama 3.1 extended context
+            'llama3.2': 131072,   # Llama 3.2 extended context
+            'llama3.3': 131072,   # Assuming similar to 3.1/3.2
+            'mistral': 32768,     # Mistral 7B v0.2+ default
+            'mixtral': 32768,     # Mixtral 8x7B default
+            'mixtral8x22b': 65536, # Mixtral 8x22B default
+            'gemma': 8192,        # Gemma default
+            'gemma2': 8192,       # Gemma 2 default
+            'gemma3': 131072,     # Gemma 3 with 128K context
+            'phi': 2048,          # Phi default (older)
+            'phi2': 2048,         # Phi-2 default
+            'phi3': 131072,       # Phi-3 variants often use 128K (mini/medium extended)
+            'qwen': 8192,         # Qwen default
+            'qwen2': 32768,       # Qwen2 default for 7B
+            'qwen2.5': 131072,    # Qwen2.5 with 128K
+            'codellama': 16384,   # CodeLlama extended
+            'codegemma': 8192,    # CodeGemma default
+            'deepseek-coder': 16384,  # DeepSeek-Coder V1 default
+            'deepseek-coder-v2': 131072,  # DeepSeek-Coder V2 with 128K
+            'deepseek-llm': 4096,     # DeepSeek-LLM default
+            'deepseek-v2': 131072,    # DeepSeek-V2 with 128K
+            'yi': 4096,           # Yi base default
+            'yi1.5': 32768,       # Yi-1.5 with 32K
+            'command-r': 131072,  # Command-R with 128K
+            'vicuna': 2048,       # Vicuna default (up to 16K in some variants)
+            'wizardlm': 16384,    # WizardLM default
+            'wizardlm2': 32768,   # WizardLM2 (Mistral-based)
+            'zephyr': 65536,      # Zephyr beta (Mistral-based extended)
+            'falcon': 2048,       # Falcon default
+            'starcoder': 8192,    # StarCoder default
+            'stablelm': 4096,     # StableLM default
+            'orca': 4096,         # Orca default
+            'orca2': 4096,        # Orca 2 default
+            'dolphin': 32768,     # Dolphin (often Mistral-based)
+            'openhermes': 8192,   # OpenHermes default
+        }
+        
+        # Extract base model name (e.g., 'llama3' from 'llama3:8b-instruct')
+        base_name = model_name.split(':')[0].lower().strip()
+        
+        if base_name in known_contexts:
+            ASCIIColors.warning(f"Using hardcoded context size for model '{model_name}': {known_contexts[base_name]}")
+            return known_contexts[base_name]
+        
+        ASCIIColors.warning(f"Context size not found for model '{model_name}'")
+        return None
+        
 if __name__ == '__main__':
     global full_streamed_text
     # Example Usage (requires an Ollama server running)

lollms_client/llm_bindings/open_router/__init__.py

@@ -29,7 +29,7 @@ class OpenRouterBinding(LollmsLLMBinding):
 
     def __init__(self,
                  model_name: str = "google/gemini-flash-1.5", # A good, fast default
-                 open_router_api_key: str = None,
+                 service_key: str|None = None,
                  **kwargs
                  ):
         """
@@ -37,11 +37,11 @@ class OpenRouterBinding(LollmsLLMBinding):
 
         Args:
             model_name (str): The name of the model to use from OpenRouter (e.g., 'anthropic/claude-3-haiku-20240307').
-            open_router_api_key (str): The API key for the OpenRouter service.
+            service_key (str): The API key for the OpenRouter service.
         """
         super().__init__(binding_name=BindingName)
         self.model_name = model_name
-        self.api_key = open_router_api_key or os.getenv("OPENROUTER_API_KEY")
+        self.api_key = service_key or os.getenv("OPENROUTER_API_KEY")
 
         if not self.api_key:
             raise ValueError("OpenRouter API key is required. Set it via 'open_router_api_key' or OPENROUTER_API_KEY env var.")
@@ -84,17 +84,50 @@ class OpenRouterBinding(LollmsLLMBinding):
                 history.append({'role': role, 'content': msg.content})
         return history
 
-    def generate_text(self, prompt: str, **kwargs) -> Union[str, dict]:
+    def generate_text(self,
+                    prompt: str,
+                    images: Optional[List[str]] = None,
+                    system_prompt: str = "",
+                    n_predict: Optional[int] = None,
+                    stream: Optional[bool] = None,
+                    temperature: float = 0.7,  # Ollama default is 0.8, common default 0.7
+                    top_k: int = 40,          # Ollama default is 40
+                    top_p: float = 0.9,       # Ollama default is 0.9
+                    repeat_penalty: float = 1.1,  # Ollama default is 1.1
+                    repeat_last_n: int = 64,  # Ollama default is 64
+                    seed: Optional[int] = None,
+                    n_threads: Optional[int] = None,
+                    ctx_size: int | None = None,
+                    streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+                    split: Optional[bool] = False,  # put to true if the prompt is a discussion
+                    user_keyword: Optional[str] = "!@>user:",
+                    ai_keyword: Optional[str] = "!@>assistant:",
+                    **kwargs
+                    ) -> Union[str, dict]:
         """
         Generate text using OpenRouter. This is a wrapper around the chat method.
         """
-        temp_discussion = LollmsDiscussion.from_messages([
-            LollmsMessage.new_message(sender_type="user", content=prompt)
-        ])
-        if kwargs.get("system_prompt"):
-            temp_discussion.system_prompt = kwargs.get("system_prompt")
+        temp_discussion = LollmsDiscussion(None)
+        temp_discussion.add_message(sender="user", content=prompt, images=images or [])
+        if system_prompt:
+            temp_discussion.system_prompt = system_prompt
         
-        return self.chat(temp_discussion, **kwargs)
+        return self.chat(temp_discussion, 
+                        n_predict=n_predict,
+                        stream=stream,
+                        temperature=temperature,
+                        top_k=top_k,
+                        top_p=top_p,
+                        repeat_penalty=repeat_penalty,
+                        repeat_last_n=repeat_last_n,
+                        seed=seed,
+                        n_threads=n_threads,
+                        ctx_size=ctx_size,
+                        streaming_callback=streaming_callback,
+                        split=split,
+                        user_keyword=user_keyword,
+                        ai_keyword=ai_keyword,
+                        **kwargs)
 
     def chat(self,
              discussion: LollmsDiscussion,

lollms_client/lollms_discussion.py

@@ -1835,4 +1835,9 @@ class LollmsDiscussion:
 
         del self.images[index]
         del self.active_images[index]
-        self.touch()
+        self.touch()
+        
+    @property
+    def system_prompt(self) -> str:
+        """Returns the system prompt for this discussion."""
+        return self._system_prompt