lollms-client 0.31.0__py3-none-any.whl → 0.32.0__py3-none-any.whl

lollms_client/llm_bindings/pythonllamacpp/__init__.py

@@ -102,12 +102,65 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
 
         self.model: Optional[Llama] = None
         self.grammar: Optional[LlamaGrammar] = None
+        self.available_models: Dict[str, str] = {}
+        
+        # Scan for available models on initialization
+        self._scan_for_models()
+
+        # If a model_path is provided, it can be a short name or a full path.
+        # If not, auto-select the first available model.
+        if not self.model_path and self.available_models:
+            self.model_path = next(iter(self.available_models.keys()))
+            ASCIIColors.info(f"No model specified. Auto-selecting first found model: '{self.model_path}'")
+        
+        # Resolve the full path to the model
+        resolved_model_path = self.model_path
+        if self.model_path in self.available_models:
+            resolved_model_path = self.available_models[self.model_path]
+        elif not os.path.exists(self.model_path):
+            if self.available_models:
+                raise FileNotFoundError(f"Model '{self.model_path}' not found. Available models: {list(self.available_models.keys())}")
+            else:
+                 raise FileNotFoundError(f"Model file '{self.model_path}' not found, and no other GGUF models were discovered in search paths.")
 
         # Resolve and load grammar if specified
         self._load_grammar_from_config()
 
         # Attempt to load the model
-        self.load_model(self.model_path)
+        self.load_model(resolved_model_path)
+
+    def _scan_for_models(self):
+        """
+        Scans for GGUF models in the specified LoLLMs paths and populates the available_models dictionary.
+        """
+        self.available_models = {}
+        
+        personal_path = self.lollms_paths.get('personal_models_path', '')
+        zoo_path = self.lollms_paths.get('models_zoo_path', '')
+        
+        scan_paths = []
+        if personal_path and os.path.isdir(personal_path):
+            scan_paths.append(personal_path)
+        if zoo_path and os.path.isdir(zoo_path) and zoo_path not in scan_paths:
+            scan_paths.append(zoo_path)
+            
+        ASCIIColors.info(f"Scanning for GGUF models in: {scan_paths}")
+        for scan_path in scan_paths:
+            for root, _, files in os.walk(scan_path):
+                for file in files:
+                    if file.endswith(".gguf"):
+                        full_path = os.path.join(root, file)
+                        # Use relative path from scan_path as the unique "short name"
+                        short_name = os.path.relpath(full_path, scan_path)
+                        # In case of name conflict, personal_path takes precedence
+                        if short_name not in self.available_models:
+                            self.available_models[short_name] = full_path
+        
+        if self.available_models:
+            ASCIIColors.green(f"Discovered {len(self.available_models)} GGUF models.")
+        else:
+            ASCIIColors.warning("No GGUF models found in specified search paths.")
+
 
     def _load_grammar_from_config(self):
         grammar_file_path = self.llama_config.get("grammar_file")
@@ -126,28 +179,14 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
                 ASCIIColors.warning(f"Grammar file not found: {full_grammar_path}")
 
     def load_model(self, model_path: str) -> bool:
+        """
+        Loads the GGUF model from the given full path.
+        """
+        if not os.path.exists(model_path):
+             raise FileNotFoundError(f"Cannot load model. File not found at: {model_path}")
+
+        ASCIIColors.info(f"Attempting to load GGUF model from: {model_path}")
         self.model_path = model_path
-        resolved_model_path = self.model_path
-        if not os.path.exists(resolved_model_path):
-            models_base_path = self.lollms_paths.get('personal_models_path', self.lollms_paths.get('models_zoo_path'))
-            if models_base_path:
-                # Assuming model_path might be relative to a binding-specific folder within models_base_path
-                # e.g. models_zoo_path/llamacpp/model_name.gguf
-                # Or it could be directly models_zoo_path/model_name.gguf
-                potential_path_direct = os.path.join(models_base_path, self.model_path)
-                potential_path_binding_specific = os.path.join(models_base_path, self.binding_name.lower(), self.model_path)
-
-                if os.path.exists(potential_path_direct):
-                    resolved_model_path = potential_path_direct
-                elif os.path.exists(potential_path_binding_specific):
-                    resolved_model_path = potential_path_binding_specific
-                else:
-                    raise FileNotFoundError(f"Model file '{self.model_path}' not found directly or in model paths: '{potential_path_direct}', '{potential_path_binding_specific}'")
-            else:
-                raise FileNotFoundError(f"Model file not found: {self.model_path}")
-        
-        ASCIIColors.info(f"Attempting to load GGUF model from: {resolved_model_path}")
-        self.model_path = resolved_model_path # Update to resolved path
 
         llama_constructor_keys = [
             "n_gpu_layers", "main_gpu", "tensor_split", "vocab_only", "use_mmap", "use_mlock",
@@ -331,12 +370,14 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
     def tokenize(self, text: str) -> List[int]:
         if not self.model:
             ASCIIColors.warning("Llama.cpp model not loaded. Tokenization fallback to tiktoken.")
+            import tiktoken
             return tiktoken.model.encoding_for_model("gpt-3.5-turbo").encode(text) 
         return self.model.tokenize(text.encode("utf-8"), add_bos=False, special=False) 
             
     def detokenize(self, tokens: List[int]) -> str:
         if not self.model:
             ASCIIColors.warning("Llama.cpp model not loaded. Detokenization fallback to tiktoken.")
+            import tiktoken
             return tiktoken.model.encoding_for_model("gpt-3.5-turbo").decode(tokens)
         try:
             return self.model.detokenize(tokens).decode("utf-8", errors="ignore")
@@ -346,13 +387,14 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
     def count_tokens(self, text: str) -> int:
         if not self.model:
             ASCIIColors.warning("Llama.cpp model not loaded. Token count fallback to tiktoken.")
+            import tiktoken
             return len(tiktoken.model.encoding_for_model("gpt-3.5-turbo").encode(text))
         return len(self.tokenize(text))
 
     def embed(self, text: str, **kwargs) -> List[float]:
         if not self.model:
              raise Exception("Llama.cpp model not loaded.")
-        if not self.llama_config.get("embedding"): # or not self.model.params.embedding:
+        if not self.llama_config.get("embedding"):
             raise Exception("Embedding support was not enabled when loading the model (set 'embedding: true' in config).")
         try:
             return self.model.embed(text)
@@ -366,7 +408,7 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
                 "error": "Model not loaded or failed to load."
             }
         
-        is_llava_model = "llava" in self.model_path.lower() or \
+        is_llava_model = "llava" in os.path.basename(self.model_path).lower() or \
                          (self.llama_config.get("chat_format", "").startswith("llava") and \
                           self.llama_config.get("clip_model_path") is not None)
 
@@ -380,17 +422,42 @@ class PythonLlamaCppBinding(LollmsLLMBinding):
             "config": self.llama_config 
         }
 
-    def listModels(self) -> List[Dict[str, str]]: # type: ignore
-        # This method is more for server-based bindings. For LlamaCpp, it describes the loaded model.
-        # It could be extended to scan lollms_paths for GGUF files.
-        if self.model:
-            return [{
-                'model_name': os.path.basename(self.model_path), 'path': self.model_path, 'loaded': True,
-                'n_ctx': str(self.model.context_params.n_ctx), 
+    def listModels(self, force_rescan: bool = False) -> List[Dict[str, str]]: # type: ignore
+        """
+        Lists available GGUF models.
+        
+        Args:
+            force_rescan: If True, rescans the model directories.
+        
+        Returns:
+            A list of dictionaries, each representing a found model.
+        """
+        if force_rescan or not self.available_models:
+            self._scan_for_models()
+
+        model_list = []
+        for short_name, full_path in self.available_models.items():
+            is_loaded = (self.model is not None and self.model_path == full_path)
+            model_entry = {
+                'model_name': short_name, 
+                'path': full_path,
+                'loaded': is_loaded
+            }
+            if is_loaded:
+                model_entry.update({
+                    'n_ctx': str(self.model.context_params.n_ctx), 
+                    'n_gpu_layers': str(self.llama_config.get("n_gpu_layers", "N/A")),
+                })
+            model_list.append(model_entry)
+            
+        if not model_list and self.model_path: # Case where a direct path was loaded but not in scan paths
+             return [{
+                'model_name': os.path.basename(self.model_path), 'path': self.model_path, 'loaded': self.model is not None,
+                'n_ctx': str(self.model.context_params.n_ctx) if self.model else "N/A", 
                 'n_gpu_layers': str(self.llama_config.get("n_gpu_layers","N/A")),
             }]
-        return [{'model_name': os.path.basename(self.model_path) if self.model_path else "Not specified", 
-                 'path': self.model_path, 'loaded': False, 'error': "Model not loaded."}]
+        return model_list
+
     
     def unload_model(self):
         if self.model:
@@ -412,47 +479,39 @@ if __name__ == '__main__':
     ASCIIColors.yellow("Testing PythonLlamaCppBinding...")
 
     # --- IMPORTANT: Configure model path ---
-    # Replace with the ACTUAL PATH to your GGUF model file.
-    # e.g., gguf_model_path = "C:/Models/Mistral-7B-Instruct-v0.2-Q4_K_M.gguf"
-    # If this path is not found, a dummy GGUF will be created for basic tests.
-    gguf_model_path = "model.gguf" # <<< REPLACE THIS OR ENSURE 'model.gguf' EXISTS
-
-    # --- LLaVA Test Configuration (Optional) ---
-    # To test LLaVA, set this to your LLaVA GGUF model path
-    llava_test_model_path = None # e.g., "path/to/your/llava-v1.6-mistral-7b.Q4_K_M.gguf"
-    # And the corresponding mmproj (clip model) GGUF path
-    llava_test_clip_model_path = None # e.g., "path/to/your/mmproj-mistral7b-f16.gguf"
-    # And set the chat format for LLaVA
-    llava_chat_format = "llava-1-6" # or "llava-1-5" depending on your model
-
-    # Attempt to create a dummy GGUF if specified path doesn't exist (for placeholder testing)
-    is_dummy_model = False
-    if not os.path.exists(gguf_model_path):
-        ASCIIColors.warning(f"Model path '{gguf_model_path}' not found.")
-        ASCIIColors.warning("Creating a tiny dummy GGUF file ('dummy_model.gguf') for placeholder testing.")
-        ASCIIColors.warning("This dummy file WILL NOT WORK for actual inference.")
-        try:
-            with open("dummy_model.gguf", "wb") as f: # Minimal valid GGUF structure
-                f.write(b"GGUF\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00")
-                key, val = "tokenizer.ggml.model", "llama"
-                f.write(len(key).to_bytes(8,'little')+key.encode()+b"\x02\x00\x00\x00\x00\x00\x00\x00"+len(val).to_bytes(8,'little')+val.encode())
-            gguf_model_path = "dummy_model.gguf"
-            is_dummy_model = True
-            ASCIIColors.info(f"Using dummy GGUF: {gguf_model_path}. Real inference tests will fail or be skipped.")
-        except Exception as e_dummy:
-            ASCIIColors.error(f"Could not create dummy GGUF: {e_dummy}. Please set a valid GGUF model path.")
-            sys.exit(1)
+    # The binding will now auto-discover models. 
+    # To test auto-selection, set gguf_model_path = None
+    # To test loading a specific model, set its "short name" here, e.g., "Mistral-7B/model.gguf"
+    gguf_model_path = None # <<< SET TO A SHORT NAME, FULL PATH, OR None FOR AUTO-SELECTION
+
+    # --- Test Setup ---
+    # Create a dummy model directory for the test
+    mock_models_dir = "test_models_dir"
+    os.makedirs(os.path.join(mock_models_dir, "subfolder"), exist_ok=True)
+    
+    dummy_gguf_content = b"GGUF\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00"
+    dummy_gguf_content += b"\x14\x00\x00\x00\x00\x00\x00\x00" + b"tokenizer.ggml.model"
+    dummy_gguf_content += b"\x02\x00\x00\x00\x00\x00\x00\x00" + b"\x05\x00\x00\x00\x00\x00\x00\x00" + b"llama"
+    
+    dummy_model_path1 = os.path.join(mock_models_dir, "dummy_model_A.gguf")
+    dummy_model_path2 = os.path.join(mock_models_dir, "subfolder", "dummy_model_B.gguf")
+
+    try:
+        with open(dummy_model_path1, "wb") as f: f.write(dummy_gguf_content)
+        with open(dummy_model_path2, "wb") as f: f.write(dummy_gguf_content)
+        ASCIIColors.info("Created dummy GGUF files for testing.")
+        # If no specific model path is given, the test will use auto-selection on these dummy files.
+        # This will fail at the INFERENCE stage but test the discovery and loading logic.
+        is_dummy_model = True
+    except Exception as e_dummy:
+        ASCIIColors.error(f"Could not create dummy GGUF files: {e_dummy}. Test may fail.")
+        is_dummy_model = False # Assume a real model path is set
 
     binding_config = {
-        "n_gpu_layers": 0, # 0 for CPU, -1 for all possible layers to GPU, or specific number
-        "n_ctx": 1024,     # Short context for testing
-        "seed": 1234,
-        "embedding": True, # Enable embedding generation for the test
-        "verbose": False,  # Set to True for detailed llama.cpp logs
-        # "grammar_file": "json.gbnf" # Example for grammar test
+        "n_gpu_layers": 0, "n_ctx": 1024, "seed": 1234, "embedding": True, "verbose": False,
     }
     
-    mock_lollms_paths = { "personal_models_path": ".", "grammars_path": "grammars_test" }
+    mock_lollms_paths = { "personal_models_path": mock_models_dir, "grammars_path": "grammars_test" }
     if not os.path.exists(mock_lollms_paths["grammars_path"]):
         os.makedirs(mock_lollms_paths["grammars_path"], exist_ok=True)
     with open(os.path.join(mock_lollms_paths["grammars_path"], "test_grammar.gbnf"), "w") as f:
@@ -461,17 +520,27 @@ if __name__ == '__main__':
     active_binding = None
     try:
         ASCIIColors.cyan("\n--- Initializing PythonLlamaCppBinding ---")
+        # Initialize with the specified path (or None for auto-selection)
         active_binding = PythonLlamaCppBinding(model_path=gguf_model_path, config=binding_config, lollms_paths=mock_lollms_paths)
-        ASCIIColors.green(f"Binding initialized. Model: {active_binding.model_path}")
-        ASCIIColors.info(f"Model Info: {json.dumps(active_binding.get_model_info(), indent=2)}")
+        
+        ASCIIColors.green(f"Binding initialized. Loaded model: {os.path.basename(active_binding.model_path)}")
+        ASCIIColors.info(f"Full path: {active_binding.model_path}")
+
+        # --- List Models ---
+        ASCIIColors.cyan("\n--- Listing Models (force_rescan=True) ---")
+        model_list = active_binding.listModels(force_rescan=True)
+        print(json.dumps(model_list, indent=2))
+        assert len(model_list) == 2, "Model discovery failed to find all dummy models."
+        assert any(m['loaded'] for m in model_list), "listModels did not correctly report a loaded model."
+
 
         if is_dummy_model:
-            ASCIIColors.warning("\nRUNNING WITH DUMMY MODEL. MOST FUNCTIONALITY TESTS WILL BE SKIPPED.")
+            ASCIIColors.warning("\nRUNNING WITH DUMMY MODEL. INFERENCE TESTS WILL BE SKIPPED.")
+            ASCIIColors.info("Discovery, loading, and listing tests passed.")
         else:
-            # --- List Models ---
-            ASCIIColors.cyan("\n--- Listing Models ---")
-            print(json.dumps(active_binding.listModels(), indent=2))
-
+            # These tests will only run if you provided a path to a REAL model
+            ASCIIColors.info("\nRUNNING WITH REAL MODEL. PROCEEDING WITH INFERENCE TESTS...")
+            
             # --- Tokenize/Detokenize ---
             ASCIIColors.cyan("\n--- Tokenize/Detokenize ---")
             sample_text = "Hello, Llama.cpp world! This is a test sentence."
@@ -508,71 +577,13 @@ if __name__ == '__main__':
             if isinstance(result, str): ASCIIColors.green(f"Full streamed text: {result}")
             else: ASCIIColors.error(f"Streaming generation failed: {result}")
             
-            # --- Text Generation with Grammar ---
-            ASCIIColors.cyan("\n--- Text Generation with Grammar ---")
-            generated_grammar_text = active_binding.generate_text(
-                "Output a greeting:", n_predict=5, stream=False, use_chat_format=False, # Grammar often better with raw completion
-                grammar=os.path.join(mock_lollms_paths["grammars_path"], "test_grammar.gbnf")
-            )
-            if isinstance(generated_grammar_text, str):
-                ASCIIColors.green(f"Generated text with grammar: '{generated_grammar_text.strip()}'")
-                assert generated_grammar_text.strip().lower() in ["hello", "world"], "Grammar constraint failed!"
-            else: ASCIIColors.error(f"Grammar generation failed: {generated_grammar_text}")
-
-            # --- Embeddings ---
-            if binding_config.get("embedding"):
-                ASCIIColors.cyan("\n--- Embeddings ---")
-                embedding_text = "This is a test for embeddings."
-                try:
-                    embedding_vector = active_binding.embed(embedding_text)
-                    ASCIIColors.green(f"Embedding for '{embedding_text}' (first 3 dims): {embedding_vector[:3]}...")
-                    ASCIIColors.info(f"Embedding vector dimension: {len(embedding_vector)}")
-                except Exception as e_emb: ASCIIColors.warning(f"Could not get embedding: {e_emb}")
-            else: ASCIIColors.yellow("\n--- Embeddings Skipped (embedding: false in config) ---")
-
-        # --- LLaVA Test (if configured and real model is LLaVA) ---
-        if not is_dummy_model and llava_test_model_path and os.path.exists(llava_test_model_path) and \
-           llava_test_clip_model_path and os.path.exists(llava_test_clip_model_path) and \
-           active_binding and active_binding.model_path.lower() == llava_test_model_path.lower():
-            
-            ASCIIColors.cyan("\n--- LLaVA Vision Test ---")
-            # This assumes the 'active_binding' was ALREADY loaded with the LLaVA model
-            # and its specific config (clip_model_path, chat_format="llava-1-x").
-            # If not, you'd need to unload and reload/reinitialize the binding for LLaVA.
-            if not (active_binding.llama_config.get("chat_format","").startswith("llava") and \
-                    active_binding.llama_config.get("clip_model_path")):
-                ASCIIColors.warning("Current binding not configured for LLaVA. Skipping LLaVA test.")
-                ASCIIColors.warning("To test LLaVA, ensure gguf_model_path points to LLaVA model and config includes 'chat_format' and 'clip_model_path'.")
-            else:
-                dummy_image_path = "dummy_llava_image.png"
-                try:
-                    from PIL import Image, ImageDraw
-                    img = Image.new('RGB', (200, 80), color = ('cyan'))
-                    d = ImageDraw.Draw(img); d.text((10,20), "LLaVA Test", fill=('black'))
-                    img.save(dummy_image_path)
-                    ASCIIColors.info(f"Created dummy image for LLaVA: {dummy_image_path}")
-
-                    llava_prompt = "What do you see in this image?"
-                    llava_response = active_binding.generate_text(
-                        prompt=llava_prompt, images=[dummy_image_path], n_predict=50, stream=False, use_chat_format=True
-                    )
-                    if isinstance(llava_response, str): ASCIIColors.green(f"LLaVA response: {llava_response}")
-                    else: ASCIIColors.error(f"LLaVA generation failed: {llava_response}")
-                except ImportError: ASCIIColors.warning("Pillow not found. Cannot create dummy image for LLaVA.")
-                except Exception as e_llava: ASCIIColors.error(f"LLaVA test error: {e_llava}"); trace_exception(e_llava)
-                finally:
-                    if os.path.exists(dummy_image_path): os.remove(dummy_image_path)
-        elif not is_dummy_model and llava_test_model_path: # If LLaVA test paths are set but model isn't LLaVA
-             ASCIIColors.yellow(f"LLaVA test paths are set, but current model '{active_binding.model_path if active_binding else 'N/A'}' is not '{llava_test_model_path}'.")
-             ASCIIColors.yellow("Skipping LLaVA-specific test section. To run, set main gguf_model_path to LLaVA model and configure LLaVA params.")
-
-
     except ImportError as e_imp:
         ASCIIColors.error(f"Import error: {e_imp}. Llama-cpp-python might not be installed/configured correctly.")
     except FileNotFoundError as e_fnf:
-        ASCIIColors.error(f"Model file error: {e_fnf}. Ensure GGUF model path is correct.")
+        ASCIIColors.error(f"Model file error: {e_fnf}. Ensure GGUF model path is correct or models are in the right directory.")
     except RuntimeError as e_rt: 
         ASCIIColors.error(f"Runtime error (often model load failure or llama.cpp issue): {e_rt}")
+        if is_dummy_model: ASCIIColors.yellow("This error is expected when using a dummy model for loading.")
         trace_exception(e_rt)
     except Exception as e_main:
         ASCIIColors.error(f"An unexpected error occurred: {e_main}")
@@ -583,8 +594,11 @@ if __name__ == '__main__':
             active_binding.unload_model()
             ASCIIColors.green("Model unloaded.")
         
-        if is_dummy_model and os.path.exists("dummy_model.gguf"):
-            os.remove("dummy_model.gguf")
+        # Cleanup dummy files and directories
+        if os.path.exists(mock_models_dir):
+            import shutil
+            shutil.rmtree(mock_models_dir)
+            ASCIIColors.info(f"Cleaned up dummy model directory: {mock_models_dir}")
         
         test_grammar_file = os.path.join(mock_lollms_paths["grammars_path"], "test_grammar.gbnf")
         if os.path.exists(test_grammar_file): os.remove(test_grammar_file)

lollms_client/lollms_core.py

@@ -65,8 +65,8 @@ class LollmsClient():
                  # General Parameters (mostly defaults for LLM generation)
                  service_key: Optional[str] = None, # Shared service key/client_id
                  verify_ssl_certificate: bool = True,
-                 ctx_size: Optional[int] = 8192,
-                 n_predict: Optional[int] = 4096,
+                 ctx_size: Optional[int|None] = None,
+                 n_predict: Optional[int|None] = None,
                  stream: bool = False,
                  temperature: float = 0.7, # Ollama default is 0.8, common default 0.7
                  top_k: int = 40,          # Ollama default is 40
@@ -277,7 +277,7 @@ class LollmsClient():
             available = self.binding_manager.get_available_bindings()
             raise ValueError(f"Failed to update LLM binding: {binding_name}. Available: {available}")
 
-    def get_ctx_size(self, model_name=None):
+    def get_ctx_size(self, model_name:str|None=None):
         if self.binding:
             ctx_size = self.binding.get_ctx_size(model_name)
             return ctx_size if ctx_size else self.default_ctx_size
@@ -524,11 +524,23 @@ class LollmsClient():
             Union[str, dict]: Generated text or error dictionary if failed.
         """
         if self.binding:
+            
+            ctx_size = ctx_size if ctx_size is not None else self.default_ctx_size if self.default_ctx_size else None
+            if ctx_size is None:
+                ctx_size = self.binding.get_ctx_size()
+                if ctx_size is None:
+                    ctx_size = 1024*8 # 1028*8= 8192 tokens, a common default for many models
+            nb_input_tokens = self.count_tokens(prompt)+ (sum([self.count_image_tokens(image) for image in images]) if images else 0)
+            if kwargs.get("debug", False):
+                ASCIIColors.magenta(f"Generating text using these parameters:")
+                ASCIIColors.magenta(f"ctx_size : {ctx_size}")
+                ASCIIColors.magenta(f"nb_input_tokens : {nb_input_tokens}")
+            
             return self.binding.generate_text(
                 prompt=prompt,
                 images=images,
                 system_prompt=system_prompt,
-                n_predict=n_predict if n_predict is not None else self.default_n_predict,
+                n_predict=n_predict if n_predict else self.default_n_predict if self.default_n_predict else ctx_size - nb_input_tokens,
                 stream=stream if stream is not None else self.default_stream,
                 temperature=temperature if temperature is not None else self.default_temperature,
                 top_k=top_k if top_k is not None else self.default_top_k,
@@ -1586,10 +1598,10 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}
                 })
         
-        # Add the new put_code_in_buffer tool definition
+        # Add the new prepare_code tool definition
         available_tools.append({
-            "name": "local_tools::put_code_in_buffer",
-            "description": """Generates and stores code into a buffer to be used by another tool. You can put the uuid of the generated code into the fields that require long code among the tools. If no tool requires code as input do not use put_code_in_buffer. put_code_in_buffer do not execute the code nor does it audit it.""",
+            "name": "local_tools::prepare_code",
+            "description": """Generates and stores code into a buffer to be used by another tool. Never put code into a tool directly, first call this to generate the code and then paste the uuid in the tool that requires code. Only use this for generating code to be sent to another tool. You can put the uuid of the generated code into the fields that require long code among the tools. If no tool requires code as input do not use prepare_code. prepare_code do not execute the code nor does it audit it.""",
             "input_schema": {"type": "object", "properties": {"prompt": {"type": "string", "description": "A detailed natural language description of the code's purpose and requirements."}, "language": {"type": "string", "description": "The programming language of the generated code. By default it uses python."}}, "required": ["prompt"]}
         })
         available_tools.append({
@@ -1631,7 +1643,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
 2.  **THINK:**
     - Does the latest observation completely fulfill the user's original request?
     - If YES, your next action MUST be to use the `final_answer` tool.
-    - If NO, what is the single next logical step needed? This may involve writing code first with `put_code_in_buffer`, then using another tool.
+    - If NO, what is the single next logical step needed? This may involve writing code first with `prepare_code`, then using another tool.
     - If you are stuck or the request is ambiguous, use `local_tools::request_clarification`.
 3.  **ACT:** Formulate your decision as a JSON object.
 ** Important ** Always use this format alias::tool_name to call the tool
@@ -1639,7 +1651,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                 action_template = {
                     "thought": "My detailed analysis of the last observation and my reasoning for the next action and how it integrates with my global plan.",
                     "action": {
-                        "tool_name": "The single tool to use (e.g., 'local_tools::put_code_in_buffer', 'local_tools::final_answer').",
+                        "tool_name": "The single tool to use (e.g., 'local_tools::prepare_code', 'local_tools::final_answer').",
                         "tool_params": {"param1": "value1"},
                         "clarification_question": "(string, ONLY if tool_name is 'local_tools::request_clarification')"
                     }
@@ -1693,20 +1705,20 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     if reasoning_step_id: log_event(f"**Reasoning Step {i+1}/{max_reasoning_steps}**",MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
                     break
 
-                # --- Handle the `put_code_in_buffer` tool specifically ---
-                if tool_name == 'local_tools::put_code_in_buffer':
-                    code_gen_id = log_event(f"Generating code...", MSG_TYPE.MSG_TYPE_STEP_START, metadata={"name": "put_code_in_buffer", "id": "gencode"})
+                # --- Handle the `prepare_code` tool specifically ---
+                if tool_name == 'local_tools::prepare_code':
+                    code_gen_id = log_event(f"Generating code...", MSG_TYPE.MSG_TYPE_STEP_START, metadata={"name": "prepare_code", "id": "gencode"})
                     code_prompt = tool_params.get("prompt", "Generate the requested code.")
                     
                     # Use a specific system prompt to get raw code
                     code_generation_system_prompt = "You are a code generation assistant. Generate ONLY the raw code based on the user's request. Do not add any explanations, markdown code fences, or other text outside of the code itself."
-                    generated_code = self.generate_code(prompt=code_prompt, system_prompt=code_generation_system_prompt + "\n----\n" + reasoning_prompt_template, **llm_generation_kwargs)
+                    generated_code = self.generate_code(prompt=code_prompt, system_prompt=code_generation_system_prompt, **llm_generation_kwargs)
                     
                     code_uuid = str(uuid.uuid4())
                     generated_code_store[code_uuid] = generated_code
                     
                     tool_result = {"status": "success", "code_id": code_uuid, "summary": f"Code generated successfully. Use this ID in the next tool call that requires code."}
-                    tool_calls_this_turn.append({"name": "put_code_in_buffer", "params": tool_params, "result": tool_result})
+                    tool_calls_this_turn.append({"name": "prepare_code", "params": tool_params, "result": tool_result})
                     observation_text = f"```json\n{json.dumps(tool_result, indent=2)}\n```"
                     current_scratchpad += f"\n\n### Step {i+1}: Observation\n- **Action:** Called `{tool_name}`\n- **Result:**\n{observation_text}"
                     log_event(f"Code generated with ID: {code_uuid}", MSG_TYPE.MSG_TYPE_OBSERVATION)
@@ -3025,7 +3037,7 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         
         if len(tokens) <= chunk_size_tokens:
             if streaming_callback:
-                streaming_callback("Text is short enough for a single process.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 0})
+                streaming_callback("Text is short enough for a single pass.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 0})
             system_prompt = ("You are a content processor expert.\n"
                             "You perform tasks on the content as requested by the user.\n\n"
                             "--- Content ---\n"

lollms_client/lollms_discussion.py

@@ -1738,6 +1738,8 @@ class LollmsDiscussion:
                 "description": "JSON object as title of the discussion."
             }
             infos = self.lollmsClient.generate_structured_content(prompt = prompt, system_prompt=system_prompt, schema = title_generation_schema)
+            if infos is None or "title" not in infos:
+                raise ValueError("Title generation failed or returned invalid data.")
             discussion_title = infos["title"]
             new_metadata = (self.metadata or {}).copy()
             new_metadata['title'] = discussion_title

lollms_client/lollms_llm_binding.py

@@ -154,7 +154,7 @@ class LollmsLLMBinding(ABC):
         """
         pass
 
-    def get_ctx_size(self, model_name=None):
+    def get_ctx_size(self, model_name:str|None=None):
         # if model_name is none use current model name
         return None
 

lollms_client/lollms_mcp_security.py

@@ -0,0 +1,84 @@
+# File name: lollms_mcp_security.py
+# Author: parisneo
+
+"""
+This script defines a custom token verifier for MCP (Model Context Protocol) using an introspection endpoint.
+The verifier queries the authorization server to check if a given token is valid. It is agnostic about how tokens are created.
+
+Key components:
+- MyTokenInfo class: Extends AccessToken and includes additional fields like user_id and username.
+- IntrospectionTokenVerifier class: Implements the logic to verify tokens by making HTTP requests to an introspection endpoint.
+- token_info_context: A context variable to store token information for easy access.
+
+The script also includes an example of how to use these components within a FastMCP instance, setting up authentication and authorization settings.
+
+Dependencies:
+- mcp.server.auth.provider
+- httpx
+- os
+- contextvars
+
+Environment Variables:
+- AUTHORIZATION_SERVER_URL: The URL of the authorization server. Default is 'http://localhost:9642'.
+"""
+
+from mcp.server.auth.provider import AccessToken, TokenVerifier
+import httpx
+import os
+from contextvars import ContextVar
+
+AUTHORIZATION_SERVER_URL = os.environ.get("AUTHORIZATION_SERVER_URL", "http://localhost:9642")
+
+class MyTokenInfo(AccessToken):
+    user_id: int | None = None
+    username: str | None = None
+
+token_info_context: ContextVar[MyTokenInfo | None] = ContextVar("token_info_context", default=None)
+
+# This is our set of valid API keys. In a real app, you'd check a database.
+class IntrospectionTokenVerifier(TokenVerifier):
+    """
+    This verifier asks the authorization server if a token is valid.
+    It is completely agnostic about how tokens are created.
+    """
+    async def verify_token(self, token: str) -> AccessToken:
+        # Make a secure HTTP call to your /introspect endpoint
+        async with httpx.AsyncClient() as client:
+            try:
+                response = await client.post(
+                    f"{AUTHORIZATION_SERVER_URL}/api/auth/introspect",
+                    data={"token": token}
+                )
+                response.raise_for_status()  # Raise an exception for 4xx/5xx errors
+            except httpx.RequestError as e:
+                print(f"ERROR: Could not connect to introspection endpoint: {e}")
+                return AccessToken(active=False, token="", client_id="", scopes=[])
+
+        # Create a TokenInfo object directly from the JSON response
+        token_info_dict = response.json()
+        token_info_dict["token"] = token
+        token_info_dict["client_id"] = str(token_info_dict.get("user_id"))
+        token_info_dict["scopes"] = []
+        token_info = MyTokenInfo(**token_info_dict)
+        token_info_context.set(token_info)
+        return MyTokenInfo(**token_info_dict)
+
+# To recover the user information, just use token_info = token_info_context.get()
+# Example use
+# resource_server_url=f"http://localhost:{port}"
+# mcp = FastMCP(
+#     name="MyMCPServer",
+#     host=host,
+#     port=port,
+#     log_level=log_level,
+#     # 1. This tells MCP to use our class for authentication.
+#     token_verifier=IntrospectionTokenVerifier(),
+#     # 2. This tells MCP to protect all tools by default.
+#     auth=AuthSettings(
+#         # The URL of the server that issues tokens
+#         issuer_url=AUTHORIZATION_SERVER_URL,
+#         # The URL of the MCP server itself
+#         resource_server_url=resource_server_url,  # The port of the MCP server
+#         required_scopes=[]  # Requires valid authentication
+#     )
+# )

{lollms_client-0.31.0.dist-info → lollms_client-0.32.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lollms_client
-Version: 0.31.0
+Version: 0.32.0
 Summary: A client library for LoLLMs generate endpoint
 Author-email: ParisNeo <parisneoai@gmail.com>
 License: Apache Software License