lollms-client 0.29.3__py3-none-any.whl → 0.31.0__py3-none-any.whl

lollms_client/__init__.py

@@ -8,7 +8,7 @@ from lollms_client.lollms_utilities import PromptReshaper # Keep general utiliti
 from lollms_client.lollms_mcp_binding import LollmsMCPBinding, LollmsMCPBindingManager
 from lollms_client.lollms_llm_binding import LollmsLLMBindingManager
 
-__version__ = "0.29.3" # Updated version
+__version__ = "0.31.0" # Updated version
 
 # Optionally, you could define __all__ if you want to be explicit about exports
 __all__ = [

lollms_client/llm_bindings/ollama/__init__.py

@@ -11,6 +11,7 @@ from typing import Optional, Callable, List, Union, Dict
 
 from ascii_colors import ASCIIColors, trace_exception
 import pipmaster as pm
+from lollms_client.lollms_utilities import ImageTokenizer
 pm.ensure_packages(["ollama","pillow","tiktoken"])
 
 
@@ -468,6 +469,24 @@ class OllamaBinding(LollmsLLMBinding):
             return -1
         #return count_tokens_ollama(text, self.model_name, self.ollama_client)
         return len(self.tokenize(text))
+
+    def count_image_tokens(self, image: str) -> int:
+        """
+        Estimate the number of tokens for an image using ImageTokenizer based on self.model_name.
+
+        Args:
+            image (str): Image to count tokens from. Either base64 string, path to image file, or URL.
+
+        Returns:
+            int: Estimated number of tokens for the image. Returns -1 on error.
+        """
+        try:
+            # Delegate token counting to ImageTokenizer
+            return ImageTokenizer(self.model_name).count_image_tokens(image)
+        except Exception as e:
+            ASCIIColors.warning(f"Could not estimate image tokens: {e}")
+            return -1
+
     def embed(self, text: str, **kwargs) -> List[float]:
         """
         Get embeddings for the input text using Ollama API.

lollms_client/lollms_core.py

@@ -430,7 +430,21 @@ class LollmsClient():
         if self.binding:
             return self.binding.count_tokens(text)
         raise RuntimeError("LLM binding not initialized.")
-    
+
+    def count_image_tokens(self, image: str) -> int:
+        """
+        Estimate the number of tokens for an image using ImageTokenizer based on self.model_name.
+
+        Args:
+            image (str): Image to count tokens from. Either base64 string, path to image file, or URL.
+
+        Returns:
+            int: Estimated number of tokens for the image. Returns -1 on error.
+        """
+        if self.binding:
+            return self.binding.count_image_tokens(image)
+        raise RuntimeError("LLM binding not initialized.")
+
     def get_model_details(self) -> dict:
         """
         Get model information from the active LLM binding.
@@ -1574,25 +1588,25 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
         
         # Add the new put_code_in_buffer tool definition
         available_tools.append({
-            "name": "put_code_in_buffer",
+            "name": "local_tools::put_code_in_buffer",
             "description": """Generates and stores code into a buffer to be used by another tool. You can put the uuid of the generated code into the fields that require long code among the tools. If no tool requires code as input do not use put_code_in_buffer. put_code_in_buffer do not execute the code nor does it audit it.""",
             "input_schema": {"type": "object", "properties": {"prompt": {"type": "string", "description": "A detailed natural language description of the code's purpose and requirements."}, "language": {"type": "string", "description": "The programming language of the generated code. By default it uses python."}}, "required": ["prompt"]}
         })
         available_tools.append({
-            "name": "view_generated_code",
+            "name": "local_tools::view_generated_code",
             "description": """Views the code that was generated and stored to the buffer. You need to have a valid uuid of the generated code.""",
             "input_schema": {"type": "object", "properties": {"code_id": {"type": "string", "description": "The case sensitive uuid of the generated code."}}, "required": ["uuid"]}
         })
         # Add the new refactor_scratchpad tool definition
         available_tools.append({
-            "name": "refactor_scratchpad",
+            "name": "local_tools::refactor_scratchpad",
             "description": "Rewrites the scratchpad content to clean it and reorganize it. Only use if the scratchpad is messy or contains too much information compared to what you need.",
             "input_schema": {"type": "object", "properties": {}}
         })
 
         formatted_tools_list = "\n".join([f"**{t['name']}**:\n{t['description']}\ninput schema:\n{json.dumps(t['input_schema'])}" for t in available_tools])
-        formatted_tools_list += "\n**request_clarification**:\nUse if the user's request is ambiguous and you can not infer a clear idea of his intent. this tool has no parameters."
-        formatted_tools_list += "\n**final_answer**:\nUse when you are ready to respond to the user. this tool has no parameters."
+        formatted_tools_list += "\n**local_tools::request_clarification**:\nUse if the user's request is ambiguous and you can not infer a clear idea of his intent. this tool has no parameters."
+        formatted_tools_list += "\n**local_tools::final_answer**:\nUse when you are ready to respond to the user. this tool has no parameters."
 
         if discovery_step_id: log_event(f"**Discovering tools** found {len(available_tools)} tools",MSG_TYPE.MSG_TYPE_STEP_END, event_id=discovery_step_id)
 
@@ -1618,15 +1632,16 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
     - Does the latest observation completely fulfill the user's original request?
     - If YES, your next action MUST be to use the `final_answer` tool.
     - If NO, what is the single next logical step needed? This may involve writing code first with `put_code_in_buffer`, then using another tool.
-    - If you are stuck or the request is ambiguous, use `request_clarification`.
+    - If you are stuck or the request is ambiguous, use `local_tools::request_clarification`.
 3.  **ACT:** Formulate your decision as a JSON object.
+** Important ** Always use this format alias::tool_name to call the tool
 """
                 action_template = {
                     "thought": "My detailed analysis of the last observation and my reasoning for the next action and how it integrates with my global plan.",
                     "action": {
-                        "tool_name": "The single tool to use (e.g., 'put_code_in_buffer', 'time_machine::get_current_time', 'final_answer').",
+                        "tool_name": "The single tool to use (e.g., 'local_tools::put_code_in_buffer', 'local_tools::final_answer').",
                         "tool_params": {"param1": "value1"},
-                        "clarification_question": "(string, ONLY if tool_name is 'request_clarification')"
+                        "clarification_question": "(string, ONLY if tool_name is 'local_tools::request_clarification')"
                     }
                 }
                 if debug: log_prompt(reasoning_prompt_template, f"REASONING PROMPT (Step {i+1})")
@@ -1664,18 +1679,22 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     break
                 
                 # --- Handle special, non-executing tools ---
-                if tool_name == "request_clarification":
+                if tool_name == "local_tools::request_clarification":
                     # Handle clarification...
-                    return {"final_answer": action.get("clarification_question", "Could you please provide more details?"), "final_scratchpad": current_scratchpad, "tool_calls": tool_calls_this_turn, "sources": sources_this_turn, "clarification_required": True, "error": None}
-
-                if tool_name == "final_answer":
+                    if isinstance(action, dict):
+                        return {"final_answer": action.get("clarification_question", "Could you please provide more details?"), "final_scratchpad": current_scratchpad, "tool_calls": tool_calls_this_turn, "sources": sources_this_turn, "clarification_required": True, "error": None}
+                    elif isinstance(action, str):  
+                        return {"final_answer": action, "final_scratchpad": current_scratchpad, "tool_calls": tool_calls_this_turn, "sources": sources_this_turn, "clarification_required": True, "error": None}
+                    else:
+                        return {"final_answer": "Could you please provide more details?", "final_scratchpad": current_scratchpad, "tool_calls": tool_calls_this_turn, "sources": sources_this_turn, "clarification_required": True, "error": None}
+                if tool_name == "local_tools::final_answer":
                     current_scratchpad += f"\n\n### Step {i+1}: Action\n- **Action:** Decided to formulate the final answer."
                     log_event("**Action**: Formulate final answer.", MSG_TYPE.MSG_TYPE_THOUGHT_CHUNK)
                     if reasoning_step_id: log_event(f"**Reasoning Step {i+1}/{max_reasoning_steps}**",MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
                     break
 
                 # --- Handle the `put_code_in_buffer` tool specifically ---
-                if tool_name == 'put_code_in_buffer':
+                if tool_name == 'local_tools::put_code_in_buffer':
                     code_gen_id = log_event(f"Generating code...", MSG_TYPE.MSG_TYPE_STEP_START, metadata={"name": "put_code_in_buffer", "id": "gencode"})
                     code_prompt = tool_params.get("prompt", "Generate the requested code.")
                     
@@ -1694,7 +1713,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     if code_gen_id: log_event(f"Generating code...", MSG_TYPE.MSG_TYPE_TOOL_CALL, metadata={"id": code_gen_id, "result": tool_result})
                     if reasoning_step_id: log_event(f"**Reasoning Step {i+1}/{max_reasoning_steps}**", MSG_TYPE.MSG_TYPE_STEP_END, event_id= reasoning_step_id)
                     continue # Go to the next reasoning step immediately
-                if tool_name == 'view_generated_code':
+                if tool_name == 'local_tools::view_generated_code':
                     code_id = tool_params.get("code_id")
                     if code_id:
                         tool_result = {"status": "success", "code_id": code_id, "generated_code":generated_code_store[code_uuid]}
@@ -1704,7 +1723,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     current_scratchpad += f"\n\n### Step {i+1}: Observation\n- **Action:** Called `{tool_name}`\n- **Result:**\n{observation_text}"
                     log_event(f"Result from `{tool_name}`:\n```\n{generated_code_store[code_uuid]}\n```\n", MSG_TYPE.MSG_TYPE_TOOL_CALL, metadata={"id": code_gen_id, "result": tool_result})
                     continue
-                if tool_name == 'refactor_scratchpad':
+                if tool_name == 'local_tools::refactor_scratchpad':
                     scratchpad_cleaning_prompt = f"""Enhance this scratchpad content to be more organized and comprehensive. Keep relevant experience information and remove any useless redundancies. Try to log learned things from the context so that you won't make the same mistakes again. Do not remove the main objective information or any crucial information that may be useful for the next iterations. Answer directly with the new scratchpad content without any comments.
 --- YOUR INTERNAL SCRATCHPAD (Work History & Analysis) ---
 {current_scratchpad}
@@ -2958,12 +2977,12 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         callback("Deep analysis complete.", MSG_TYPE.MSG_TYPE_STEP_END)
         return final_output
 
-    def summarize(
+    def long_context_processing(
         self,
-        text_to_summarize: str,
+        text_to_process: str,
         contextual_prompt: Optional[str] = None,
-        chunk_size_tokens: int = 1500,
-        overlap_tokens: int = 250,
+        chunk_size_tokens: int|None = None,
+        overlap_tokens: int = 0,
         streaming_callback: Optional[Callable] = None,
         **kwargs
     ) -> str:
@@ -2975,7 +2994,7 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         2.  **Synthesize:** It then takes all the chunk summaries and performs a final summarization pass to create a single, coherent, and comprehensive summary.
 
         Args:
-            text_to_summarize (str): The long text content to be summarized.
+            text_to_process (str): The long text content to be summarized.
             contextual_prompt (Optional[str], optional): A specific instruction to guide the summary's focus. 
                                                        For example, "Summarize the text focusing on the financial implications."
                                                        Defaults to None.
@@ -2993,25 +3012,40 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         Returns:
             str: The final, comprehensive summary of the text.
         """
-        if not text_to_summarize.strip():
+        if not text_to_process and len(kwargs.get("images",[]))==0:
             return ""
-
-        # Use the binding's tokenizer for accurate chunking
-        tokens = self.binding.tokenize(text_to_summarize)
+        if not text_to_process:
+            text_to_process=""
+            tokens = []
+        else:
+            # Use the binding's tokenizer for accurate chunking
+            tokens = self.binding.tokenize(text_to_process)
+        if chunk_size_tokens is None:
+            chunk_size_tokens = self.default_ctx_size//2
         
         if len(tokens) <= chunk_size_tokens:
             if streaming_callback:
-                streaming_callback("Text is short enough for a single summary.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 0})
+                streaming_callback("Text is short enough for a single process.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 0})
+            system_prompt = ("You are a content processor expert.\n"
+                            "You perform tasks on the content as requested by the user.\n\n"
+                            "--- Content ---\n"
+                            f"{text_to_process}\n\n"
+                            "** Important **\n"
+                            "Strictly adhere to the user prompt.\n"
+                            "Do not add comments unless asked to do so.\n"
+                            )
+            if "system_prompt" in kwargs:
+                system_prompt += "-- Extra instructions --\n"+ kwargs["system_prompt"] +"\n"
+                del kwargs["system_prompt"]            
+            prompt_objective = contextual_prompt or "Provide a comprehensive summary of the content."
+            final_prompt = f"{prompt_objective}"
             
-            prompt_objective = contextual_prompt or "Provide a comprehensive summary of the following text."
-            final_prompt = f"{prompt_objective}\n\n--- Text to Summarize ---\n{text_to_summarize}"
-            
-            summary = self.generate_text(final_prompt, **kwargs)
+            processed_output = self.generate_text(final_prompt, system_prompt=system_prompt, **kwargs)
             
             if streaming_callback:
-                streaming_callback("Summary generated.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 100})
+                streaming_callback("Content processed.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 100})
             
-            return summary
+            return processed_output
 
         # --- Stage 1: Chunking and Independent Summarization ---
         chunks = []
@@ -3028,43 +3062,62 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         
         # Define the prompt for summarizing each chunk
         summarization_objective = contextual_prompt or "Summarize the key points of the following text excerpt."
-        chunk_summary_prompt_template = f"{summarization_objective}\n\n--- Text Excerpt ---\n{{chunk_text}}"
+        system_prompt = ("You are a sequential document processing agent.\n"
+                         "The process is done in two phases:\n"
+                         "** Phase1 : **\n"
+                         "Sequencially extracting information from the text chunks and adding them to the scratchpad.\n"
+                         "** Phase2: **\n"
+                         "Synthesizing a comprehensive Response using the scratchpad content given the objective formatting instructions if applicable.\n"
+                         "We are now performing ** Phase 1 **, and we are processing chunk number {{chunk_id}}.\n"
+                         "Your job is to extract information from the current chunk given previous chunks extracted information placed in scratchpad as well as the current chunk content.\n"
+                         "Add the information to the scratchpad while strictly adhering to the Global objective extraction instructions:\n"
+                         "-- Sequencial Scratchpad --\n"
+                         "{{scratchpad}}\n"
+                         "** Important **\n"
+                         "Respond only with the extracted information from the current chunk without repeating things that are already in the scratchpad.\n"
+                         "Strictly adhere to the Global objective content for the extraction phase.\n"
+                         "Do not add comments.\n"
+                        )
+        if "system_prompt" in kwargs:
+            system_prompt += "-- Extra instructions --\n"+ kwargs["system_prompt"] +"\n"
+            del kwargs["system_prompt"]
+        chunk_summary_prompt_template = f"--- Global objective ---\n{summarization_objective}\n\n--- Text Excerpt ---\n{{chunk_text}}"
 
         for i, chunk in enumerate(chunks):
             progress_before = (i / total_steps) * 100
             if streaming_callback:
                 streaming_callback(
-                    f"Summarizing chunk {i + 1} of {len(chunks)}...", 
+                    f"Processing chunk {i + 1} of {len(chunks)}...", 
                     MSG_TYPE.MSG_TYPE_STEP_START, 
                     {"id": f"chunk_{i+1}", "progress": progress_before}
                 )
 
             prompt = chunk_summary_prompt_template.format(chunk_text=chunk)
-            
+            processed_system_prompt = system_prompt.format(chunk_id=i,scratchpad="\n\n---\n\n".join(chunk_summaries))
             try:
                 # Generate summary for the current chunk
-                chunk_summary = self.generate_text(prompt, **kwargs)
+                chunk_summary = self.generate_text(prompt, system_prompt=processed_system_prompt, **kwargs)
                 chunk_summaries.append(chunk_summary)
                 
                 progress_after = ((i + 1) / total_steps) * 100
                 if streaming_callback:
                     streaming_callback(
-                        f"Chunk {i + 1} summarized. Progress: {progress_after:.0f}%", 
+                        f"Chunk {i + 1} processed. Progress: {progress_after:.0f}%", 
                         MSG_TYPE.MSG_TYPE_STEP_END, 
-                        {"id": f"chunk_{i+1}", "summary_snippet": chunk_summary[:100], "progress": progress_after}
+                        {"id": f"chunk_{i+1}", "output_snippet": chunk_summary[:100], "progress": progress_after}
                     )
             except Exception as e:
                 trace_exception(e)
                 if streaming_callback:
-                    streaming_callback(f"Failed to summarize chunk {i+1}: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION)
+                    streaming_callback(f"Failed to process chunk {i+1}: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION)
                 # Still add a placeholder to not break the chain
-                chunk_summaries.append(f"[Error summarizing chunk {i+1}]")
+                chunk_summaries.append(f"[Error processing chunk {i+1}]")
 
         # --- Stage 2: Final Synthesis of All Chunk Summaries ---
         progress_before_synthesis = (len(chunks) / total_steps) * 100
         if streaming_callback:
             streaming_callback(
-                "Synthesizing all chunk summaries into a final version...", 
+                "Processing the scratchpad content into a final version...", 
                 MSG_TYPE.MSG_TYPE_STEP_START, 
                 {"id": "final_synthesis", "progress": progress_before_synthesis}
             )
@@ -3073,16 +3126,29 @@ Provide the final aggregated answer in {output_format} format, directly addressi
         
         # Define the prompt for the final synthesis
         synthesis_objective = contextual_prompt or "Create a single, final, coherent, and comprehensive summary."
+        system_prompt = ("You are a sequential document processing agent.\n"
+                         "The process is done in two phases:\n"
+                         "** Phase1 : **\n"
+                         "Sequencially extracting information from the text chunks and adding them to the scratchpad.\n"
+                         "** Phase2: **\n"
+                         "Synthesizing a comprehensive Response using the scratchpad content given the objective formatting instructions if applicable.\n"
+                         "\n"
+                         "We are now performing ** Phase 2 **.\n"
+                         "Your job is to use the extracted information to fulfill the user prompt objectives.\n"
+                         "Make sure you respect the user formatting if provided and if not, then use markdown output format."
+                         "-- Sequencial Scratchpad --\n"
+                         f"{combined_summaries}\n"
+                         "** Important **\n"
+                         "Respond only with the requested task without extra comments unless told to.\n"
+                         "Strictly adhere to the Global objective content for the extraction phase.\n"
+                         "Do not add comments.\n"
+                        )
         final_synthesis_prompt = (
-            "You are a master synthesizer. You will be given a series of partial summaries from a long document. "
-            f"Your task is to synthesize them into one high-quality summary. {synthesis_objective}\n\n"
-            "Please remove any redundancy and ensure a smooth, logical flow.\n\n"
-            "--- Collection of Summaries ---\n"
-            f"{combined_summaries}\n\n"
-            "--- Final Comprehensive Summary ---"
+            f"--- Global objective ---\n{synthesis_objective}\n\n"
+            "--- Final Response ---"
         )
 
-        final_summary = self.generate_text(final_synthesis_prompt, **kwargs)
+        final_answer = self.generate_text(final_synthesis_prompt, system_prompt=system_prompt, **kwargs)
         
         if streaming_callback:
             streaming_callback(
@@ -3091,7 +3157,7 @@ Provide the final aggregated answer in {output_format} format, directly addressi
                 {"id": "final_synthesis", "progress": 100}
             )
 
-        return final_summary.strip()
+        return final_answer.strip()
 
 def chunk_text(text, tokenizer, detokenizer, chunk_size, overlap, use_separators=True):
     """