lollms-client 1.4.1__py3-none-any.whl → 1.7.10__py3-none-any.whl

lollms_client/lollms_core.py

@@ -1,4 +1,6 @@
 # lollms_client/lollms_core.py
+# author: ParisNeo
+# description: LollmsClient definition file
 import requests
 from ascii_colors import ASCIIColors, trace_exception
 from lollms_client.lollms_types import MSG_TYPE, ELF_COMPLETION_FORMAT
@@ -89,21 +91,6 @@ class LollmsClient():
             stt_binding_config (Optional[Dict]): Additional config for the STT binding.
             ttv_binding_config (Optional[Dict]): Additional config for the TTV binding.
             ttm_binding_config (Optional[Dict]): Additional config for the TTM binding.
-            service_key (Optional[str]): Shared authentication key or client_id.
-            verify_ssl_certificate (bool): Whether to verify SSL certificates.
-            ctx_size (Optional[int]): Default context size for LLM.
-            n_predict (Optional[int]): Default max tokens for LLM.
-            stream (bool): Default streaming mode for LLM.
-            temperature (float): Default temperature for LLM.
-            top_k (int): Default top_k for LLM.
-            top_p (float): Default top_p for LLM.
-            repeat_penalty (float): Default repeat penalty for LLM.
-            repeat_last_n (int): Default repeat last n for LLM.
-            seed (Optional[int]): Default seed for LLM.
-            n_threads (int): Default threads for LLM.
-            streaming_callback (Optional[Callable]): Default streaming callback for LLM.
-            user_name (str): Default user name for prompts.
-            ai_name (str): Default AI name for prompts.
 
         Raises:
             ValueError: If the primary LLM binding cannot be created.
@@ -143,103 +130,134 @@ class LollmsClient():
                 ASCIIColors.warning(f"Failed to create LLM binding: {llm_binding_name}. Available: {available}")
 
         if tts_binding_name:
-            self.tts = self.tts_binding_manager.create_binding(
-                binding_name=tts_binding_name,
-                **{
-                    k: v
-                    for k, v in (tts_binding_config or {}).items()
-                    if k != "binding_name"
-                }
-            )
-            if self.tts is None:
-                ASCIIColors.warning(f"Failed to create TTS binding: {tts_binding_name}. Available: {self.tts_binding_manager.get_available_bindings()}")
-
-        if tti_binding_name:
-            if tti_binding_config:
-                self.tti = self.tti_binding_manager.create_binding(
-                    binding_name=tti_binding_name,
-                    **{
+            try:
+                params = {
                         k: v
-                        for k, v in (tti_binding_config or {}).items()
+                        for k, v in (tts_binding_config or {}).items()
                         if k != "binding_name"
                     }
+                self.tts = self.tts_binding_manager.create_binding(
+                    binding_name=tts_binding_name,
+                    **params
                 )
-            else:
-                self.tti = self.tti_binding_manager.create_binding(
-                    binding_name=tti_binding_name
-                )
-            if self.tti is None:
-                ASCIIColors.warning(f"Failed to create TTI binding: {tti_binding_name}. Available: {self.tti_binding_manager.get_available_bindings()}")
+                if self.tts is None:
+                    ASCIIColors.warning(f"Failed to create TTS binding: {tts_binding_name}. Available: {self.tts_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating TTS binding: {str(e)}")
+                self.tts = None
 
+        if tti_binding_name:
+            try:
+                if tti_binding_config:
+                    self.tti = self.tti_binding_manager.create_binding(
+                        binding_name=tti_binding_name,
+                        **{
+                            k: v
+                            for k, v in (tti_binding_config or {}).items()
+                            if k != "binding_name"
+                        }
+                    )
+                else:
+                    self.tti = self.tti_binding_manager.create_binding(
+                        binding_name=tti_binding_name
+                    )
+                if self.tti is None:
+                    ASCIIColors.warning(f"Failed to create TTI binding: {tti_binding_name}. Available: {self.tti_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating TTI binding: {str(e)}")
+                self.tti = None
+                
         if stt_binding_name:
-            if stt_binding_config:
-                self.stt = self.stt_binding_manager.create_binding(
-                    binding_name=stt_binding_name,
-                    **{
-                        k: v
-                        for k, v in (stt_binding_config or {}).items()
-                        if k != "binding_name"
-                    }
-                )
+            try:
+                if stt_binding_config:
+                    self.stt = self.stt_binding_manager.create_binding(
+                        binding_name=stt_binding_name,
+                        **{
+                            k: v
+                            for k, v in (stt_binding_config or {}).items()
+                            if k != "binding_name"
+                        }
+                    )
 
-            else:
-                self.stt = self.stt_binding_manager.create_binding(
-                    binding_name=stt_binding_name,
-                )
-            if self.stt is None:
-                ASCIIColors.warning(f"Failed to create STT binding: {stt_binding_name}. Available: {self.stt_binding_manager.get_available_bindings()}")
+                else:
+                    self.stt = self.stt_binding_manager.create_binding(
+                        binding_name=stt_binding_name,
+                    )
+                if self.stt is None:
+                    ASCIIColors.warning(f"Failed to create STT binding: {stt_binding_name}. Available: {self.stt_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating STT binding: {str(e)}")
+                self.stt = None
+                
         if ttv_binding_name:
-            if ttv_binding_config:
-                self.ttv = self.ttv_binding_manager.create_binding(
-                    binding_name=ttv_binding_name,
-                    **{
-                        k: v
-                        for k, v in ttv_binding_config.items()
-                        if k != "binding_name"
-                    }
-                )
+            try:
+                if ttv_binding_config:
+                    self.ttv = self.ttv_binding_manager.create_binding(
+                        binding_name=ttv_binding_name,
+                        **{
+                            k: v
+                            for k, v in ttv_binding_config.items()
+                            if k != "binding_name"
+                        }
+                    )
 
-            else:
-                self.ttv = self.ttv_binding_manager.create_binding(
-                    binding_name=ttv_binding_name
-                )
-            if self.ttv is None:
-                ASCIIColors.warning(f"Failed to create TTV binding: {ttv_binding_name}. Available: {self.ttv_binding_manager.get_available_bindings()}")
+                else:
+                    self.ttv = self.ttv_binding_manager.create_binding(
+                        binding_name=ttv_binding_name
+                    )
+                if self.ttv is None:
+                    ASCIIColors.warning(f"Failed to create TTV binding: {ttv_binding_name}. Available: {self.ttv_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating TTV binding: {str(e)}")
+                self.ttv = None
 
         if ttm_binding_name:
-            if ttm_binding_config:
-                self.ttm = self.ttm_binding_manager.create_binding(
-                    binding_name=ttm_binding_name,
-                    **{
-                        k: v
-                        for k, v in (ttm_binding_config or {}).items()
-                        if k != "binding_name"
-                    }
-                )
-            else:
-                self.ttm = self.ttm_binding_manager.create_binding(
-                    binding_name=ttm_binding_name
-                )
-            if self.ttm is None:
-                ASCIIColors.warning(f"Failed to create TTM binding: {ttm_binding_name}. Available: {self.ttm_binding_manager.get_available_bindings()}")
+            try:
+                if ttm_binding_config:
+                    self.ttm = self.ttm_binding_manager.create_binding(
+                        binding_name=ttm_binding_name,
+                        **{
+                            k: v
+                            for k, v in (ttm_binding_config or {}).items()
+                            if k != "binding_name"
+                        }
+                    )
+                else:
+                    self.ttm = self.ttm_binding_manager.create_binding(
+                        binding_name=ttm_binding_name
+                    )
+                if self.ttm is None:
+                    ASCIIColors.warning(f"Failed to create TTM binding: {ttm_binding_name}. Available: {self.ttm_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating TTM binding: {str(e)}")
+                self.ttm = None
 
         if mcp_binding_name:
-            if mcp_binding_config:
-                self.mcp = self.mcp_binding_manager.create_binding(
-                    binding_name=mcp_binding_name,
-                    **{
-                        k: v
-                        for k, v in (mcp_binding_config or {}).items()
-                        if k != "binding_name"
-                    }
-                )
-            else:
-                self.mcp = self.mcp_binding_manager.create_binding(
-                    mcp_binding_name
-                )
-            if self.mcp is None:
-                ASCIIColors.warning(f"Failed to create MCP binding: {mcp_binding_name}. Available: {self.mcp_binding_manager.get_available_bindings()}")
-
+            try:
+                if mcp_binding_config:
+                    self.mcp = self.mcp_binding_manager.create_binding(
+                        binding_name=mcp_binding_name,
+                        **{
+                            k: v
+                            for k, v in (mcp_binding_config or {}).items()
+                            if k != "binding_name"
+                        }
+                    )
+                else:
+                    self.mcp = self.mcp_binding_manager.create_binding(
+                        mcp_binding_name
+                    )
+                if self.mcp is None:
+                    ASCIIColors.warning(f"Failed to create MCP binding: {mcp_binding_name}. Available: {self.mcp_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating MCP binding: {str(e)}")
+                self.mcp = None            
         # --- Store Default Generation Parameters ---
 
         # --- Prompt Formatting Attributes ---
@@ -268,8 +286,8 @@ class LollmsClient():
             raise ValueError(f"Failed to update LLM binding: {binding_name}. Available: {available}")
 
     def get_ctx_size(self, model_name:str|None=None):
-        if self.llm:
-            ctx_size = self.llm.get_ctx_size(model_name)
+        if self.llm and self.llm.model_name:
+            ctx_size = self.llm.get_ctx_size(model_name or self.llm.model_name)
             return ctx_size if ctx_size else self.llm.default_ctx_size
         else:
             return None
@@ -470,25 +488,28 @@ class LollmsClient():
         return self.llm_binding_manager.get_available_bindings()
 
     def generate_text(self,
-                     prompt: str,
-                     images: Optional[List[str]] = None,
-                     system_prompt: str = "",
-                     n_predict: Optional[int] = None,
-                     stream: Optional[bool] = None,
-                     temperature: Optional[float] = None,
-                     top_k: Optional[int] = None,
-                     top_p: Optional[float] = None,
-                     repeat_penalty: Optional[float] = None,
-                     repeat_last_n: Optional[int] = None,
-                     seed: Optional[int] = None,
-                     n_threads: Optional[int] = None,
-                     ctx_size: int | None = None,
-                     streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
-                     split:Optional[bool]=False, # put to true if the prompt is a discussion
-                     user_keyword:Optional[str]="!@>user:",
-                     ai_keyword:Optional[str]="!@>assistant:",
-                     **kwargs
-                     ) -> Union[str, dict]:
+                    prompt: str,
+                    images: Optional[List[str]] = None,
+                    system_prompt: str = "",
+                    n_predict: Optional[int] = None,
+                    stream: Optional[bool] = None,
+                    temperature: Optional[float] = None,
+                    top_k: Optional[int] = None,
+                    top_p: Optional[float] = None,
+                    repeat_penalty: Optional[float] = None,
+                    repeat_last_n: Optional[int] = None,
+                    seed: Optional[int] = None,
+                    n_threads: Optional[int] = None,
+                    ctx_size: int | None = None,
+                    streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+                    split:Optional[bool]=False, # put to true if the prompt is a discussion
+                    user_keyword:Optional[str]="!@>user:",
+                    ai_keyword:Optional[str]="!@>assistant:",
+                    think: Optional[bool] = False,
+                    reasoning_effort: Optional[bool] = "low", # low, medium, high
+                    reasoning_summary: Optional[bool] = "auto", # auto
+                    **kwargs
+                    ) -> Union[str, dict]:
         """
         Generate text using the active LLM binding, using instance defaults if parameters are not provided.
 
@@ -509,12 +530,15 @@ class LollmsClient():
             split:Optional[bool]: put to true if the prompt is a discussion
             user_keyword:Optional[str]: when splitting we use this to extract user prompt 
             ai_keyword:Optional[str]": when splitting we use this to extract ai prompt
+            think: Optional[bool]: Activate thinking or deactivate it
+            reasoning_effort: Optional[bool]: If think is active, this specifies what effort to put into the thinking
+            reasoning_summary: Optional[bool]: If think is active, this specifies if a summary will be generated
 
         Returns:
             Union[str, dict]: Generated text or error dictionary if failed.
         """
         if self.llm:
-            
+            images = [str(image) for image in images] if images else None
             ctx_size = ctx_size if ctx_size is not None else self.llm.default_ctx_size if self.llm.default_ctx_size else None
             if ctx_size is None:
                 ctx_size = self.llm.get_ctx_size()
@@ -543,7 +567,11 @@ class LollmsClient():
                 streaming_callback=streaming_callback if streaming_callback is not None else self.llm.default_streaming_callback,
                 split= split,
                 user_keyword=user_keyword,
-                ai_keyword=ai_keyword
+                ai_keyword=ai_keyword,
+                think=think,
+                reasoning_effort = reasoning_effort,
+                reasoning_summary=reasoning_summary
+                
             )
         raise RuntimeError("LLM binding not initialized.")
 
@@ -560,6 +588,9 @@ class LollmsClient():
                      n_threads: Optional[int] = None,
                      ctx_size: int | None = None,
                      streaming_callback: Optional[Callable[[str, MSG_TYPE], None]] = None,
+                     think: Optional[bool] = False,
+                     reasoning_effort: Optional[bool] = "low", # low, medium, high
+                     reasoning_summary: Optional[bool] = "auto", # auto
                      **kwargs
                      ) -> Union[str, dict]:
         """
@@ -596,25 +627,31 @@ class LollmsClient():
                 n_threads=n_threads if n_threads is not None else self.llm.default_n_threads,
                 ctx_size = ctx_size if ctx_size is not None else self.llm.default_ctx_size,
                 streaming_callback=streaming_callback if streaming_callback is not None else self.llm.default_streaming_callback,
+                think=think,
+                reasoning_effort=reasoning_effort,
+                reasoning_summary=reasoning_summary
             )
         raise RuntimeError("LLM binding not initialized.")
 
     def chat(self,
-             discussion: LollmsDiscussion,
-             branch_tip_id: Optional[str] = None,
-             n_predict: Optional[int] = None,
-             stream: Optional[bool] = None,
-             temperature: Optional[float] = None,
-             top_k: Optional[int] = None,
-             top_p: Optional[float] = None,
-             repeat_penalty: Optional[float] = None,
-             repeat_last_n: Optional[int] = None,
-             seed: Optional[int] = None,
-             n_threads: Optional[int] = None,
-             ctx_size: Optional[int] = None,
-             streaming_callback: Optional[Callable[[str, MSG_TYPE, Dict], bool]] = None,
-             **kwargs
-             ) -> Union[str, dict]:
+            discussion: LollmsDiscussion,
+            branch_tip_id: Optional[str] = None,
+            n_predict: Optional[int] = None,
+            stream: Optional[bool] = None,
+            temperature: Optional[float] = None,
+            top_k: Optional[int] = None,
+            top_p: Optional[float] = None,
+            repeat_penalty: Optional[float] = None,
+            repeat_last_n: Optional[int] = None,
+            seed: Optional[int] = None,
+            n_threads: Optional[int] = None,
+            ctx_size: Optional[int] = None,
+            streaming_callback: Optional[Callable[[str, MSG_TYPE, Dict], bool]] = None,
+            think: Optional[bool] = False,
+            reasoning_effort: Optional[bool] = "low", # low, medium, high
+            reasoning_summary: Optional[bool] = "auto", # auto
+            **kwargs
+            ) -> Union[str, dict]:
         """
         High-level method to perform a chat generation using a LollmsDiscussion object.
 
@@ -654,7 +691,10 @@ class LollmsClient():
                 seed=seed if seed is not None else self.llm.default_seed,
                 n_threads=n_threads if n_threads is not None else self.llm.default_n_threads,
                 ctx_size = ctx_size if ctx_size is not None else self.llm.default_ctx_size,
-                streaming_callback=streaming_callback if streaming_callback is not None else self.llm.default_streaming_callback
+                streaming_callback=streaming_callback if streaming_callback is not None else self.llm.default_streaming_callback,
+                think = think,
+                reasoning_effort = reasoning_effort,
+                reasoning_summary = reasoning_summary,
             )
         raise RuntimeError("LLM binding not initialized.")
 
@@ -674,10 +714,10 @@ class LollmsClient():
         raise RuntimeError("LLM binding not initialized.")
 
 
-    def listModels(self):
+    def list_models(self):
         """Lists models available to the current LLM binding."""
         if self.llm:
-            return self.llm.listModels()
+            return self.llm.list_models()
         raise RuntimeError("LLM binding not initialized.")
 
     # --- Convenience Methods for Lollms LLM Binding Features ---
@@ -709,7 +749,11 @@ class LollmsClient():
                         repeat_penalty=None,
                         repeat_last_n=None,
                         callback=None,
-                        debug=False
+                        think: Optional[bool] = False,
+                        reasoning_effort: Optional[bool] = "low", # low, medium, high
+                        reasoning_summary: Optional[bool] = "auto", # auto
+                        debug=False,
+                        **kwargs
                         ):
         """
         Generates multiple code blocks based on a prompt.
@@ -751,7 +795,10 @@ Don't forget encapsulate the code inside a html code tag. This is mandatory.
             top_p=top_p,
             repeat_penalty=repeat_penalty,
             repeat_last_n=repeat_last_n,
-            streaming_callback=callback # Assuming generate_text handles streaming callback
+            streaming_callback=callback, # Assuming generate_text handles streaming callback
+            think=think,
+            reasoning_effort = reasoning_effort,
+            reasoning_summary=reasoning_summary
             )
 
         if isinstance(response, dict) and not response.get("status", True): # Check for error dict
@@ -1408,44 +1455,6 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
         )
         new_scratchpad_text = self.generate_text(prompt=synthesis_prompt, n_predict=1024, temperature=0.0)
         return self.remove_thinking_blocks(new_scratchpad_text).strip()
-    def _synthesize_knowledge(
-        self,
-        previous_scratchpad: str,
-        tool_name: str,
-        tool_params: dict,
-        tool_result: dict
-    ) -> str:
-        """
-        A dedicated LLM call to interpret a tool's output and update the knowledge scratchpad.
-        """
-        # Sanitize tool_result for LLM to avoid sending large binary/base64 data
-        sanitized_result = tool_result.copy()
-        if 'image_path' in sanitized_result:
-            sanitized_result['summary'] = f"An image was successfully generated and saved to '{sanitized_result['image_path']}'."
-            # Remove keys that might contain large data if they exist
-            sanitized_result.pop('image_base64', None)
-        elif 'file_path' in sanitized_result and 'content' in sanitized_result:
-             sanitized_result['summary'] = f"Content was successfully written to '{sanitized_result['file_path']}'."
-             sanitized_result.pop('content', None)
-
-
-        synthesis_prompt = (
-            "You are a data analyst assistant. Your sole job is to interpret the output of a tool and integrate it into the existing research summary (knowledge scratchpad).\n\n"
-            "--- PREVIOUS KNOWLEDGE SCRATCHPAD ---\n"
-            f"{previous_scratchpad}\n\n"
-            "--- ACTION JUST TAKEN ---\n"
-            f"Tool Called: `{tool_name}`\n"
-            f"Parameters: {json.dumps(tool_params)}\n\n"
-            "--- RAW TOOL OUTPUT ---\n"
-            f"```json\n{json.dumps(sanitized_result, indent=2)}\n```\n\n"
-            "--- YOUR TASK ---\n"
-            "Read the 'RAW TOOL OUTPUT' and explain what it means in plain language. Then, integrate this new information with the 'PREVIOUS KNOWLEDGE SCRATCHPAD' to create a new, complete, and self-contained summary.\n"
-            "Your output should be ONLY the text of the new scratchpad, with no extra commentary or formatting.\n\n"
-            "--- NEW KNOWLEDGE SCRATCHPAD ---\n"
-        )
-        new_scratchpad_text = self.generate_text(prompt=synthesis_prompt, n_predict=1024, temperature=0.0)
-        return self.remove_thinking_blocks(new_scratchpad_text).strip()
-    
 
     def _get_friendly_action_description(self, tool_name: str, requires_code: bool, requires_image: bool) -> str:
             """Convert technical tool names to user-friendly descriptions for logging."""
@@ -1458,7 +1467,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                 return "🎨 Creating an image based on your request"
             
             # Handle RAG (data store) tools by their pattern
-            elif "research::" in tool_name:
+            elif "rag::" in tool_name:
                 # Extract the friendly name of the data source
                 source_name = tool_name.split("::")[-1].replace("_", " ").title()
                 return f"🔍 Searching {source_name} for relevant information"
@@ -1474,7 +1483,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                 # Clean up the technical tool name for a more readable display
                 clean_name = tool_name.replace("_", " ").replace("::", " - ").title()
                 return f"🔧 Using the {clean_name} tool"
-            
+
     def generate_with_mcp_rag(
             self,
             prompt: str,
@@ -1509,7 +1518,8 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
             final_answer_temperature=0.7
         if rag_top_k is None:
             rag_top_k=5
-
+            
+        tools_infos = []
         def log_event(desc, event_type=MSG_TYPE.MSG_TYPE_CHUNK, meta=None, event_id=None) -> Optional[str]:
             if not streaming_callback: return None
             is_start = event_type == MSG_TYPE.MSG_TYPE_STEP_START
@@ -1536,38 +1546,44 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
             mcp_tools = self.mcp.discover_tools(force_refresh=True)
             if isinstance(use_mcps, list): 
                 filtered_tools = [t for t in mcp_tools if t["name"] in use_mcps]
+                tools_infos+=[f"    🛠️{f['name']}" for f in filtered_tools]
                 all_discovered_tools.extend(filtered_tools)
                 log_event(f"  ✅ Loaded {len(filtered_tools)} specific MCP tools: {', '.join(use_mcps)}", MSG_TYPE.MSG_TYPE_INFO)
             elif use_mcps is True: 
+                tools_infos+=[f"    🛠️{f['name']}" for f in mcp_tools]
                 all_discovered_tools.extend(mcp_tools)
                 log_event(f"  ✅ Loaded {len(mcp_tools)} MCP tools", MSG_TYPE.MSG_TYPE_INFO)
         
         if use_data_store:
             log_event(f"  📚 Setting up {len(use_data_store)} knowledge bases...", MSG_TYPE.MSG_TYPE_INFO)
             for name, info in use_data_store.items():
-                tool_name, description, call_fn = f"research::{name}", f"Queries the '{name}' knowledge base.", None
+                ASCIIColors.info(f"use_data_store item:\n{name}\n{info}")
+                tool_name, description, call_fn = f"rag::{name}", f"Queries the '{name}' knowledge base.", None
                 if callable(info): call_fn = info
                 elif isinstance(info, dict):
                     if "callable" in info and callable(info["callable"]): call_fn = info["callable"]
-                    description = info.get("description", description)
+                    description = info.get("description", "This is a datastore with the following description: \n" + description)
                 if call_fn:
                     visible_tools.append({"name": tool_name, "description": description, "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}})
                     rag_registry[tool_name] = call_fn
                     rag_tool_specs[tool_name] = {"default_top_k": rag_top_k, "default_min_sim": rag_min_similarity_percent}
-                    log_event(f"    📖 Ready: {name}", MSG_TYPE.MSG_TYPE_INFO)
-        
+                    tools_infos.append(f"    📖 {name}")
         visible_tools.extend(all_discovered_tools)
         built_in_tools = [
             {"name": "local_tools::final_answer", "description": "Provide the final answer directly to the user.", "input_schema": {}},
             {"name": "local_tools::request_clarification", "description": "Ask the user for more specific information when the request is ambiguous.", "input_schema": {"type": "object", "properties": {"question": {"type": "string"}}, "required": ["question"]}},
             {"name": "local_tools::revise_plan", "description": "Update the execution plan based on new discoveries or changing requirements.", "input_schema": {"type": "object", "properties": {"reason": {"type": "string"}, "new_plan": {"type": "array"}}, "required": ["reason", "new_plan"]}}
         ]
+        tools_infos+=[f"    🔨 final_answer","    🔨 request_clarification","    🔨 revise_plan"]
+        
+        
         if getattr(self, "tti", None): 
             built_in_tools.append({"name": "local_tools::generate_image", "description": "Generate an image from a text description.", "input_schema": {"type": "object", "properties": {"prompt": {"type": "string"}}, "required": ["prompt"]}})
         
         all_visible_tools = visible_tools + built_in_tools
         tool_summary = "\n".join([f"- **{t['name']}**: {t['description']}" for t in all_visible_tools[:20]])
         
+        log_event("\n".join(tools_infos), MSG_TYPE.MSG_TYPE_INFO)
         log_event(f"✅ Ready with {len(all_visible_tools)} total capabilities", MSG_TYPE.MSG_TYPE_STEP_END, event_id=discovery_step_id, meta={"tool_count": len(all_visible_tools), "mcp_tools": len(all_discovered_tools), "rag_tools": len(rag_registry)})
 
         # Enhanced triage with better prompting
@@ -1577,7 +1593,8 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
         try:
             triage_prompt = f"""Analyze this user request to determine the most efficient execution strategy.
 
-USER REQUEST: "{prompt}"
+DISCUSSION and Final Prompt:
+{prompt}
 CONTEXT: {context or "No additional context provided"}
 IMAGES PROVIDED: {"Yes" if images else "No"}
 
@@ -1587,7 +1604,7 @@ AVAILABLE CAPABILITIES:
 Based on the request complexity and available tools, choose the optimal strategy:
 
 1. **DIRECT_ANSWER**: For simple greetings, basic questions, or requests that don't require any tools
-   - Use when: The request can be fully answered with your existing knowledge
+   - Use when: The request can be fully answered with your existing knowledge with confidence, and no tool seems to add any significant value to the answer
    - Example: "Hello", "What is Python?", "Explain quantum physics"
 
 2. **REQUEST_CLARIFICATION**: When the request is too vague or ambiguous
@@ -1603,20 +1620,18 @@ Based on the request complexity and available tools, choose the optimal strategy
    - Example: "Research X, then create a report comparing it to Y"
 
 Provide your analysis in JSON format:
-{{"thought": "Detailed reasoning about the request complexity and requirements", "strategy": "ONE_OF_THE_FOUR_OPTIONS", "confidence": 0.8, "text_output": "Direct answer or clarification question if applicable", "required_tool_name": "specific tool name if SINGLE_TOOL strategy", "estimated_steps": 3}}"""
-            
-            log_prompt("Triage Prompt", triage_prompt)
+{{"thought": "Detailed reasoning about the request complexity and requirements", "strategy": "ONE_OF_THE_FOUR_OPTIONS", "confidence": percentage float value, eg 80, "text_output": "Direct answer or clarification question if applicable", "required_tool_name": "specific tool name if SINGLE_TOOL strategy", "estimated_steps": 3}}"""
             
             triage_schema = {
                 "thought": "string", "strategy": "string", "confidence": "number",
                 "text_output": "string", "required_tool_name": "string", "estimated_steps": "number"
             }
-            strategy_data = self.generate_structured_content(prompt=triage_prompt, schema=triage_schema, temperature=0.1, **llm_generation_kwargs)
+            strategy_data = self.generate_structured_content(prompt=triage_prompt, schema=triage_schema, temperature=0.1, system_prompt=system_prompt, **llm_generation_kwargs)
             strategy = strategy_data.get("strategy") if strategy_data else "COMPLEX_PLAN"
             
-            log_event(f"Strategy analysis complete", MSG_TYPE.MSG_TYPE_INFO, meta={
+            log_event(f"Strategy analysis complete.\n**confidence**: {strategy_data.get('confidence', 0.5)}\n**reasoning**: {strategy_data.get('thought', 'None')}", MSG_TYPE.MSG_TYPE_INFO, meta={
                 "strategy": strategy,
-                "confidence": strategy_data.get("confidence", 0.5),
+                "confidence": strategy_data.get("confidence", 50),
                 "estimated_steps": strategy_data.get("estimated_steps", 1),
                 "reasoning": strategy_data.get("thought", "")
             })
@@ -1653,1553 +1668,90 @@ Provide your analysis in JSON format:
                     raise ValueError(f"Strategy analysis selected unavailable tool: '{tool_name}'")
                 
                 log_event(f"Selected tool: {tool_name}", MSG_TYPE.MSG_TYPE_INFO)
-                
-                # Enhanced parameter generation prompt
-                param_prompt = f"""Generate the optimal parameters for the selected tool to fulfill the user's request.
-
-USER REQUEST: "{prompt}"
-SELECTED TOOL: {json.dumps(tool_spec, indent=2)}
-CONTEXT: {context or "None"}
-
-Analyze the user's request carefully and provide the most appropriate parameters.
-If the request has implicit requirements, infer them intelligently.
-
-Output the parameters as JSON: {{"tool_params": {{...}}}}"""
-                
-                log_prompt("Parameter Generation Prompt", param_prompt)
-                param_data = self.generate_structured_content(prompt=param_prompt, schema={"tool_params": "object"}, temperature=0.1, **llm_generation_kwargs)
-                tool_params = param_data.get("tool_params", {}) if param_data else {}
-                
-                log_event(f"Generated parameters: {json.dumps(tool_params)}", MSG_TYPE.MSG_TYPE_INFO)
-                
-                start_time, sources, tool_result = time.time(), [], {}
-                if tool_name in rag_registry:
-                    query = tool_params.get("query", prompt)
-                    log_event(f"Searching knowledge base with query: '{query}'", MSG_TYPE.MSG_TYPE_INFO)
-                    rag_fn = rag_registry[tool_name]
-                    raw_results = rag_fn(query=query, rag_top_k=rag_top_k, rag_min_similarity_percent=rag_min_similarity_percent)
-                    docs = [d for d in (raw_results.get("results", []) if isinstance(raw_results, dict) else raw_results or [])]
-                    tool_result = {"status": "success", "results": docs}
-                    sources = [{"source": tool_name, "metadata": d.get("metadata", {}), "score": d.get("score", 0.0)} for d in docs]
-                    log_event(f"Retrieved {len(docs)} relevant documents", MSG_TYPE.MSG_TYPE_INFO)
-                elif hasattr(self, "mcp") and "local_tools" not in tool_name:
-                    log_event(f"Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={"tool_name": tool_name, "params": tool_params})
-                    tool_result = self.mcp.execute_tool(tool_name, tool_params, lollms_client_instance=self)
-                    log_event(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={"result_status": tool_result.get("status", "unknown")})
-                else:
-                    tool_result = {"status": "failure", "error": f"Tool '{tool_name}' could not be executed in single-step mode."}
-                
-                if tool_result.get("status","success") != "success" or "error" in tool_result:
-                    error_detail = tool_result.get("error", "Unknown tool error in single-step mode.")
-                    raise RuntimeError(error_detail)
-
-                response_time = time.time() - start_time
-                tool_calls_this_turn = [{"name": tool_name, "params": tool_params, "result": tool_result, "response_time": response_time}]
-                
-                # Enhanced synthesis prompt
-                synthesis_prompt = f"""Create a comprehensive and user-friendly response based on the tool execution results.
-
-USER REQUEST: "{prompt}"
-TOOL USED: {tool_name}
-TOOL RESULT: {json.dumps(tool_result, indent=2)}
-
-Guidelines for your response:
-1. Be direct and helpful
-2. Synthesize the information clearly
-3. Address the user's specific needs
-4. If the tool provided data, present it in an organized way
-5. If relevant, mention any limitations or additional context
-
-RESPONSE:"""
-                
-                log_event("Synthesizing final response", MSG_TYPE.MSG_TYPE_INFO)
-                final_answer = self.generate_text(prompt=synthesis_prompt, system_prompt=system_prompt, stream=streaming_callback is not None, streaming_callback=streaming_callback, temperature=final_answer_temperature, **llm_generation_kwargs)
-                final_answer = self.remove_thinking_blocks(final_answer)
-                
-                log_event("✅ Single-tool execution completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id)
-                return {"final_answer": final_answer, "tool_calls": tool_calls_this_turn, "sources": sources, "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: SINGLE_TOOL\nTool: {tool_name}\nResult: Success\nResponse Time: {response_time:.2f}s"}
-
-            except Exception as e:
-                log_event(f"Single-tool execution failed: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=synthesis_id)
-                log_event("Escalating to complex planning approach", MSG_TYPE.MSG_TYPE_INFO)
-
-        # Execute complex reasoning with enhanced capabilities
-        return self._execute_complex_reasoning_loop(
-            prompt=prompt, context=context, system_prompt=system_prompt,
-            reasoning_system_prompt=reasoning_system_prompt, images=images,
-            max_reasoning_steps=max_reasoning_steps, decision_temperature=decision_temperature,
-            final_answer_temperature=final_answer_temperature, streaming_callback=streaming_callback,
-            debug=debug, enable_self_reflection=enable_self_reflection,
-            all_visible_tools=all_visible_tools, rag_registry=rag_registry, rag_tool_specs=rag_tool_specs,
-            log_event_fn=log_event, log_prompt_fn=log_prompt, max_scratchpad_size=max_scratchpad_size,
-            **llm_generation_kwargs
-        )
-
-    def _execute_complex_reasoning_loop(
-        self, prompt, context, system_prompt, reasoning_system_prompt, images,
-        max_reasoning_steps, decision_temperature, final_answer_temperature,
-        streaming_callback, debug, enable_self_reflection, all_visible_tools,
-        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
-    ) -> Dict[str, Any]:
-        
-        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
-        
-        def _get_friendly_action_description(tool_name, requires_code, requires_image):
-            descriptions = {
-                "local_tools::final_answer": "📋 Preparing final answer",
-                "local_tools::request_clarification": "❓ Requesting clarification",
-                "local_tools::generate_image": "🎨 Creating image",
-                "local_tools::revise_plan": "📝 Revising execution plan"
-            }
-            if tool_name in descriptions:
-                return descriptions[tool_name]
-            if "research::" in tool_name: 
-                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
-            if requires_code: 
-                return "💻 Processing code"
-            if requires_image: 
-                return "🖼️ Analyzing images"
-            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
-
-        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
-            """Enhanced scratchpad compression that preserves key decisions and recent context"""
-            if len(scratchpad) <= target_size:
-                return scratchpad
-            
-            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
-            
-            # Extract key components
-            lines = scratchpad.split('\n')
-            plan_section = []
-            decisions = []
-            recent_observations = []
-            
-            current_section = None
-            for i, line in enumerate(lines):
-                if "### Execution Plan" in line or "### Updated Plan" in line:
-                    current_section = "plan"
-                elif "### Step" in line and ("Thought" in line or "Decision" in line):
-                    current_section = "decision"
-                elif "### Step" in line and "Observation" in line:
-                    current_section = "observation"
-                elif line.startswith("###"):
-                    current_section = None
-                
-                if current_section == "plan" and line.strip():
-                    plan_section.append(line)
-                elif current_section == "decision" and line.strip():
-                    decisions.append((i, line))
-                elif current_section == "observation" and line.strip():
-                    recent_observations.append((i, line))
-            
-            # Keep most recent items and important decisions
-            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
-            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
-            
-            compressed_parts = [
-                f"### Original Request\n{original_request}",
-                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
-                f"### Recent Key Decisions"
-            ]
-            
-            for _, decision in recent_decisions:
-                compressed_parts.append(decision)
-            
-            compressed_parts.append("### Recent Observations")
-            for _, obs in recent_obs:
-                compressed_parts.append(obs)
-            
-            compressed = '\n'.join(compressed_parts)
-            if len(compressed) > target_size:
-                # Final trim if still too long
-                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
-            
-            return compressed
-
-        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
-        asset_store: Dict[str, Dict] = {}
-        decision_history = []  # Track all decisions made
-        
-        # Enhanced planning phase
-        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
-        execution_plan = planner.decompose_task(original_user_prompt, context or "")
-        current_plan_version = 1
-        
-        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
-            "plan_version": current_plan_version,
-            "total_tasks": len(execution_plan.tasks),
-            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
-        })
-        
-        for i, task in enumerate(execution_plan.tasks):
-            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
-            
-        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
-        
-        # Enhanced initial state
-        initial_state_parts = [
-            f"### Original User Request\n{original_user_prompt}",
-            f"### Context\n{context or 'No additional context provided'}",
-            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
-            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
-        ]
-        
-        for i, task in enumerate(execution_plan.tasks): 
-            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
-        
-        if images:
-            initial_state_parts.append(f"### Provided Assets")
-            for img_b64 in images:
-                img_uuid = str(uuid.uuid4())
-                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
-                initial_state_parts.append(f"- Image asset: {img_uuid}")
-        
-        current_scratchpad = "\n".join(initial_state_parts)
-        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
-
-        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
-        completed_tasks, current_task_index = set(), 0
-        plan_revision_count = 0
-        
-        # Main reasoning loop with enhanced decision tracking
-        for i in range(max_reasoning_steps):
-            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
-            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
-            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
-            
-            try:
-                # Enhanced scratchpad management
-                if len(current_scratchpad) > max_scratchpad_size:
-                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
-                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
-                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
-                
-                # Enhanced reasoning prompt with better decision tracking
-                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
-
-=== AVAILABLE ACTIONS ===
-{formatted_tools_list}
-
-=== YOUR COMPLETE ANALYSIS HISTORY ===
-{current_scratchpad}
-=== END ANALYSIS HISTORY ===
-
-=== DECISION GUIDELINES ===
-1. **Review your progress**: Look at what you've already discovered and accomplished
-2. **Consider your current task**: Focus on the next logical step in your plan
-3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
-4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
-5. **Stay focused**: Each action should clearly advance toward the final goal
-
-=== YOUR NEXT DECISION ===
-Choose the single most appropriate action to take right now. Consider:
-- What specific step are you currently working on?
-- What information do you still need?
-- What would be most helpful for the user?
-
-Provide your decision as JSON:
-{{
-    "reasoning": "Explain your current thinking and why this action makes sense now",
-    "action": {{
-        "tool_name": "exact_tool_name",
-        "requires_code_input": false,
-        "requires_image_input": false,
-        "confidence": 0.8
-    }},
-    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
-}}"""
-
-                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
-                decision_data = self.generate_structured_content(
-                    prompt=reasoning_prompt, 
-                    schema={
-                        "reasoning": "string", 
-                        "action": "object",
-                        "plan_status": "string"
-                    }, 
-                    system_prompt=reasoning_system_prompt, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                
-                if not (decision_data and isinstance(decision_data.get("action"), dict)):
-                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
-                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
-                    continue
-
-                action = decision_data.get("action", {})
-                reasoning = decision_data.get("reasoning", "No reasoning provided")
-                plan_status = decision_data.get("plan_status", "on_track")
-                tool_name = action.get("tool_name")
-                requires_code = action.get("requires_code_input", False)
-                requires_image = action.get("requires_image_input", False)
-                confidence = action.get("confidence", 0.5)
-                
-                # Track the decision
-                decision_history.append({
-                    "step": i+1,
-                    "tool_name": tool_name,
-                    "reasoning": reasoning,
-                    "confidence": confidence,
-                    "plan_status": plan_status
-                })
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
-                
-                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
-                    "tool_name": tool_name,
-                    "confidence": confidence,
-                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
-                })
-                
-                # Handle plan revision
-                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
-                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
-                    tool_name = "local_tools::revise_plan"  # Force plan revision
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-
-                # Handle plan revision
-                if tool_name == "local_tools::revise_plan":
-                    plan_revision_count += 1
-                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
-                    
-                    try:
-                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-REASON FOR REVISION: {reasoning}
-
-Create a new plan that reflects your current understanding. Consider:
-1. What have you already accomplished?
-2. What new information have you discovered?
-3. What steps are still needed?
-4. How can you be more efficient?
-
-Provide your revision as JSON:
-{{
-    "revision_reason": "Clear explanation of why the plan needed to change",
-    "new_plan": [
-        {{"step": 1, "description": "First revised step", "status": "pending"}},
-        {{"step": 2, "description": "Second revised step", "status": "pending"}}
-    ],
-    "confidence": 0.8
-}}"""
-
-                        revision_data = self.generate_structured_content(
-                            prompt=revision_prompt,
-                            schema={
-                                "revision_reason": "string",
-                                "new_plan": "array", 
-                                "confidence": "number"
-                            },
-                            temperature=0.3,
-                            **llm_generation_kwargs
-                        )
-                        
-                        if revision_data and revision_data.get("new_plan"):
-                            # Update the plan
-                            current_plan_version += 1
-                            new_tasks = []
-                            for task_data in revision_data["new_plan"]:
-                                task = TaskDecomposition()  # Assuming this class exists
-                                task.description = task_data.get("description", "Undefined step")
-                                task.status = TaskStatus.PENDING  # Reset all to pending
-                                new_tasks.append(task)
-                            
-                            execution_plan.tasks = new_tasks
-                            current_task_index = 0  # Reset to beginning
-                            
-                            # Update scratchpad with new plan
-                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
-                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
-                            current_scratchpad += f"**New Tasks**:\n"
-                            for i, task in enumerate(execution_plan.tasks):
-                                current_scratchpad += f"  {i+1}. {task.description}\n"
-                            
-                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
-                                "plan_version": current_plan_version,
-                                "new_task_count": len(execution_plan.tasks),
-                                "revision_reason": revision_data.get("revision_reason", "")
-                            })
-                            
-                            # Continue with the new plan
-                            continue
-                        else:
-                            raise ValueError("Failed to generate valid plan revision")
-                            
-                    except Exception as e:
-                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
-                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
-
-                # Prepare parameters for tool execution
-                param_assets = {}
-                if requires_code:
-                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
-                    code_prompt = f"""Generate the specific code needed for the current step.
-
-CURRENT CONTEXT:
-{current_scratchpad}
-
-CURRENT TASK: {tool_name}
-USER REQUEST: "{original_user_prompt}"
-
-Generate clean, functional code that addresses the specific requirements. Focus on:
-1. Solving the immediate problem
-2. Being clear and readable
-3. Including necessary imports and dependencies
-4. Adding helpful comments where appropriate
-
-CODE:"""
-
-                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
-                    code_uuid = f"code_asset_{uuid.uuid4()}"
-                    asset_store[code_uuid] = {"type": "code", "content": code_content}
-                    param_assets['code_asset_id'] = code_uuid
-                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    
-                if requires_image:
-                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
-                    if image_assets:
-                        param_assets['image_asset_id'] = image_assets[0]
-                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    else:
-                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
-
-                # Enhanced parameter generation
-                param_prompt = f"""Generate the optimal parameters for this tool execution.
-
-TOOL: {tool_name}
-CURRENT CONTEXT: {current_scratchpad}
-CURRENT REASONING: {reasoning}
-AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
-
-Based on your analysis and the current step you're working on, provide the most appropriate parameters.
-Be specific and purposeful in your parameter choices.
-
-Output format: {{"tool_params": {{...}}}}"""
-
-                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
-                param_data = self.generate_structured_content(
-                    prompt=param_prompt, 
-                    schema={"tool_params": "object"}, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                tool_params = param_data.get("tool_params", {}) if param_data else {}
-                
-                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
-                
-                # Hydrate parameters with assets
-                def _hydrate(data: Any, store: Dict) -> Any:
-                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
-                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
-                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
-                    return data
-                    
-                hydrated_params = _hydrate(tool_params, asset_store)
-                
-                # Execute the tool with detailed logging
-                start_time = time.time()
-                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
-                
-                try:
-                    if tool_name in rag_registry:
-                        query = hydrated_params.get("query", "")
-                        if not query:
-                            # Fall back to using reasoning as query
-                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
-                            
-                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        top_k = rag_tool_specs[tool_name]["default_top_k"]
-                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
-                        
-                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
-                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
-                        
-                        docs = []
-                        for d in raw_iter or []:
-                            doc_data = {
-                                "text": d.get("text", str(d)), 
-                                "score": d.get("score", 0) * 100, 
-                                "metadata": d.get("metadata", {})
-                            }
-                            docs.append(doc_data)
-                        
-                        kept = [x for x in docs if x['score'] >= min_sim]
-                        tool_result = {
-                            "status": "success", 
-                            "results": kept, 
-                            "total_found": len(docs),
-                            "kept_after_filtering": len(kept),
-                            "query_used": query
-                        }
-                        
-                        sources_this_turn.extend([{
-                            "source": tool_name, 
-                            "metadata": x["metadata"], 
-                            "score": x["score"]
-                        } for x in kept])
-                        
-                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
-                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
-                            "tool_name": tool_name,
-                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
-                        })
-                        
-                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
-                        
-                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
-                            "result_status": tool_result.get("status", "unknown"),
-                            "has_error": "error" in tool_result
-                        })
-                        
-                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
-                        image_prompt = hydrated_params.get("prompt", "")
-                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        # This would call your text-to-image functionality
-                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
-                        if image_result:
-                            image_uuid = f"generated_image_{uuid.uuid4()}"
-                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
-                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
-                        else:
-                            tool_result = {"status": "failure", "error": "Image generation failed"}
-                            
-                    else:
-                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
-                        
-                except Exception as e:
-                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
-                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
-                    tool_result = {"status": "failure", "error": error_msg}
-
-                response_time = time.time() - start_time
-                success = tool_result.get("status") == "success"
-                
-                # Record performance
-                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
-                
-                # Update task status
-                if success and current_task_index < len(execution_plan.tasks):
-                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
-                    completed_tasks.add(current_task_index)
-                    current_task_index += 1
-                
-                # Enhanced observation logging
-                observation_text = json.dumps(tool_result, indent=2)
-                if len(observation_text) > 1000:
-                    # Truncate very long results for scratchpad
-                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
-                    observation_text = json.dumps(truncated_result, indent=2)
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
-                current_scratchpad += f"**Tool Used**: {tool_name}\n"
-                current_scratchpad += f"**Success**: {success}\n"
-                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
-                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
-                
-                # Track tool call
-                tool_calls_this_turn.append({
-                    "name": tool_name,
-                    "params": tool_params,
-                    "result": tool_result,
-                    "response_time": response_time,
-                    "confidence": confidence,
-                    "reasoning": reasoning
-                })
-                
-                if success:
-                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "response_time": response_time,
-                        "confidence": confidence
-                    })
-                else:
-                    error_detail = tool_result.get("error", "No error detail provided.")
-                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "error": error_detail,
-                        "confidence": confidence
-                    })
-                    
-                    # Add failure handling to scratchpad
-                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
-                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
-                
-                # Log current progress
-                completed_count = len(completed_tasks)
-                total_tasks = len(execution_plan.tasks)
-                if total_tasks > 0:
-                    progress = (completed_count / total_tasks) * 100
-                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
-                
-                # Check if all tasks are completed
-                if completed_count >= total_tasks:
-                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                    
-            except Exception as ex:
-                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
-                trace_exception(ex)
-                
-                # Add error to scratchpad for context
-                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
-                
-                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
-
-        # Enhanced self-reflection
-        if enable_self_reflection and len(tool_calls_this_turn) > 0:
-            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
-            try:
-                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-TOOLS USED: {len(tool_calls_this_turn)}
-PLAN REVISIONS: {plan_revision_count}
-
-COMPLETE ANALYSIS:
-{current_scratchpad}
-
-Evaluate your performance on multiple dimensions:
-
-1. **Goal Achievement**: Did you fully address the user's request?
-2. **Process Efficiency**: Was your approach optimal given the available tools?
-3. **Information Quality**: Is the information you gathered accurate and relevant?
-4. **Decision Making**: Were your tool choices and parameters appropriate?
-5. **Adaptability**: How well did you handle unexpected results or plan changes?
-
-Provide your assessment as JSON:
-{{
-    "goal_achieved": true,
-    "effectiveness_score": 0.85,
-    "process_efficiency": 0.8,
-    "information_quality": 0.9,
-    "decision_making": 0.85,
-    "adaptability": 0.7,
-    "overall_confidence": 0.82,
-    "strengths": ["Clear reasoning", "Good tool selection"],
-    "areas_for_improvement": ["Could have been more efficient"],
-    "summary": "Successfully completed the user's request with high quality results",
-    "key_insights": ["Discovered that X was more important than initially thought"]
-}}"""
-
-                reflection_data = self.generate_structured_content(
-                    prompt=reflection_prompt,
-                    schema={
-                        "goal_achieved": "boolean",
-                        "effectiveness_score": "number",
-                        "process_efficiency": "number", 
-                        "information_quality": "number",
-                        "decision_making": "number",
-                        "adaptability": "number",
-                        "overall_confidence": "number",
-                        "strengths": "array",
-                        "areas_for_improvement": "array",
-                        "summary": "string",
-                        "key_insights": "array"
-                    },
-                    temperature=0.3,
-                    **llm_generation_kwargs
-                )
-                
-                if reflection_data:
-                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
-                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
-                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
-                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
-                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
-                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
-                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
-                    
-                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
-                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
-                        "goal_achieved": reflection_data.get('goal_achieved', False),
-                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
-                    })
-                else:
-                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-                    
-            except Exception as e:
-                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-
-        # Enhanced final synthesis
-        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
-        
-        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CONTEXT: {context or "No additional context"}
-
-COMPLETE ANALYSIS AND WORK:
-{current_scratchpad}
-
-GUIDELINES for your response:
-1. **Be Complete**: Address all aspects of the user's request
-2. **Be Clear**: Organize your response logically and use clear language  
-3. **Be Helpful**: Provide actionable information and insights
-4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
-5. **Be Concise**: While being thorough, avoid unnecessary verbosity
-6. **Cite Sources**: If you used research tools, reference the information appropriately
-
-Your response should feel natural and conversational while being informative and valuable.
-
-FINAL RESPONSE:"""
-        
-        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
-        
-        final_answer_text = self.generate_text(
-            prompt=final_answer_prompt,
-            system_prompt=system_prompt,
-            stream=streaming_callback is not None,
-            streaming_callback=streaming_callback,
-            temperature=final_answer_temperature,
-            **llm_generation_kwargs
-        )
-        
-        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
-            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
-            return {
-                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
-                "error": final_answer_text["error"],
-                "final_scratchpad": current_scratchpad,
-                "tool_calls": tool_calls_this_turn,
-                "sources": sources_this_turn,
-                "decision_history": decision_history
-            }
-        
-        final_answer = self.remove_thinking_blocks(final_answer_text)
-        
-        # Calculate overall performance metrics
-        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
-        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
-        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
-        
-        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
-            "final_answer_length": len(final_answer),
-            "total_tools_used": len(tool_calls_this_turn),
-            "success_rate": success_rate,
-            "overall_confidence": overall_confidence
-        })
-
-        return {
-            "final_answer": final_answer,
-            "final_scratchpad": current_scratchpad,
-            "tool_calls": tool_calls_this_turn,
-            "sources": sources_this_turn,
-            "decision_history": decision_history,
-            "performance_stats": {
-                "total_steps": len(tool_calls_this_turn),
-                "successful_steps": successful_calls,
-                "success_rate": success_rate,
-                "average_confidence": overall_confidence,
-                "plan_revisions": plan_revision_count,
-                "total_reasoning_steps": len(decision_history)
-            },
-            "plan_evolution": {
-                "initial_tasks": len(execution_plan.tasks),
-                "final_version": current_plan_version,
-                "total_revisions": plan_revision_count
-            },
-            "clarification_required": False,
-            "overall_confidence": overall_confidence,
-            "error": None
-        }
-
-
-    def _execute_complex_reasoning_loop(
-        self, prompt, context, system_prompt, reasoning_system_prompt, images,
-        max_reasoning_steps, decision_temperature, final_answer_temperature,
-        streaming_callback, debug, enable_self_reflection, all_visible_tools,
-        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
-    ) -> Dict[str, Any]:
-        
-        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
-        
-        def _get_friendly_action_description(tool_name, requires_code, requires_image):
-            descriptions = {
-                "local_tools::final_answer": "📋 Preparing final answer",
-                "local_tools::request_clarification": "❓ Requesting clarification",
-                "local_tools::generate_image": "🎨 Creating image",
-                "local_tools::revise_plan": "📝 Revising execution plan"
-            }
-            if tool_name in descriptions:
-                return descriptions[tool_name]
-            if "research::" in tool_name: 
-                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
-            if requires_code: 
-                return "💻 Processing code"
-            if requires_image: 
-                return "🖼️ Analyzing images"
-            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
-
-        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
-            """Enhanced scratchpad compression that preserves key decisions and recent context"""
-            if len(scratchpad) <= target_size:
-                return scratchpad
-            
-            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
-            
-            # Extract key components
-            lines = scratchpad.split('\n')
-            plan_section = []
-            decisions = []
-            recent_observations = []
-            
-            current_section = None
-            for i, line in enumerate(lines):
-                if "### Execution Plan" in line or "### Updated Plan" in line:
-                    current_section = "plan"
-                elif "### Step" in line and ("Thought" in line or "Decision" in line):
-                    current_section = "decision"
-                elif "### Step" in line and "Observation" in line:
-                    current_section = "observation"
-                elif line.startswith("###"):
-                    current_section = None
-                
-                if current_section == "plan" and line.strip():
-                    plan_section.append(line)
-                elif current_section == "decision" and line.strip():
-                    decisions.append((i, line))
-                elif current_section == "observation" and line.strip():
-                    recent_observations.append((i, line))
-            
-            # Keep most recent items and important decisions
-            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
-            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
-            
-            compressed_parts = [
-                f"### Original Request\n{original_request}",
-                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
-                f"### Recent Key Decisions"
-            ]
-            
-            for _, decision in recent_decisions:
-                compressed_parts.append(decision)
-            
-            compressed_parts.append("### Recent Observations")
-            for _, obs in recent_obs:
-                compressed_parts.append(obs)
-            
-            compressed = '\n'.join(compressed_parts)
-            if len(compressed) > target_size:
-                # Final trim if still too long
-                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
-            
-            return compressed
-
-        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
-        asset_store: Dict[str, Dict] = {}
-        decision_history = []  # Track all decisions made
-        
-        # Enhanced planning phase
-        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
-        execution_plan = planner.decompose_task(original_user_prompt, context or "")
-        current_plan_version = 1
-        
-        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
-            "plan_version": current_plan_version,
-            "total_tasks": len(execution_plan.tasks),
-            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
-        })
-        
-        for i, task in enumerate(execution_plan.tasks):
-            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
-            
-        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
-        
-        # Enhanced initial state
-        initial_state_parts = [
-            f"### Original User Request\n{original_user_prompt}",
-            f"### Context\n{context or 'No additional context provided'}",
-            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
-            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
-        ]
-        
-        for i, task in enumerate(execution_plan.tasks): 
-            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
-        
-        if images:
-            initial_state_parts.append(f"### Provided Assets")
-            for img_b64 in images:
-                img_uuid = str(uuid.uuid4())
-                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
-                initial_state_parts.append(f"- Image asset: {img_uuid}")
-        
-        current_scratchpad = "\n".join(initial_state_parts)
-        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
-
-        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
-        completed_tasks, current_task_index = set(), 0
-        plan_revision_count = 0
-        
-        # Main reasoning loop with enhanced decision tracking
-        for i in range(max_reasoning_steps):
-            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
-            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
-            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
-            
-            try:
-                # Enhanced scratchpad management
-                if len(current_scratchpad) > max_scratchpad_size:
-                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
-                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
-                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
-                
-                # Enhanced reasoning prompt with better decision tracking
-                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
-
-=== AVAILABLE ACTIONS ===
-{formatted_tools_list}
-
-=== YOUR COMPLETE ANALYSIS HISTORY ===
-{current_scratchpad}
-=== END ANALYSIS HISTORY ===
-
-=== DECISION GUIDELINES ===
-1. **Review your progress**: Look at what you've already discovered and accomplished
-2. **Consider your current task**: Focus on the next logical step in your plan
-3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
-4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
-5. **Stay focused**: Each action should clearly advance toward the final goal
-
-=== YOUR NEXT DECISION ===
-Choose the single most appropriate action to take right now. Consider:
-- What specific step are you currently working on?
-- What information do you still need?
-- What would be most helpful for the user?
-
-Provide your decision as JSON:
-{{
-    "reasoning": "Explain your current thinking and why this action makes sense now",
-    "action": {{
-        "tool_name": "exact_tool_name",
-        "requires_code_input": false,
-        "requires_image_input": false,
-        "confidence": 0.8
-    }},
-    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
-}}"""
-
-                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
-                decision_data = self.generate_structured_content(
-                    prompt=reasoning_prompt, 
-                    schema={
-                        "reasoning": "string", 
-                        "action": "object",
-                        "plan_status": "string"
-                    }, 
-                    system_prompt=reasoning_system_prompt, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                
-                if not (decision_data and isinstance(decision_data.get("action"), dict)):
-                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
-                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
-                    continue
-
-                action = decision_data.get("action", {})
-                reasoning = decision_data.get("reasoning", "No reasoning provided")
-                plan_status = decision_data.get("plan_status", "on_track")
-                tool_name = action.get("tool_name")
-                requires_code = action.get("requires_code_input", False)
-                requires_image = action.get("requires_image_input", False)
-                confidence = action.get("confidence", 0.5)
-                
-                # Track the decision
-                decision_history.append({
-                    "step": i+1,
-                    "tool_name": tool_name,
-                    "reasoning": reasoning,
-                    "confidence": confidence,
-                    "plan_status": plan_status
-                })
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
-                
-                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
-                    "tool_name": tool_name,
-                    "confidence": confidence,
-                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
-                })
-                
-                # Handle plan revision
-                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
-                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
-                    tool_name = "local_tools::revise_plan"  # Force plan revision
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-
-                # Handle plan revision
-                if tool_name == "local_tools::revise_plan":
-                    plan_revision_count += 1
-                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
-                    
-                    try:
-                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-REASON FOR REVISION: {reasoning}
-
-Create a new plan that reflects your current understanding. Consider:
-1. What have you already accomplished?
-2. What new information have you discovered?
-3. What steps are still needed?
-4. How can you be more efficient?
-
-Provide your revision as JSON:
-{{
-    "revision_reason": "Clear explanation of why the plan needed to change",
-    "new_plan": [
-        {{"step": 1, "description": "First revised step", "status": "pending"}},
-        {{"step": 2, "description": "Second revised step", "status": "pending"}}
-    ],
-    "confidence": 0.8
-}}"""
-
-                        revision_data = self.generate_structured_content(
-                            prompt=revision_prompt,
-                            schema={
-                                "revision_reason": "string",
-                                "new_plan": "array", 
-                                "confidence": "number"
-                            },
-                            temperature=0.3,
-                            **llm_generation_kwargs
-                        )
-                        
-                        if revision_data and revision_data.get("new_plan"):
-                            # Update the plan
-                            current_plan_version += 1
-                            new_tasks = []
-                            for task_data in revision_data["new_plan"]:
-                                task = TaskDecomposition()  # Assuming this class exists
-                                task.description = task_data.get("description", "Undefined step")
-                                task.status = TaskStatus.PENDING  # Reset all to pending
-                                new_tasks.append(task)
-                            
-                            execution_plan.tasks = new_tasks
-                            current_task_index = 0  # Reset to beginning
-                            
-                            # Update scratchpad with new plan
-                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
-                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
-                            current_scratchpad += f"**New Tasks**:\n"
-                            for i, task in enumerate(execution_plan.tasks):
-                                current_scratchpad += f"  {i+1}. {task.description}\n"
-                            
-                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
-                                "plan_version": current_plan_version,
-                                "new_task_count": len(execution_plan.tasks),
-                                "revision_reason": revision_data.get("revision_reason", "")
-                            })
-                            
-                            # Continue with the new plan
-                            continue
-                        else:
-                            raise ValueError("Failed to generate valid plan revision")
-                            
-                    except Exception as e:
-                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
-                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
-
-                # Prepare parameters for tool execution
-                param_assets = {}
-                if requires_code:
-                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
-                    code_prompt = f"""Generate the specific code needed for the current step.
-
-CURRENT CONTEXT:
-{current_scratchpad}
-
-CURRENT TASK: {tool_name}
-USER REQUEST: "{original_user_prompt}"
-
-Generate clean, functional code that addresses the specific requirements. Focus on:
-1. Solving the immediate problem
-2. Being clear and readable
-3. Including necessary imports and dependencies
-4. Adding helpful comments where appropriate
-
-CODE:"""
-
-                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
-                    code_uuid = f"code_asset_{uuid.uuid4()}"
-                    asset_store[code_uuid] = {"type": "code", "content": code_content}
-                    param_assets['code_asset_id'] = code_uuid
-                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    
-                if requires_image:
-                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
-                    if image_assets:
-                        param_assets['image_asset_id'] = image_assets[0]
-                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    else:
-                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
-
-                # Enhanced parameter generation
-                param_prompt = f"""Generate the optimal parameters for this tool execution.
-
-TOOL: {tool_name}
-CURRENT CONTEXT: {current_scratchpad}
-CURRENT REASONING: {reasoning}
-AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
-
-Based on your analysis and the current step you're working on, provide the most appropriate parameters.
-Be specific and purposeful in your parameter choices.
-
-Output format: {{"tool_params": {{...}}}}"""
-
-                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
-                param_data = self.generate_structured_content(
-                    prompt=param_prompt, 
-                    schema={"tool_params": "object"}, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                tool_params = param_data.get("tool_params", {}) if param_data else {}
-                
-                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
-                
-                # Hydrate parameters with assets
-                def _hydrate(data: Any, store: Dict) -> Any:
-                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
-                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
-                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
-                    return data
-                    
-                hydrated_params = _hydrate(tool_params, asset_store)
-                
-                # Execute the tool with detailed logging
-                start_time = time.time()
-                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
-                
-                try:
-                    if tool_name in rag_registry:
-                        query = hydrated_params.get("query", "")
-                        if not query:
-                            # Fall back to using reasoning as query
-                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
-                            
-                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        top_k = rag_tool_specs[tool_name]["default_top_k"]
-                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
-                        
-                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
-                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
-                        
-                        docs = []
-                        for d in raw_iter or []:
-                            doc_data = {
-                                "text": d.get("text", str(d)), 
-                                "score": d.get("score", 0) * 100, 
-                                "metadata": d.get("metadata", {})
-                            }
-                            docs.append(doc_data)
-                        
-                        kept = [x for x in docs if x['score'] >= min_sim]
-                        tool_result = {
-                            "status": "success", 
-                            "results": kept, 
-                            "total_found": len(docs),
-                            "kept_after_filtering": len(kept),
-                            "query_used": query
-                        }
-                        
-                        sources_this_turn.extend([{
-                            "source": tool_name, 
-                            "metadata": x["metadata"], 
-                            "score": x["score"]
-                        } for x in kept])
-                        
-                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
-                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
-                            "tool_name": tool_name,
-                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
-                        })
-                        
-                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
-                        
-                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
-                            "result_status": tool_result.get("status", "unknown"),
-                            "has_error": "error" in tool_result
-                        })
-                        
-                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
-                        image_prompt = hydrated_params.get("prompt", "")
-                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        # This would call your text-to-image functionality
-                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
-                        if image_result:
-                            image_uuid = f"generated_image_{uuid.uuid4()}"
-                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
-                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
-                        else:
-                            tool_result = {"status": "failure", "error": "Image generation failed"}
-                            
-                    else:
-                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
-                        
-                except Exception as e:
-                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
-                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
-                    tool_result = {"status": "failure", "error": error_msg}
-
-                response_time = time.time() - start_time
-                success = tool_result.get("status") == "success"
-                
-                # Record performance
-                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
-                
-                # Update task status
-                if success and current_task_index < len(execution_plan.tasks):
-                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
-                    completed_tasks.add(current_task_index)
-                    current_task_index += 1
-                
-                # Enhanced observation logging
-                observation_text = json.dumps(tool_result, indent=2)
-                if len(observation_text) > 1000:
-                    # Truncate very long results for scratchpad
-                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
-                    observation_text = json.dumps(truncated_result, indent=2)
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
-                current_scratchpad += f"**Tool Used**: {tool_name}\n"
-                current_scratchpad += f"**Success**: {success}\n"
-                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
-                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
-                
-                # Track tool call
-                tool_calls_this_turn.append({
-                    "name": tool_name,
-                    "params": tool_params,
-                    "result": tool_result,
-                    "response_time": response_time,
-                    "confidence": confidence,
-                    "reasoning": reasoning
-                })
-                
-                if success:
-                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "response_time": response_time,
-                        "confidence": confidence
-                    })
-                else:
-                    error_detail = tool_result.get("error", "No error detail provided.")
-                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "error": error_detail,
-                        "confidence": confidence
-                    })
-                    
-                    # Add failure handling to scratchpad
-                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
-                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
-                
-                # Log current progress
-                completed_count = len(completed_tasks)
-                total_tasks = len(execution_plan.tasks)
-                if total_tasks > 0:
-                    progress = (completed_count / total_tasks) * 100
-                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
-                
-                # Check if all tasks are completed
-                if completed_count >= total_tasks:
-                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                    
-            except Exception as ex:
-                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
-                trace_exception(ex)
-                
-                # Add error to scratchpad for context
-                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
-                
-                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
-
-        # Enhanced self-reflection
-        if enable_self_reflection and len(tool_calls_this_turn) > 0:
-            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
-            try:
-                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-TOOLS USED: {len(tool_calls_this_turn)}
-PLAN REVISIONS: {plan_revision_count}
-
-COMPLETE ANALYSIS:
-{current_scratchpad}
-
-Evaluate your performance on multiple dimensions:
-
-1. **Goal Achievement**: Did you fully address the user's request?
-2. **Process Efficiency**: Was your approach optimal given the available tools?
-3. **Information Quality**: Is the information you gathered accurate and relevant?
-4. **Decision Making**: Were your tool choices and parameters appropriate?
-5. **Adaptability**: How well did you handle unexpected results or plan changes?
-
-Provide your assessment as JSON:
-{{
-    "goal_achieved": true,
-    "effectiveness_score": 0.85,
-    "process_efficiency": 0.8,
-    "information_quality": 0.9,
-    "decision_making": 0.85,
-    "adaptability": 0.7,
-    "overall_confidence": 0.82,
-    "strengths": ["Clear reasoning", "Good tool selection"],
-    "areas_for_improvement": ["Could have been more efficient"],
-    "summary": "Successfully completed the user's request with high quality results",
-    "key_insights": ["Discovered that X was more important than initially thought"]
-}}"""
+                
+                # Enhanced parameter generation prompt
+                param_prompt = f"""Generate the optimal parameters for the selected tool to fulfill the user's request.
 
-                reflection_data = self.generate_structured_content(
-                    prompt=reflection_prompt,
-                    schema={
-                        "goal_achieved": "boolean",
-                        "effectiveness_score": "number",
-                        "process_efficiency": "number", 
-                        "information_quality": "number",
-                        "decision_making": "number",
-                        "adaptability": "number",
-                        "overall_confidence": "number",
-                        "strengths": "array",
-                        "areas_for_improvement": "array",
-                        "summary": "string",
-                        "key_insights": "array"
-                    },
-                    temperature=0.3,
-                    **llm_generation_kwargs
-                )
+FULL discussion and USER REQUEST: 
+{prompt}
+SELECTED TOOL: {json.dumps(tool_spec, indent=2)}
+CONTEXT: {context or "None"}
+
+Analyze the user's request carefully and provide the most appropriate parameters.
+If the request has implicit requirements, infer them intelligently.
+
+Output the parameters as JSON: {{"tool_params": {{...}}}}"""
                 
-                if reflection_data:
-                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
-                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
-                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
-                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
-                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
-                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
-                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
-                    
-                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
-                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
-                        "goal_achieved": reflection_data.get('goal_achieved', False),
-                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
-                    })
+                log_prompt("Parameter Generation Prompt", param_prompt)
+                param_data = self.generate_structured_content(prompt=param_prompt, schema={"tool_params": "object"}, temperature=0.1, **llm_generation_kwargs)
+                tool_params = param_data.get("tool_params", {}) if param_data else {}
+                
+                log_event(f"Generated parameters: {json.dumps(tool_params)}", MSG_TYPE.MSG_TYPE_INFO)
+                
+                start_time, sources, tool_result = time.time(), [], {}
+                if tool_name in rag_registry:
+                    query = tool_params.get("query", prompt)
+                    log_event(f"Searching knowledge base with query: '{query}'", MSG_TYPE.MSG_TYPE_INFO)
+                    rag_fn = rag_registry[tool_name]
+                    raw_results = rag_fn(query=query, rag_top_k=rag_top_k, rag_min_similarity_percent=rag_min_similarity_percent)
+                    docs = [d for d in (raw_results.get("results", []) if isinstance(raw_results, dict) else raw_results or [])]
+                    tool_result = {"status": "success", "results": docs}
+                    sources = [{"title":d["title"], "content":d["content"], "source": tool_name, "metadata": d.get("metadata", {}), "score": d.get("score", 0.0)} for d in docs]
+                    log_event(sources, MSG_TYPE.MSG_TYPE_SOURCES_LIST)
+                    log_event(f"Retrieved {len(docs)} relevant documents", MSG_TYPE.MSG_TYPE_INFO)
+                elif hasattr(self, "mcp") and "local_tools" not in tool_name:
+                    log_event(f"Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={"tool_name": tool_name, "params": tool_params})
+                    tool_result = self.mcp.execute_tool(tool_name, tool_params, lollms_client_instance=self)
+                    log_event(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={"result_status": tool_result.get("status", "unknown")})
                 else:
-                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-                    
-            except Exception as e:
-                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+                    tool_result = {"status": "failure", "error": f"Tool '{tool_name}' could not be executed in single-step mode."}
+                
+                if tool_result.get("status","success") != "success" or "error" in tool_result:
+                    error_detail = tool_result.get("error", "Unknown tool error in single-step mode.")
+                    raise RuntimeError(error_detail)
 
-        # Enhanced final synthesis
-        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
-        
-        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
+                response_time = time.time() - start_time
+                tool_calls_this_turn = [{"name": tool_name, "params": tool_params, "result": tool_result, "response_time": response_time}]
+                
+                # Enhanced synthesis prompt
+                synthesis_prompt = f"""Create a comprehensive and user-friendly response based on the tool execution results.
 
-ORIGINAL REQUEST: "{original_user_prompt}"
-CONTEXT: {context or "No additional context"}
+FULL DISCUSSON and USER REQUEST:
+{prompt}
+TOOL USED: {tool_name}
+TOOL RESULT: {json.dumps(tool_result, indent=2)}
 
-COMPLETE ANALYSIS AND WORK:
-{current_scratchpad}
+Guidelines for your response:
+1. Be direct and helpful
+2. Synthesize the information clearly
+3. Address the user's specific needs
+4. If the tool provided data, present it in an organized way
+5. If relevant, mention any limitations or additional context
 
-GUIDELINES for your response:
-1. **Be Complete**: Address all aspects of the user's request
-2. **Be Clear**: Organize your response logically and use clear language  
-3. **Be Helpful**: Provide actionable information and insights
-4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
-5. **Be Concise**: While being thorough, avoid unnecessary verbosity
-6. **Cite Sources**: If you used research tools, reference the information appropriately
+RESPONSE:"""
+                
+                log_event("Synthesizing final response", MSG_TYPE.MSG_TYPE_INFO)
+                final_answer = self.generate_text(prompt=synthesis_prompt, system_prompt=system_prompt, stream=streaming_callback is not None, streaming_callback=streaming_callback, temperature=final_answer_temperature, **llm_generation_kwargs)
+                final_answer = self.remove_thinking_blocks(final_answer)
+                
+                log_event("✅ Single-tool execution completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id)
+                return {"final_answer": final_answer, "tool_calls": tool_calls_this_turn, "sources": sources, "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: SINGLE_TOOL\nTool: {tool_name}\nResult: Success\nResponse Time: {response_time:.2f}s"}
 
-Your response should feel natural and conversational while being informative and valuable.
+            except Exception as e:
+                log_event(f"Single-tool execution failed: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=synthesis_id)
+                log_event("Escalating to complex planning approach", MSG_TYPE.MSG_TYPE_INFO)
 
-FINAL RESPONSE:"""
-        
-        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
-        
-        final_answer_text = self.generate_text(
-            prompt=final_answer_prompt,
-            system_prompt=system_prompt,
-            stream=streaming_callback is not None,
-            streaming_callback=streaming_callback,
-            temperature=final_answer_temperature,
+        # Execute complex reasoning with enhanced capabilities
+        return self._execute_complex_reasoning_loop(
+            prompt=prompt, context=context, system_prompt=system_prompt,
+            reasoning_system_prompt=reasoning_system_prompt, images=images,
+            max_reasoning_steps=max_reasoning_steps, decision_temperature=decision_temperature,
+            final_answer_temperature=final_answer_temperature, streaming_callback=streaming_callback,
+            debug=debug, enable_self_reflection=enable_self_reflection,
+            all_visible_tools=all_visible_tools, rag_registry=rag_registry, rag_tool_specs=rag_tool_specs,
+            log_event_fn=log_event, log_prompt_fn=log_prompt, max_scratchpad_size=max_scratchpad_size,
             **llm_generation_kwargs
         )
-        
-        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
-            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
-            return {
-                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
-                "error": final_answer_text["error"],
-                "final_scratchpad": current_scratchpad,
-                "tool_calls": tool_calls_this_turn,
-                "sources": sources_this_turn,
-                "decision_history": decision_history
-            }
-        
-        final_answer = self.remove_thinking_blocks(final_answer_text)
-        
-        # Calculate overall performance metrics
-        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
-        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
-        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
-        
-        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
-            "final_answer_length": len(final_answer),
-            "total_tools_used": len(tool_calls_this_turn),
-            "success_rate": success_rate,
-            "overall_confidence": overall_confidence
-        })
-
-        return {
-            "final_answer": final_answer,
-            "final_scratchpad": current_scratchpad,
-            "tool_calls": tool_calls_this_turn,
-            "sources": sources_this_turn,
-            "decision_history": decision_history,
-            "performance_stats": {
-                "total_steps": len(tool_calls_this_turn),
-                "successful_steps": successful_calls,
-                "success_rate": success_rate,
-                "average_confidence": overall_confidence,
-                "plan_revisions": plan_revision_count,
-                "total_reasoning_steps": len(decision_history)
-            },
-            "plan_evolution": {
-                "initial_tasks": len(execution_plan.tasks),
-                "final_version": current_plan_version,
-                "total_revisions": plan_revision_count
-            },
-            "clarification_required": False,
-            "overall_confidence": overall_confidence,
-            "error": None
-        }
-
 
     def _execute_complex_reasoning_loop(
         self, prompt, context, system_prompt, reasoning_system_prompt, images,
@@ -3219,7 +1771,7 @@ FINAL RESPONSE:"""
             }
             if tool_name in descriptions:
                 return descriptions[tool_name]
-            if "research::" in tool_name: 
+            if "rag::" in tool_name: 
                 return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
             if requires_code: 
                 return "💻 Processing code"
@@ -3288,7 +1840,7 @@ FINAL RESPONSE:"""
         
         # Enhanced planning phase
         planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
-        execution_plan = planner.decompose_task(original_user_prompt, context or "")
+        execution_plan = planner.decompose_task(original_user_prompt, context or "", "\n".join([f"{tool['name']}:{tool['description']}" for tool in all_visible_tools]))
         current_plan_version = 1
         
         log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
@@ -3370,7 +1922,7 @@ Provide your decision as JSON:
         "tool_name": "exact_tool_name",
         "requires_code_input": false,
         "requires_image_input": false,
-        "confidence": 0.8
+        "confidence": 80
     }},
     "plan_status": "on_track" // or "needs_revision" if you want to change the plan
 }}"""
@@ -3399,7 +1951,7 @@ Provide your decision as JSON:
                 tool_name = action.get("tool_name")
                 requires_code = action.get("requires_code_input", False)
                 requires_image = action.get("requires_image_input", False)
-                confidence = action.get("confidence", 0.5)
+                confidence = action.get("confidence", 50)
                 
                 # Track the decision
                 decision_history.append({
@@ -3435,6 +1987,34 @@ Provide your decision as JSON:
 CURRENT ANALYSIS:
 {current_scratchpad}
 
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                
+                # Handle clarification request
+                if tool_name == "local_tools::request_clarification":
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
 Generate a clear, specific question that will help you proceed effectively:"""
                     
                     question = self.generate_text(clarification_prompt, temperature=0.3)
@@ -3922,7 +2502,8 @@ FINAL RESPONSE:"""
                         repeat_last_n:int|None=None,
                         callback=None,
                         debug:bool=False,
-                        override_all_prompts:bool=False ):
+                        override_all_prompts:bool=False,
+                        **kwargs ):
         """
         Generates a single code block based on a prompt.
         Uses the underlying LLM binding via `generate_text`.
@@ -5734,184 +4315,229 @@ Provide the final aggregated answer in {output_format} format, directly addressi
     def long_context_processing(
         self,
         text_to_process: str,
-        contextual_prompt: Optional[str] = None,
-        chunk_size_tokens: int|None = None,
-        overlap_tokens: int = 0,
+        contextual_prompt: str,
+        system_prompt: str | None = None,
+        strategy_override: str | None = None, # 'narrative' or 'structured'
         streaming_callback: Optional[Callable] = None,
+        debug: bool = True,
         **kwargs
     ) -> str:
         """
-        Summarizes a long text that may not fit into the model's context window.
+        Adaptive Long Context Processing.
+        Automatically detects if content is Narrative (Prose) or Structured (Data/Logs)
+        and applies the optimal processing strategy.
+        """
 
-        This method works in two stages:
-        1.  **Chunk & Summarize:** It breaks the text into overlapping chunks and summarizes each one individually.
-        2.  **Synthesize:** It then takes all the chunk summaries and performs a final summarization pass to create a single, coherent, and comprehensive summary.
+        # --- Helper: Token Counting ---
+        def get_tokens(text):
+            return len(self.tokenize(text))
+        
+        # --- Helper: Smart Chunking ---
+        def smart_chunk_text(text, max_chunk_tokens, mode='narrative'):
+            # For data, we split by lines to avoid breaking rows. For text, paragraphs.
+            delimiter = '\n' if mode == 'structured' else '\n\n'
+            segments = text.split(delimiter)
+            
+            chunks = []
+            current_chunk = []
+            current_len = 0
+            
+            for seg in segments:
+                seg_tokens = get_tokens(seg)
+                # Hard limit safety for massive single lines
+                if seg_tokens > max_chunk_tokens:
+                    # If a single row/para is massive, we force split it
+                    if current_chunk:
+                        chunks.append(delimiter.join(current_chunk))
+                        current_chunk = []
+                        current_len = 0
+                    chunks.append(seg)
+                    continue
 
-        Args:
-            text_to_process (str): The long text content to be summarized.
-            contextual_prompt (Optional[str], optional): A specific instruction to guide the summary's focus. 
-                                                       For example, "Summarize the text focusing on the financial implications."
-                                                       Defaults to None.
-            chunk_size_tokens (int, optional): The number of tokens in each text chunk. This should be well
-                                             within the model's context limit to allow space for prompts.
-                                             Defaults to 1500.
-            overlap_tokens (int, optional): The number of tokens to overlap between chunks to ensure context
-                                          is not lost at the boundaries. Defaults to 250.
-            streaming_callback (Optional[Callable], optional): A callback function to receive real-time updates
-                                                             on the process (e.g., which chunk is being processed).
-                                                             It receives a message, a message type, and optional metadata.
-                                                             Defaults to None.
-            **kwargs: Additional keyword arguments to be passed to the generation method (e.g., temperature, top_p).
+                if current_len + seg_tokens > max_chunk_tokens and current_chunk:
+                    chunks.append(delimiter.join(current_chunk))
+                    current_chunk = []
+                    current_len = 0
+                
+                current_chunk.append(seg)
+                current_len += seg_tokens
+            
+            if current_chunk:
+                chunks.append(delimiter.join(current_chunk))
+            return chunks
 
-        Returns:
-            str: The final, comprehensive summary of the text.
-        """
-        if not text_to_process and len(kwargs.get("images",[]))==0 and contextual_prompt is None:
-            return ""
-        if not text_to_process:
-            text_to_process=""
-            tokens = []
-        else:
-            # Use the binding's tokenizer for accurate chunking
-            tokens = self.llm.tokenize(text_to_process)
-        if chunk_size_tokens is None:
-            chunk_size_tokens = self.llm.default_ctx_size//2
+        # --- 0. Pre-computation ---
+        context_size = self.llm.get_context_size() or 8192
+        total_tokens = get_tokens(text_to_process)
         
-        if len(tokens) <= chunk_size_tokens:
-            if streaming_callback:
-                streaming_callback("Text is short enough for a single pass.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 0})
-            system_prompt = ("You are a content processor expert.\n"
-                            "You perform tasks on the content as requested by the user.\n\n"
-                            "--- Content ---\n"
-                            f"{text_to_process}\n\n"
-                            "** Important **\n"
-                            "Strictly adhere to the user prompt.\n"
-                            "Do not add comments unless asked to do so.\n"
-                            )
-            if "system_prompt" in kwargs:
-                system_prompt += "-- Extra instructions --\n"+ kwargs["system_prompt"] +"\n"
-                del kwargs["system_prompt"]            
-            prompt_objective = contextual_prompt or "Provide a comprehensive summary of the content."
-            final_prompt = f"{prompt_objective}"
+        if debug: print(f"🔧 DEBUG: Input Tokens: {total_tokens:,} | Context: {context_size:,}")
+
+        # One-shot optimization
+        if total_tokens < (context_size * 0.7):
+            if debug: print("🔧 DEBUG: One-shot path.")
+            if streaming_callback: streaming_callback("Processing in single pass...", MSG_TYPE.MSG_TYPE_STEP_START)
+            prompt = f"{contextual_prompt}\n\n--- CONTENT ---\n{text_to_process}"
+            return self.remove_thinking_blocks(self.llm.generate_text(prompt, system_prompt=system_prompt, **kwargs))
+
+        # --- 1. Phase 1: The Scout (Strategy Selection) ---
+        content_type = "narrative"
+        schema_info = ""
+        
+        if strategy_override:
+            content_type = strategy_override
+        else:
+            if debug: print("🔧 DEBUG: Scout is analyzing content nature...")
+            if streaming_callback: streaming_callback("Analyzing content structure...", MSG_TYPE.MSG_TYPE_STEP)
             
-            processed_output = self.generate_text(final_prompt, system_prompt=system_prompt, **kwargs)
+            # Peek at the first 1000 characters (enough to see CSV headers or JSON braces)
+            sample_text = text_to_process[:4000]
             
-            if streaming_callback:
-                streaming_callback("Content processed.", MSG_TYPE.MSG_TYPE_STEP, {"progress": 100})
+            scout_prompt = (
+                f"Analyze this text sample:\n---\n{sample_text}\n---\n\n"
+                f"Is this primarily:\n"
+                f"A) NARRATIVE (Prose, Articles, Transcripts, Story)\n"
+                f"B) STRUCTURED_DATA (CSV, Markdown Tables, JSON, Logs, Code Lists)\n\n"
+                f"Return JSON with keys: 'type' ('narrative' or 'structured') and 'schema_notes' (brief description of columns/fields if structured)."
+            )
             
-            return processed_output
+            try:
+                scout_res = self.remove_thinking_blocks(self.llm.generate_text(scout_prompt, **kwargs))
+                # Simple parsing heuristic if JSON fails
+                if "structured" in scout_res.lower() and "narrative" not in scout_res.lower().split("type"):
+                    content_type = "structured"
+                    schema_info = scout_res # Keep the whole reasoning as context
+                else:
+                    content_type = "narrative"
+            except:
+                content_type = "narrative" # Default fail-safe
 
-        # --- Stage 1: Chunking and Independent Summarization ---
-        chunks = []
-        step = chunk_size_tokens - overlap_tokens
-        for i in range(0, len(tokens), step):
-            chunk_tokens = tokens[i:i + chunk_size_tokens]
-            chunk_text = self.llm.detokenize(chunk_tokens)
-            chunks.append(chunk_text)
+        if debug: print(f"🔧 DEBUG: Strategy Selected: {content_type.upper()}")
 
-        chunk_summaries = []
+        # --- 2. Phase 2: Execution ---
         
-        # Total steps include each chunk plus the final synthesis step
-        total_steps = len(chunks) + 1
+        # Calculate Chunk Size (Leave room for prompts and outputs)
+        reserved_overhead = 2500 
+        safe_chunk_size = context_size - reserved_overhead
         
-        # Define the prompt for summarizing each chunk
-        summarization_objective = contextual_prompt or "Summarize the key points of the following text excerpt."
-        system_prompt = ("You are a sequential document processing agent.\n"
-                         "The process is done in two phases:\n"
-                         "** Phase1 : **\n"
-                         "Sequencially extracting information from the text chunks and adding them to the scratchpad.\n"
-                         "** Phase2: **\n"
-                         "Synthesizing a comprehensive Response using the scratchpad content given the objective formatting instructions if applicable.\n"
-                         "We are now performing ** Phase 1 **, and we are processing chunk number {{chunk_id}}.\n"
-                         "Your job is to extract information from the current chunk given previous chunks extracted information placed in scratchpad as well as the current chunk content.\n"
-                         "Add the information to the scratchpad while strictly adhering to the Global objective extraction instructions:\n"
-                         "-- Sequencial Scratchpad --\n"
-                         "{{scratchpad}}\n"
-                         "** Important **\n"
-                         "Respond only with the extracted information from the current chunk without repeating things that are already in the scratchpad.\n"
-                         "Strictly adhere to the Global objective content for the extraction phase.\n"
-                         "Do not add comments.\n"
-                        )
-        if "system_prompt" in kwargs:
-            system_prompt += "-- Extra instructions --\n"+ kwargs["system_prompt"] +"\n"
-            del kwargs["system_prompt"]
-        chunk_summary_prompt_template = f"--- Global objective ---\n{summarization_objective}\n\n--- Text Excerpt ---\n{{chunk_text}}"
+        # Split
+        chunks = smart_chunk_text(text_to_process, safe_chunk_size, mode=content_type)
+        total_chunks = len(chunks)
+        extraction_accumulator = []
+
+        # =======================================================
+        # STRATEGY A: NARRATIVE (Context-Aware Dual Stream)
+        # =======================================================
+        if content_type == "narrative":
+            context_scratchpad = "Start of content."
+            
+            # Generate Criteria (from previous strategy)
+            strategy_prompt = f"Objective: {contextual_prompt}\nList bullet points of what to extract from this narrative."
+            extraction_criteria = self.remove_thinking_blocks(self.llm.generate_text(strategy_prompt, **kwargs))
 
-        for i, chunk in enumerate(chunks):
-            progress_before = (i / total_steps) * 100
-            if streaming_callback:
-                streaming_callback(
-                    f"Processing chunk {i + 1} of {len(chunks)}...", 
-                    MSG_TYPE.MSG_TYPE_STEP_START, 
-                    {"id": f"chunk_{i+1}", "progress": progress_before}
+            for i, chunk in enumerate(chunks):
+                if streaming_callback: 
+                    streaming_callback(f"Narrative Chunk {i+1}/{total_chunks}", MSG_TYPE.MSG_TYPE_STEP_START, {"progress": (i/total_chunks)*80})
+                
+                prompt = (
+                    f"### OBJECTIVE\n{contextual_prompt}\n\n"
+                    f"### CRITERIA\n{extraction_criteria}\n\n"
+                    f"### PREVIOUS CONTEXT\n{context_scratchpad}\n\n"
+                    f"### CURRENT CHUNK\n{chunk}\n\n"
+                    f"### TASK\n"
+                    f"1. Update Context: Summarize flow for next chunk.\n"
+                    f"2. Extract Data: Extract facts matching criteria.\n"
+                    f"Output format: <context_update>...</context_update> <extracted_data>...</extracted_data>"
                 )
-
-            prompt = chunk_summary_prompt_template.format(chunk_text=chunk)
-            processed_system_prompt = system_prompt.format(chunk_id=i,scratchpad="\n\n---\n\n".join(chunk_summaries))
-            try:
-                # Generate summary for the current chunk
-                chunk_summary = self.generate_text(prompt, system_prompt=processed_system_prompt, **kwargs)
-                chunk_summaries.append(chunk_summary)
                 
-                progress_after = ((i + 1) / total_steps) * 100
-                if streaming_callback:
-                    streaming_callback(
-                        f"Chunk {i + 1} processed. Progress: {progress_after:.0f}%", 
-                        MSG_TYPE.MSG_TYPE_STEP_END, 
-                        {"id": f"chunk_{i+1}", "output_snippet": chunk_summary[:100], "progress": progress_after}
+                res = self.remove_thinking_blocks(self.llm.generate_text(prompt, **kwargs))
+                
+                # Parse
+                new_ctx = "Flow continues..."
+                new_data = res
+                if "<context_update>" in res: 
+                    new_ctx = res.split("<context_update>")[1].split("</context_update>")[0]
+                if "<extracted_data>" in res: 
+                    new_data = res.split("<extracted_data>")[1].split("</extracted_data>")[0]
+                
+                context_scratchpad = new_ctx
+                extraction_accumulator.append(new_data)
+                
+                # Compression check
+                if get_tokens("\n".join(extraction_accumulator)) > context_size * 0.6:
+                    if debug: print("🔧 DEBUG: Compressing narrative accumulator...")
+                    comp_prompt = f"Summarize these extracted notes without losing facts:\n\n" + "\n".join(extraction_accumulator)
+                    compressed = self.remove_thinking_blocks(self.llm.generate_text(comp_prompt, **kwargs))
+                    extraction_accumulator = [compressed]
+
+        # =======================================================
+        # STRATEGY B: STRUCTURED DATA (Map-Reduce / Batch)
+        # =======================================================
+        else:
+            # For data, we don't need narrative context. We need SCHEMA context.
+            if debug: print(f"🔧 DEBUG: Schema Context: {schema_info}")
+            
+            for i, chunk in enumerate(chunks):
+                if streaming_callback: 
+                    streaming_callback(f"Data Batch {i+1}/{total_chunks}", MSG_TYPE.MSG_TYPE_STEP_START, {"progress": (i/total_chunks)*80})
+                
+                # The Data Prompt is different: It treats the chunk as a standalone dataset
+                prompt = (
+                    f"### ROLE\n"
+                    f"You are a Data Analyst. You are processing batch {i+1} of {total_chunks}.\n\n"
+                    f"### GLOBAL OBJECTIVE\n"
+                    f"{contextual_prompt}\n\n"
+                    f"### DATA SCHEMA / HINTS\n"
+                    f"{schema_info}\n\n"
+                    f"### DATA BATCH\n"
+                    f"{chunk}\n\n"
+                    f"### INSTRUCTIONS\n"
+                    f"Analyze this specific batch of data to fulfill the objective.\n"
+                    f"- If the objective asks for aggregation (counts, averages), calculate them for THIS BATCH only.\n"
+                    f"- If the objective asks for pattern extraction, list patterns found in THIS BATCH.\n"
+                    f"- If the objective asks for row-by-row extraction, process only the relevant rows.\n"
+                    f"- Output strictly the findings. Do not summarize the 'idea' of the data, extract the ACTUAL data/metrics."
+                )
+                
+                res = self.remove_thinking_blocks(self.llm.generate_text(prompt, **kwargs))
+                extraction_accumulator.append(f"--- Batch {i+1} Findings ---\n{res}")
+
+                # Compression for Data (Map-Reduce style)
+                # If we have too many batch findings, we do an intermediate "Reduce" step
+                if get_tokens("\n".join(extraction_accumulator)) > context_size * 0.6:
+                    if debug: print("🔧 DEBUG: Reducing intermediate data batches...")
+                    reduce_prompt = (
+                        f"### OBJECTIVE\n{contextual_prompt}\n\n"
+                        f"### INTERMEDIATE BATCH RESULTS\n" + "\n".join(extraction_accumulator) + "\n\n"
+                        f"### TASK\n"
+                        f"Aggregate these batch results into a consolidated report. "
+                        f"Sum up counts, combine lists, and merge patterns. Discard redundant headers."
                     )
-            except Exception as e:
-                trace_exception(e)
-                if streaming_callback:
-                    streaming_callback(f"Failed to process chunk {i+1}: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION)
-                # Still add a placeholder to not break the chain
-                chunk_summaries.append(f"[Error processing chunk {i+1}]")
-
-        # --- Stage 2: Final Synthesis of All Chunk Summaries ---
-        progress_before_synthesis = (len(chunks) / total_steps) * 100
-        if streaming_callback:
-            streaming_callback(
-                "Processing the scratchpad content into a final version...", 
-                MSG_TYPE.MSG_TYPE_STEP_START, 
-                {"id": "final_synthesis", "progress": progress_before_synthesis}
-            )
+                    reduced = self.remove_thinking_blocks(self.llm.generate_text(reduce_prompt, **kwargs))
+                    extraction_accumulator = [reduced]
 
-        combined_summaries = "\n\n---\n\n".join(chunk_summaries)
+        # --- 3. Phase 3: Final Synthesis ---
+        if debug: print("🔧 DEBUG: Final Synthesis")
+        if streaming_callback: streaming_callback("Synthesizing final report...", MSG_TYPE.MSG_TYPE_STEP_START, {"progress": 95})
         
-        # Define the prompt for the final synthesis
-        synthesis_objective = contextual_prompt or "Create a single, final, coherent, and comprehensive summary."
-        system_prompt = ("You are a sequential document processing agent.\n"
-                         "The process is done in two phases:\n"
-                         "** Phase1 : **\n"
-                         "Sequencially extracting information from the text chunks and adding them to the scratchpad.\n"
-                         "** Phase2: **\n"
-                         "Synthesizing a comprehensive Response using the scratchpad content given the objective formatting instructions if applicable.\n"
-                         "\n"
-                         "We are now performing ** Phase 2 **.\n"
-                         "Your job is to use the extracted information to fulfill the user prompt objectives.\n"
-                         "Make sure you respect the user formatting if provided and if not, then use markdown output format."
-                         "-- Sequencial Scratchpad --\n"
-                         f"{combined_summaries}\n"
-                         "** Important **\n"
-                         "Respond only with the requested task without extra comments unless told to.\n"
-                         "Strictly adhere to the Global objective content for the extraction phase.\n"
-                         "Do not add comments.\n"
-                        )
-        final_synthesis_prompt = (
-            f"--- Global objective ---\n{synthesis_objective}\n\n"
-            "--- Final Response ---"
+        final_data_block = "\n".join(extraction_accumulator)
+        
+        final_prompt = (
+            f"### GLOBAL OBJECTIVE\n{contextual_prompt}\n\n"
+            f"### PROCESSED DATA/EVIDENCE\n{final_data_block}\n\n"
+            f"### INSTRUCTIONS\n"
+            f"Synthesize the final answer based ONLY on the processed evidence above.\n"
+            f"If this was a data analysis task, provide the final metrics/aggregations.\n"
+            f"If this was a narrative task, provide the final summary.\n"
+            f"Format clearly with Markdown."
         )
-
-        final_answer = self.generate_text(final_synthesis_prompt, system_prompt=system_prompt, **kwargs)
         
-        if streaming_callback:
-            streaming_callback(
-                "Final summary synthesized.", 
-                MSG_TYPE.MSG_TYPE_STEP_END, 
-                {"id": "final_synthesis", "progress": 100}
-            )
+        final_answer = self.remove_thinking_blocks(self.llm.generate_text(final_prompt, system_prompt=system_prompt, **kwargs))
+        
+        if streaming_callback: streaming_callback("Done.", MSG_TYPE.MSG_TYPE_STEP_END, {"progress": 100})
+        
+        return final_answer
 
-        return final_answer.strip()
+        
 
 def chunk_text(text, tokenizer, detokenizer, chunk_size, overlap, use_separators=True):
     """