lollms-client 1.3.8__py3-none-any.whl → 1.4.1__py3-none-any.whl

lollms_client/lollms_core.py

@@ -1474,6 +1474,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                 # Clean up the technical tool name for a more readable display
                 clean_name = tool_name.replace("_", " ").replace("::", " - ").title()
                 return f"🔧 Using the {clean_name} tool"
+            
     def generate_with_mcp_rag(
             self,
             prompt: str,
@@ -1494,6 +1495,7 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
             debug: bool = False,
             enable_parallel_execution: bool = True,
             enable_self_reflection: bool = True,
+            max_scratchpad_size: int = 20000,
             **llm_generation_kwargs
         ) -> Dict[str, Any]:
         
@@ -1525,13 +1527,23 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
             ASCIIColors.red(f"Prompt size:{prompt_size}/{self.llm.default_ctx_size}")
             ASCIIColors.cyan(f"** DEBUG: DONE **")
 
-        discovery_step_id = log_event("🔧 Setting up capabilities...", MSG_TYPE.MSG_TYPE_STEP_START)
+        # Enhanced discovery phase with more detailed logging
+        discovery_step_id = log_event("🔧 Discovering and configuring available capabilities...", MSG_TYPE.MSG_TYPE_STEP_START)
         all_discovered_tools, visible_tools, rag_registry, rag_tool_specs = [], [], {}, {}
+        
         if use_mcps and hasattr(self, 'mcp'):
+            log_event("  📡 Connecting to MCP services...", MSG_TYPE.MSG_TYPE_INFO)
             mcp_tools = self.mcp.discover_tools(force_refresh=True)
-            if isinstance(use_mcps, list): all_discovered_tools.extend([t for t in mcp_tools if t["name"] in use_mcps])
-            elif use_mcps is True: all_discovered_tools.extend(mcp_tools)
+            if isinstance(use_mcps, list): 
+                filtered_tools = [t for t in mcp_tools if t["name"] in use_mcps]
+                all_discovered_tools.extend(filtered_tools)
+                log_event(f"  ✅ Loaded {len(filtered_tools)} specific MCP tools: {', '.join(use_mcps)}", MSG_TYPE.MSG_TYPE_INFO)
+            elif use_mcps is True: 
+                all_discovered_tools.extend(mcp_tools)
+                log_event(f"  ✅ Loaded {len(mcp_tools)} MCP tools", MSG_TYPE.MSG_TYPE_INFO)
+        
         if use_data_store:
+            log_event(f"  📚 Setting up {len(use_data_store)} knowledge bases...", MSG_TYPE.MSG_TYPE_INFO)
             for name, info in use_data_store.items():
                 tool_name, description, call_fn = f"research::{name}", f"Queries the '{name}' knowledge base.", None
                 if callable(info): call_fn = info
@@ -1542,75 +1554,138 @@ Provide your response as a single JSON object inside a JSON markdown tag. Use th
                     visible_tools.append({"name": tool_name, "description": description, "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]}})
                     rag_registry[tool_name] = call_fn
                     rag_tool_specs[tool_name] = {"default_top_k": rag_top_k, "default_min_sim": rag_min_similarity_percent}
+                    log_event(f"    📖 Ready: {name}", MSG_TYPE.MSG_TYPE_INFO)
+        
         visible_tools.extend(all_discovered_tools)
-        built_in_tools = [{"name": "local_tools::final_answer", "description": "Provide the final answer directly to the user.", "input_schema": {}}]
-        if getattr(self, "tti", None): built_in_tools.append({"name": "local_tools::generate_image", "description": "Generate an image from a text description.", "input_schema": {"type": "object", "properties": {"prompt": {"type": "string"}}, "required": ["prompt"]}})
+        built_in_tools = [
+            {"name": "local_tools::final_answer", "description": "Provide the final answer directly to the user.", "input_schema": {}},
+            {"name": "local_tools::request_clarification", "description": "Ask the user for more specific information when the request is ambiguous.", "input_schema": {"type": "object", "properties": {"question": {"type": "string"}}, "required": ["question"]}},
+            {"name": "local_tools::revise_plan", "description": "Update the execution plan based on new discoveries or changing requirements.", "input_schema": {"type": "object", "properties": {"reason": {"type": "string"}, "new_plan": {"type": "array"}}, "required": ["reason", "new_plan"]}}
+        ]
+        if getattr(self, "tti", None): 
+            built_in_tools.append({"name": "local_tools::generate_image", "description": "Generate an image from a text description.", "input_schema": {"type": "object", "properties": {"prompt": {"type": "string"}}, "required": ["prompt"]}})
+        
         all_visible_tools = visible_tools + built_in_tools
-        tool_summary = "\n".join([f"- {t['name']}: {t['description']}" for t in all_visible_tools[:15]])
-        log_event(f"✅ Ready with {len(all_visible_tools)} capabilities", MSG_TYPE.MSG_TYPE_STEP_END, event_id=discovery_step_id)
+        tool_summary = "\n".join([f"- **{t['name']}**: {t['description']}" for t in all_visible_tools[:20]])
+        
+        log_event(f"✅ Ready with {len(all_visible_tools)} total capabilities", MSG_TYPE.MSG_TYPE_STEP_END, event_id=discovery_step_id, meta={"tool_count": len(all_visible_tools), "mcp_tools": len(all_discovered_tools), "rag_tools": len(rag_registry)})
 
-        triage_step_id = log_event("🤔 Analyzing the best approach...", MSG_TYPE.MSG_TYPE_STEP_START)
+        # Enhanced triage with better prompting
+        triage_step_id = log_event("🤔 Analyzing request complexity and optimal approach...", MSG_TYPE.MSG_TYPE_STEP_START)
         strategy = "COMPLEX_PLAN"
         strategy_data = {}
         try:
-            triage_prompt = f"""Analyze the user's request and determine the most efficient strategy.
+            triage_prompt = f"""Analyze this user request to determine the most efficient execution strategy.
+
 USER REQUEST: "{prompt}"
-AVAILABLE TOOLS:\n{tool_summary}
-Choose a strategy:
-- "DIRECT_ANSWER": For greetings or simple questions that need no tools.
-- "REQUEST_CLARIFICATION": If the request is ambiguous and you need more information from the user.
-- "SINGLE_TOOL": If the request can be resolved with one tool call.
-- "COMPLEX_PLAN": For multi-step requests requiring multiple tools or complex reasoning.
+CONTEXT: {context or "No additional context provided"}
+IMAGES PROVIDED: {"Yes" if images else "No"}
 
-Provide your decision as JSON: {{"thought": "...", "strategy": "...", "text_output": "Your direct answer or clarification question.", "required_tool_name": "..."}}"""
+AVAILABLE CAPABILITIES:
+{tool_summary}
+
+Based on the request complexity and available tools, choose the optimal strategy:
+
+1. **DIRECT_ANSWER**: For simple greetings, basic questions, or requests that don't require any tools
+   - Use when: The request can be fully answered with your existing knowledge
+   - Example: "Hello", "What is Python?", "Explain quantum physics"
+
+2. **REQUEST_CLARIFICATION**: When the request is too vague or ambiguous
+   - Use when: The request lacks essential details needed to proceed
+   - Example: "Help me with my code" (what code? what issue?)
+
+3. **SINGLE_TOOL**: For straightforward requests that need exactly one tool
+   - Use when: The request clearly maps to a single, specific tool operation
+   - Example: "Search for information about X", "Generate an image of Y"
+
+4. **COMPLEX_PLAN**: For multi-step requests requiring coordination of multiple tools
+   - Use when: The request involves multiple steps, data analysis, or complex reasoning
+   - Example: "Research X, then create a report comparing it to Y"
+
+Provide your analysis in JSON format:
+{{"thought": "Detailed reasoning about the request complexity and requirements", "strategy": "ONE_OF_THE_FOUR_OPTIONS", "confidence": 0.8, "text_output": "Direct answer or clarification question if applicable", "required_tool_name": "specific tool name if SINGLE_TOOL strategy", "estimated_steps": 3}}"""
+            
+            log_prompt("Triage Prompt", triage_prompt)
             
             triage_schema = {
-                "thought": "string", "strategy": "string", 
-                "text_output": "string", "required_tool_name": "string"
+                "thought": "string", "strategy": "string", "confidence": "number",
+                "text_output": "string", "required_tool_name": "string", "estimated_steps": "number"
             }
             strategy_data = self.generate_structured_content(prompt=triage_prompt, schema=triage_schema, temperature=0.1, **llm_generation_kwargs)
             strategy = strategy_data.get("strategy") if strategy_data else "COMPLEX_PLAN"
+            
+            log_event(f"Strategy analysis complete", MSG_TYPE.MSG_TYPE_INFO, meta={
+                "strategy": strategy,
+                "confidence": strategy_data.get("confidence", 0.5),
+                "estimated_steps": strategy_data.get("estimated_steps", 1),
+                "reasoning": strategy_data.get("thought", "")
+            })
+            
         except Exception as e:
-            log_event(f"Triage failed, defaulting to complex plan. Error: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=triage_step_id)
+            log_event(f"Triage analysis failed: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=triage_step_id)
+            log_event("Defaulting to complex planning approach", MSG_TYPE.MSG_TYPE_WARNING)
 
         if force_mcp_use and strategy == "DIRECT_ANSWER":
             strategy = "COMPLEX_PLAN"
-        log_event(f"✅ Approach decided: {strategy.replace('_', ' ').title()}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=triage_step_id)
+            log_event("Forcing tool usage - switching to complex planning", MSG_TYPE.MSG_TYPE_INFO)
+            
+        log_event(f"✅ Strategy selected: {strategy.replace('_', ' ').title()}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=triage_step_id, meta={"final_strategy": strategy})
 
+        # Handle simple strategies
         if strategy == "DIRECT_ANSWER":
             final_answer = strategy_data.get("text_output", "I can help with that.")
+            log_event("Providing direct response", MSG_TYPE.MSG_TYPE_INFO)
             if streaming_callback: streaming_callback(final_answer, MSG_TYPE.MSG_TYPE_CONTENT, {})
-            return {"final_answer": final_answer, "tool_calls": [], "sources": [], "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: DIRECT_ANSWER\nThought: {strategy_data.get('thought')}"}
+            return {"final_answer": final_answer, "tool_calls": [], "sources": [], "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: DIRECT_ANSWER\nConfidence: {strategy_data.get('confidence', 0.9)}\nReasoning: {strategy_data.get('thought')}"}
 
         if strategy == "REQUEST_CLARIFICATION":
-            clarification_question = strategy_data.get("text_output", "Could you please provide more details?")
-            return {"final_answer": clarification_question, "tool_calls": [], "sources": [], "error": None, "clarification_required": True, "final_scratchpad": f"Strategy: REQUEST_CLARIFICATION\nThought: {strategy_data.get('thought')}"}
+            clarification_question = strategy_data.get("text_output", "Could you please provide more details about what specifically you'd like me to help with?")
+            log_event("Requesting clarification from user", MSG_TYPE.MSG_TYPE_INFO)
+            return {"final_answer": clarification_question, "tool_calls": [], "sources": [], "error": None, "clarification_required": True, "final_scratchpad": f"Strategy: REQUEST_CLARIFICATION\nConfidence: {strategy_data.get('confidence', 0.8)}\nReasoning: {strategy_data.get('thought')}"}
 
+        # Enhanced single tool execution
         if strategy == "SINGLE_TOOL":
-            synthesis_id = log_event("⚡ Taking a direct approach...", MSG_TYPE.MSG_TYPE_STEP_START)
+            synthesis_id = log_event("⚡ Executing single-tool strategy...", MSG_TYPE.MSG_TYPE_STEP_START)
             try:
                 tool_name = strategy_data.get("required_tool_name")
                 tool_spec = next((t for t in all_visible_tools if t['name'] == tool_name), None)
                 if not tool_spec:
-                    raise ValueError(f"LLM chose an unavailable tool: '{tool_name}'")
+                    raise ValueError(f"Strategy analysis selected unavailable tool: '{tool_name}'")
                 
-                param_prompt = f"""Given the user request, generate the correct parameters for the selected tool.
+                log_event(f"Selected tool: {tool_name}", MSG_TYPE.MSG_TYPE_INFO)
+                
+                # Enhanced parameter generation prompt
+                param_prompt = f"""Generate the optimal parameters for the selected tool to fulfill the user's request.
+
 USER REQUEST: "{prompt}"
 SELECTED TOOL: {json.dumps(tool_spec, indent=2)}
-Output ONLY the JSON for the tool's parameters: {{"tool_params": {{...}}}}"""
+CONTEXT: {context or "None"}
+
+Analyze the user's request carefully and provide the most appropriate parameters.
+If the request has implicit requirements, infer them intelligently.
+
+Output the parameters as JSON: {{"tool_params": {{...}}}}"""
+                
+                log_prompt("Parameter Generation Prompt", param_prompt)
                 param_data = self.generate_structured_content(prompt=param_prompt, schema={"tool_params": "object"}, temperature=0.1, **llm_generation_kwargs)
                 tool_params = param_data.get("tool_params", {}) if param_data else {}
                 
+                log_event(f"Generated parameters: {json.dumps(tool_params)}", MSG_TYPE.MSG_TYPE_INFO)
+                
                 start_time, sources, tool_result = time.time(), [], {}
                 if tool_name in rag_registry:
                     query = tool_params.get("query", prompt)
+                    log_event(f"Searching knowledge base with query: '{query}'", MSG_TYPE.MSG_TYPE_INFO)
                     rag_fn = rag_registry[tool_name]
                     raw_results = rag_fn(query=query, rag_top_k=rag_top_k, rag_min_similarity_percent=rag_min_similarity_percent)
                     docs = [d for d in (raw_results.get("results", []) if isinstance(raw_results, dict) else raw_results or [])]
                     tool_result = {"status": "success", "results": docs}
                     sources = [{"source": tool_name, "metadata": d.get("metadata", {}), "score": d.get("score", 0.0)} for d in docs]
+                    log_event(f"Retrieved {len(docs)} relevant documents", MSG_TYPE.MSG_TYPE_INFO)
                 elif hasattr(self, "mcp") and "local_tools" not in tool_name:
+                    log_event(f"Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={"tool_name": tool_name, "params": tool_params})
                     tool_result = self.mcp.execute_tool(tool_name, tool_params, lollms_client_instance=self)
+                    log_event(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={"result_status": tool_result.get("status", "unknown")})
                 else:
                     tool_result = {"status": "failure", "error": f"Tool '{tool_name}' could not be executed in single-step mode."}
                 
@@ -1621,19 +1696,34 @@ Output ONLY the JSON for the tool's parameters: {{"tool_params": {{...}}}}"""
                 response_time = time.time() - start_time
                 tool_calls_this_turn = [{"name": tool_name, "params": tool_params, "result": tool_result, "response_time": response_time}]
                 
-                synthesis_prompt = f"""The user asked: "{prompt}"
-I used the tool '{tool_name}' and got this result: {json.dumps(tool_result, indent=2)}
-Synthesize a direct, user-friendly final answer."""
+                # Enhanced synthesis prompt
+                synthesis_prompt = f"""Create a comprehensive and user-friendly response based on the tool execution results.
+
+USER REQUEST: "{prompt}"
+TOOL USED: {tool_name}
+TOOL RESULT: {json.dumps(tool_result, indent=2)}
+
+Guidelines for your response:
+1. Be direct and helpful
+2. Synthesize the information clearly
+3. Address the user's specific needs
+4. If the tool provided data, present it in an organized way
+5. If relevant, mention any limitations or additional context
+
+RESPONSE:"""
+                
+                log_event("Synthesizing final response", MSG_TYPE.MSG_TYPE_INFO)
                 final_answer = self.generate_text(prompt=synthesis_prompt, system_prompt=system_prompt, stream=streaming_callback is not None, streaming_callback=streaming_callback, temperature=final_answer_temperature, **llm_generation_kwargs)
                 final_answer = self.remove_thinking_blocks(final_answer)
                 
-                log_event("✅ Direct answer ready!", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id)
-                return {"final_answer": final_answer, "tool_calls": tool_calls_this_turn, "sources": sources, "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: SINGLE_TOOL\nTool: {tool_name}\nResult: {json.dumps(tool_result)}"}
+                log_event("✅ Single-tool execution completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id)
+                return {"final_answer": final_answer, "tool_calls": tool_calls_this_turn, "sources": sources, "error": None, "clarification_required": False, "final_scratchpad": f"Strategy: SINGLE_TOOL\nTool: {tool_name}\nResult: Success\nResponse Time: {response_time:.2f}s"}
 
             except Exception as e:
-                log_event(f"Direct approach failed: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=synthesis_id)
-                log_event("Escalating to a more detailed plan.", MSG_TYPE.MSG_TYPE_INFO)
+                log_event(f"Single-tool execution failed: {e}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=synthesis_id)
+                log_event("Escalating to complex planning approach", MSG_TYPE.MSG_TYPE_INFO)
 
+        # Execute complex reasoning with enhanced capabilities
         return self._execute_complex_reasoning_loop(
             prompt=prompt, context=context, system_prompt=system_prompt,
             reasoning_system_prompt=reasoning_system_prompt, images=images,
@@ -1641,7 +1731,7 @@ Synthesize a direct, user-friendly final answer."""
             final_answer_temperature=final_answer_temperature, streaming_callback=streaming_callback,
             debug=debug, enable_self_reflection=enable_self_reflection,
             all_visible_tools=all_visible_tools, rag_registry=rag_registry, rag_tool_specs=rag_tool_specs,
-            log_event_fn=log_event, log_prompt_fn=log_prompt,
+            log_event_fn=log_event, log_prompt_fn=log_prompt, max_scratchpad_size=max_scratchpad_size,
             **llm_generation_kwargs
         )
 
@@ -1649,170 +1739,2170 @@ Synthesize a direct, user-friendly final answer."""
         self, prompt, context, system_prompt, reasoning_system_prompt, images,
         max_reasoning_steps, decision_temperature, final_answer_temperature,
         streaming_callback, debug, enable_self_reflection, all_visible_tools,
-        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, **llm_generation_kwargs
+        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
     ) -> Dict[str, Any]:
         
         planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
         
         def _get_friendly_action_description(tool_name, requires_code, requires_image):
-            if tool_name == "local_tools::final_answer": return "📋 Ready to provide your answer"
-            if tool_name == "local_tools::request_clarification": return "❓ Need to ask for clarification"
-            if tool_name == "local_tools::generate_image": return "🎨 Creating an image for you"
-            if "research::" in tool_name: return f"🔍 Searching {tool_name.split('::')[-1]} for information"
-            if requires_code: return "💻 Working on a coding solution"
-            if requires_image: return "🖼️ Analyzing the provided images"
+            descriptions = {
+                "local_tools::final_answer": "📋 Preparing final answer",
+                "local_tools::request_clarification": "❓ Requesting clarification",
+                "local_tools::generate_image": "🎨 Creating image",
+                "local_tools::revise_plan": "📝 Revising execution plan"
+            }
+            if tool_name in descriptions:
+                return descriptions[tool_name]
+            if "research::" in tool_name: 
+                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
+            if requires_code: 
+                return "💻 Processing code"
+            if requires_image: 
+                return "🖼️ Analyzing images"
             return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
 
+        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
+            """Enhanced scratchpad compression that preserves key decisions and recent context"""
+            if len(scratchpad) <= target_size:
+                return scratchpad
+            
+            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
+            
+            # Extract key components
+            lines = scratchpad.split('\n')
+            plan_section = []
+            decisions = []
+            recent_observations = []
+            
+            current_section = None
+            for i, line in enumerate(lines):
+                if "### Execution Plan" in line or "### Updated Plan" in line:
+                    current_section = "plan"
+                elif "### Step" in line and ("Thought" in line or "Decision" in line):
+                    current_section = "decision"
+                elif "### Step" in line and "Observation" in line:
+                    current_section = "observation"
+                elif line.startswith("###"):
+                    current_section = None
+                
+                if current_section == "plan" and line.strip():
+                    plan_section.append(line)
+                elif current_section == "decision" and line.strip():
+                    decisions.append((i, line))
+                elif current_section == "observation" and line.strip():
+                    recent_observations.append((i, line))
+            
+            # Keep most recent items and important decisions
+            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
+            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
+            
+            compressed_parts = [
+                f"### Original Request\n{original_request}",
+                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
+                f"### Recent Key Decisions"
+            ]
+            
+            for _, decision in recent_decisions:
+                compressed_parts.append(decision)
+            
+            compressed_parts.append("### Recent Observations")
+            for _, obs in recent_obs:
+                compressed_parts.append(obs)
+            
+            compressed = '\n'.join(compressed_parts)
+            if len(compressed) > target_size:
+                # Final trim if still too long
+                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
+            
+            return compressed
+
         original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
         asset_store: Dict[str, Dict] = {}
+        decision_history = []  # Track all decisions made
         
-        planning_step_id = log_event_fn("📋 Creating a detailed plan...", MSG_TYPE.MSG_TYPE_STEP_START)
+        # Enhanced planning phase
+        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
         execution_plan = planner.decompose_task(original_user_prompt, context or "")
-        log_event_fn(f"✅ Plan ready ({len(execution_plan.tasks)} steps)", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
+        current_plan_version = 1
+        
+        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
+            "plan_version": current_plan_version,
+            "total_tasks": len(execution_plan.tasks),
+            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
+        })
+        
+        for i, task in enumerate(execution_plan.tasks):
+            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
+            
+        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
+        
+        # Enhanced initial state
+        initial_state_parts = [
+            f"### Original User Request\n{original_user_prompt}",
+            f"### Context\n{context or 'No additional context provided'}",
+            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
+            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
+        ]
+        
+        for i, task in enumerate(execution_plan.tasks): 
+            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
         
-        initial_state_parts = [f"### Execution Plan\n- Total tasks: {len(execution_plan.tasks)}"]
-        for i, task in enumerate(execution_plan.tasks): initial_state_parts.append(f"  {i+1}. {task.description}")
         if images:
+            initial_state_parts.append(f"### Provided Assets")
             for img_b64 in images:
                 img_uuid = str(uuid.uuid4())
                 asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
-                initial_state_parts.append(f"- User provided image, asset ID: {img_uuid}")
+                initial_state_parts.append(f"- Image asset: {img_uuid}")
+        
         current_scratchpad = "\n".join(initial_state_parts)
+        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
 
         formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
         completed_tasks, current_task_index = set(), 0
+        plan_revision_count = 0
         
+        # Main reasoning loop with enhanced decision tracking
         for i in range(max_reasoning_steps):
-            step_desc = f"🤔 Working on: {execution_plan.tasks[current_task_index].description}" if current_task_index < len(execution_plan.tasks) else f"🤔 Analyzing next steps... ({i+1}/{max_reasoning_steps})"
+            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
+            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
             reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
             
             try:
-                if len(current_scratchpad) > 12000:
-                    current_scratchpad = memory_manager.compress_scratchpad(current_scratchpad, original_user_prompt, 8000)
+                # Enhanced scratchpad management
+                if len(current_scratchpad) > max_scratchpad_size:
+                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
+                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
+                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
                 
-                reasoning_prompt = f"""--- AVAILABLE ACTIONS ---\n{formatted_tools_list}\n--- YOUR INTERNAL SCRATCHPAD ---\n{current_scratchpad}\n--- END SCRATCHPAD ---\n
-INSTRUCTIONS: Observe, think, and then act. Choose the single best next action to achieve: "{original_user_prompt}".
-Produce ONLY this JSON: {{"thought": "short reasoning", "action": {{"tool_name": "...", "requires_code_input": false, "requires_image_input": false}}}}"""
-                decision_data = self.generate_structured_content(prompt=reasoning_prompt, schema={"thought": "string", "action": "object"}, system_prompt=reasoning_system_prompt, temperature=decision_temperature, **llm_generation_kwargs)
+                # Enhanced reasoning prompt with better decision tracking
+                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
+
+=== AVAILABLE ACTIONS ===
+{formatted_tools_list}
+
+=== YOUR COMPLETE ANALYSIS HISTORY ===
+{current_scratchpad}
+=== END ANALYSIS HISTORY ===
+
+=== DECISION GUIDELINES ===
+1. **Review your progress**: Look at what you've already discovered and accomplished
+2. **Consider your current task**: Focus on the next logical step in your plan
+3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
+4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
+5. **Stay focused**: Each action should clearly advance toward the final goal
+
+=== YOUR NEXT DECISION ===
+Choose the single most appropriate action to take right now. Consider:
+- What specific step are you currently working on?
+- What information do you still need?
+- What would be most helpful for the user?
+
+Provide your decision as JSON:
+{{
+    "reasoning": "Explain your current thinking and why this action makes sense now",
+    "action": {{
+        "tool_name": "exact_tool_name",
+        "requires_code_input": false,
+        "requires_image_input": false,
+        "confidence": 0.8
+    }},
+    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
+}}"""
+
+                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
+                decision_data = self.generate_structured_content(
+                    prompt=reasoning_prompt, 
+                    schema={
+                        "reasoning": "string", 
+                        "action": "object",
+                        "plan_status": "string"
+                    }, 
+                    system_prompt=reasoning_system_prompt, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
                 
                 if not (decision_data and isinstance(decision_data.get("action"), dict)):
-                    log_event_fn("LLM failed to produce a valid action JSON.", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
-                    current_scratchpad += "\n\n### Step Failure\n- Error: Invalid decision JSON from LLM."
+                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
+                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
                     continue
 
                 action = decision_data.get("action", {})
-                tool_name, requires_code, requires_image = action.get("tool_name"), action.get("requires_code_input", False), action.get("requires_image_input", False)
-                current_scratchpad += f"\n\n### Step {i+1}: Thought\n{decision_data.get('thought', '')}"
+                reasoning = decision_data.get("reasoning", "No reasoning provided")
+                plan_status = decision_data.get("plan_status", "on_track")
+                tool_name = action.get("tool_name")
+                requires_code = action.get("requires_code_input", False)
+                requires_image = action.get("requires_image_input", False)
+                confidence = action.get("confidence", 0.5)
+                
+                # Track the decision
+                decision_history.append({
+                    "step": i+1,
+                    "tool_name": tool_name,
+                    "reasoning": reasoning,
+                    "confidence": confidence,
+                    "plan_status": plan_status
+                })
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
+                
+                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
+                    "tool_name": tool_name,
+                    "confidence": confidence,
+                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
+                })
+                
+                # Handle plan revision
+                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
+                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
+                    tool_name = "local_tools::revise_plan"  # Force plan revision
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                
+                # Handle clarification request
+                if tool_name == "local_tools::request_clarification":
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
                 
-                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP)
-                if tool_name == "local_tools::final_answer": break
+                # Handle clarification request
                 if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"Based on your thought process, what is the single question you need to ask the user?\n\nSCRATCHPAD:\n{current_scratchpad}\n\nQUESTION:"
-                    question = self.generate_text(clarification_prompt)
-                    return {"final_answer": self.remove_thinking_blocks(question), "clarification_required": True, "final_scratchpad": current_scratchpad, "tool_calls": tool_calls_this_turn, "sources": sources_this_turn, "error": None}
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+
+                # Handle plan revision
+                if tool_name == "local_tools::revise_plan":
+                    plan_revision_count += 1
+                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
+                    
+                    try:
+                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
 
+ORIGINAL REQUEST: "{original_user_prompt}"
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+REASON FOR REVISION: {reasoning}
+
+Create a new plan that reflects your current understanding. Consider:
+1. What have you already accomplished?
+2. What new information have you discovered?
+3. What steps are still needed?
+4. How can you be more efficient?
+
+Provide your revision as JSON:
+{{
+    "revision_reason": "Clear explanation of why the plan needed to change",
+    "new_plan": [
+        {{"step": 1, "description": "First revised step", "status": "pending"}},
+        {{"step": 2, "description": "Second revised step", "status": "pending"}}
+    ],
+    "confidence": 0.8
+}}"""
+
+                        revision_data = self.generate_structured_content(
+                            prompt=revision_prompt,
+                            schema={
+                                "revision_reason": "string",
+                                "new_plan": "array", 
+                                "confidence": "number"
+                            },
+                            temperature=0.3,
+                            **llm_generation_kwargs
+                        )
+                        
+                        if revision_data and revision_data.get("new_plan"):
+                            # Update the plan
+                            current_plan_version += 1
+                            new_tasks = []
+                            for task_data in revision_data["new_plan"]:
+                                task = TaskDecomposition()  # Assuming this class exists
+                                task.description = task_data.get("description", "Undefined step")
+                                task.status = TaskStatus.PENDING  # Reset all to pending
+                                new_tasks.append(task)
+                            
+                            execution_plan.tasks = new_tasks
+                            current_task_index = 0  # Reset to beginning
+                            
+                            # Update scratchpad with new plan
+                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
+                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
+                            current_scratchpad += f"**New Tasks**:\n"
+                            for i, task in enumerate(execution_plan.tasks):
+                                current_scratchpad += f"  {i+1}. {task.description}\n"
+                            
+                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
+                                "plan_version": current_plan_version,
+                                "new_task_count": len(execution_plan.tasks),
+                                "revision_reason": revision_data.get("revision_reason", "")
+                            })
+                            
+                            # Continue with the new plan
+                            continue
+                        else:
+                            raise ValueError("Failed to generate valid plan revision")
+                            
+                    except Exception as e:
+                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
+                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
+
+                # Prepare parameters for tool execution
                 param_assets = {}
                 if requires_code:
-                    code_prompt = f"Generate only the raw code required for the current step.\n\nSCRATCHPAD:\n{current_scratchpad}\n\nCODE:"
+                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
+                    code_prompt = f"""Generate the specific code needed for the current step.
+
+CURRENT CONTEXT:
+{current_scratchpad}
+
+CURRENT TASK: {tool_name}
+USER REQUEST: "{original_user_prompt}"
+
+Generate clean, functional code that addresses the specific requirements. Focus on:
+1. Solving the immediate problem
+2. Being clear and readable
+3. Including necessary imports and dependencies
+4. Adding helpful comments where appropriate
+
+CODE:"""
+
                     code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
                     code_uuid = f"code_asset_{uuid.uuid4()}"
                     asset_store[code_uuid] = {"type": "code", "content": code_content}
                     param_assets['code_asset_id'] = code_uuid
-                    log_event_fn("Code asset generated.", MSG_TYPE.MSG_TYPE_STEP)
+                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    
                 if requires_image:
                     image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
                     if image_assets:
                         param_assets['image_asset_id'] = image_assets[0]
+                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    else:
+                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
+
+                # Enhanced parameter generation
+                param_prompt = f"""Generate the optimal parameters for this tool execution.
+
+TOOL: {tool_name}
+CURRENT CONTEXT: {current_scratchpad}
+CURRENT REASONING: {reasoning}
+AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
+
+Based on your analysis and the current step you're working on, provide the most appropriate parameters.
+Be specific and purposeful in your parameter choices.
+
+Output format: {{"tool_params": {{...}}}}"""
+
+                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
+                param_data = self.generate_structured_content(
+                    prompt=param_prompt, 
+                    schema={"tool_params": "object"}, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
+                tool_params = param_data.get("tool_params", {}) if param_data else {}
+                
+                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
+                
+                # Hydrate parameters with assets
+                def _hydrate(data: Any, store: Dict) -> Any:
+                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
+                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
+                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
+                    return data
+                    
+                hydrated_params = _hydrate(tool_params, asset_store)
+                
+                # Execute the tool with detailed logging
+                start_time = time.time()
+                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
+                
+                try:
+                    if tool_name in rag_registry:
+                        query = hydrated_params.get("query", "")
+                        if not query:
+                            # Fall back to using reasoning as query
+                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
+                            
+                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        top_k = rag_tool_specs[tool_name]["default_top_k"]
+                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
+                        
+                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
+                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
+                        
+                        docs = []
+                        for d in raw_iter or []:
+                            doc_data = {
+                                "text": d.get("text", str(d)), 
+                                "score": d.get("score", 0) * 100, 
+                                "metadata": d.get("metadata", {})
+                            }
+                            docs.append(doc_data)
+                        
+                        kept = [x for x in docs if x['score'] >= min_sim]
+                        tool_result = {
+                            "status": "success", 
+                            "results": kept, 
+                            "total_found": len(docs),
+                            "kept_after_filtering": len(kept),
+                            "query_used": query
+                        }
+                        
+                        sources_this_turn.extend([{
+                            "source": tool_name, 
+                            "metadata": x["metadata"], 
+                            "score": x["score"]
+                        } for x in kept])
+                        
+                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
+                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
+                            "tool_name": tool_name,
+                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
+                        })
+                        
+                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
+                        
+                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
+                            "result_status": tool_result.get("status", "unknown"),
+                            "has_error": "error" in tool_result
+                        })
+                        
+                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
+                        image_prompt = hydrated_params.get("prompt", "")
+                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        # This would call your text-to-image functionality
+                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
+                        if image_result:
+                            image_uuid = f"generated_image_{uuid.uuid4()}"
+                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
+                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
+                        else:
+                            tool_result = {"status": "failure", "error": "Image generation failed"}
+                            
+                    else:
+                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
+                        
+                except Exception as e:
+                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
+                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
+                    tool_result = {"status": "failure", "error": error_msg}
+
+                response_time = time.time() - start_time
+                success = tool_result.get("status") == "success"
+                
+                # Record performance
+                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
+                
+                # Update task status
+                if success and current_task_index < len(execution_plan.tasks):
+                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
+                    completed_tasks.add(current_task_index)
+                    current_task_index += 1
+                
+                # Enhanced observation logging
+                observation_text = json.dumps(tool_result, indent=2)
+                if len(observation_text) > 1000:
+                    # Truncate very long results for scratchpad
+                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
+                    observation_text = json.dumps(truncated_result, indent=2)
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
+                current_scratchpad += f"**Tool Used**: {tool_name}\n"
+                current_scratchpad += f"**Success**: {success}\n"
+                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
+                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
+                
+                # Track tool call
+                tool_calls_this_turn.append({
+                    "name": tool_name,
+                    "params": tool_params,
+                    "result": tool_result,
+                    "response_time": response_time,
+                    "confidence": confidence,
+                    "reasoning": reasoning
+                })
+                
+                if success:
+                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "response_time": response_time,
+                        "confidence": confidence
+                    })
+                else:
+                    error_detail = tool_result.get("error", "No error detail provided.")
+                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "error": error_detail,
+                        "confidence": confidence
+                    })
+                    
+                    # Add failure handling to scratchpad
+                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
+                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
+                
+                # Log current progress
+                completed_count = len(completed_tasks)
+                total_tasks = len(execution_plan.tasks)
+                if total_tasks > 0:
+                    progress = (completed_count / total_tasks) * 100
+                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
+                
+                # Check if all tasks are completed
+                if completed_count >= total_tasks:
+                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                    
+            except Exception as ex:
+                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
+                trace_exception(ex)
+                
+                # Add error to scratchpad for context
+                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
+                
+                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
+
+        # Enhanced self-reflection
+        if enable_self_reflection and len(tool_calls_this_turn) > 0:
+            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
+            try:
+                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+TOOLS USED: {len(tool_calls_this_turn)}
+PLAN REVISIONS: {plan_revision_count}
+
+COMPLETE ANALYSIS:
+{current_scratchpad}
+
+Evaluate your performance on multiple dimensions:
+
+1. **Goal Achievement**: Did you fully address the user's request?
+2. **Process Efficiency**: Was your approach optimal given the available tools?
+3. **Information Quality**: Is the information you gathered accurate and relevant?
+4. **Decision Making**: Were your tool choices and parameters appropriate?
+5. **Adaptability**: How well did you handle unexpected results or plan changes?
+
+Provide your assessment as JSON:
+{{
+    "goal_achieved": true,
+    "effectiveness_score": 0.85,
+    "process_efficiency": 0.8,
+    "information_quality": 0.9,
+    "decision_making": 0.85,
+    "adaptability": 0.7,
+    "overall_confidence": 0.82,
+    "strengths": ["Clear reasoning", "Good tool selection"],
+    "areas_for_improvement": ["Could have been more efficient"],
+    "summary": "Successfully completed the user's request with high quality results",
+    "key_insights": ["Discovered that X was more important than initially thought"]
+}}"""
+
+                reflection_data = self.generate_structured_content(
+                    prompt=reflection_prompt,
+                    schema={
+                        "goal_achieved": "boolean",
+                        "effectiveness_score": "number",
+                        "process_efficiency": "number", 
+                        "information_quality": "number",
+                        "decision_making": "number",
+                        "adaptability": "number",
+                        "overall_confidence": "number",
+                        "strengths": "array",
+                        "areas_for_improvement": "array",
+                        "summary": "string",
+                        "key_insights": "array"
+                    },
+                    temperature=0.3,
+                    **llm_generation_kwargs
+                )
+                
+                if reflection_data:
+                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
+                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
+                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
+                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
+                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
+                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
+                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
+                    
+                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
+                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
+                        "goal_achieved": reflection_data.get('goal_achieved', False),
+                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
+                    })
+                else:
+                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+                    
+            except Exception as e:
+                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+
+        # Enhanced final synthesis
+        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
+        
+        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+CONTEXT: {context or "No additional context"}
+
+COMPLETE ANALYSIS AND WORK:
+{current_scratchpad}
+
+GUIDELINES for your response:
+1. **Be Complete**: Address all aspects of the user's request
+2. **Be Clear**: Organize your response logically and use clear language  
+3. **Be Helpful**: Provide actionable information and insights
+4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
+5. **Be Concise**: While being thorough, avoid unnecessary verbosity
+6. **Cite Sources**: If you used research tools, reference the information appropriately
+
+Your response should feel natural and conversational while being informative and valuable.
+
+FINAL RESPONSE:"""
+        
+        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
+        
+        final_answer_text = self.generate_text(
+            prompt=final_answer_prompt,
+            system_prompt=system_prompt,
+            stream=streaming_callback is not None,
+            streaming_callback=streaming_callback,
+            temperature=final_answer_temperature,
+            **llm_generation_kwargs
+        )
+        
+        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
+            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
+            return {
+                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
+                "error": final_answer_text["error"],
+                "final_scratchpad": current_scratchpad,
+                "tool_calls": tool_calls_this_turn,
+                "sources": sources_this_turn,
+                "decision_history": decision_history
+            }
+        
+        final_answer = self.remove_thinking_blocks(final_answer_text)
+        
+        # Calculate overall performance metrics
+        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
+        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
+        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
+        
+        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
+            "final_answer_length": len(final_answer),
+            "total_tools_used": len(tool_calls_this_turn),
+            "success_rate": success_rate,
+            "overall_confidence": overall_confidence
+        })
+
+        return {
+            "final_answer": final_answer,
+            "final_scratchpad": current_scratchpad,
+            "tool_calls": tool_calls_this_turn,
+            "sources": sources_this_turn,
+            "decision_history": decision_history,
+            "performance_stats": {
+                "total_steps": len(tool_calls_this_turn),
+                "successful_steps": successful_calls,
+                "success_rate": success_rate,
+                "average_confidence": overall_confidence,
+                "plan_revisions": plan_revision_count,
+                "total_reasoning_steps": len(decision_history)
+            },
+            "plan_evolution": {
+                "initial_tasks": len(execution_plan.tasks),
+                "final_version": current_plan_version,
+                "total_revisions": plan_revision_count
+            },
+            "clarification_required": False,
+            "overall_confidence": overall_confidence,
+            "error": None
+        }
+
+
+    def _execute_complex_reasoning_loop(
+        self, prompt, context, system_prompt, reasoning_system_prompt, images,
+        max_reasoning_steps, decision_temperature, final_answer_temperature,
+        streaming_callback, debug, enable_self_reflection, all_visible_tools,
+        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
+    ) -> Dict[str, Any]:
+        
+        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
+        
+        def _get_friendly_action_description(tool_name, requires_code, requires_image):
+            descriptions = {
+                "local_tools::final_answer": "📋 Preparing final answer",
+                "local_tools::request_clarification": "❓ Requesting clarification",
+                "local_tools::generate_image": "🎨 Creating image",
+                "local_tools::revise_plan": "📝 Revising execution plan"
+            }
+            if tool_name in descriptions:
+                return descriptions[tool_name]
+            if "research::" in tool_name: 
+                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
+            if requires_code: 
+                return "💻 Processing code"
+            if requires_image: 
+                return "🖼️ Analyzing images"
+            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
+
+        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
+            """Enhanced scratchpad compression that preserves key decisions and recent context"""
+            if len(scratchpad) <= target_size:
+                return scratchpad
+            
+            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
+            
+            # Extract key components
+            lines = scratchpad.split('\n')
+            plan_section = []
+            decisions = []
+            recent_observations = []
+            
+            current_section = None
+            for i, line in enumerate(lines):
+                if "### Execution Plan" in line or "### Updated Plan" in line:
+                    current_section = "plan"
+                elif "### Step" in line and ("Thought" in line or "Decision" in line):
+                    current_section = "decision"
+                elif "### Step" in line and "Observation" in line:
+                    current_section = "observation"
+                elif line.startswith("###"):
+                    current_section = None
+                
+                if current_section == "plan" and line.strip():
+                    plan_section.append(line)
+                elif current_section == "decision" and line.strip():
+                    decisions.append((i, line))
+                elif current_section == "observation" and line.strip():
+                    recent_observations.append((i, line))
+            
+            # Keep most recent items and important decisions
+            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
+            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
+            
+            compressed_parts = [
+                f"### Original Request\n{original_request}",
+                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
+                f"### Recent Key Decisions"
+            ]
+            
+            for _, decision in recent_decisions:
+                compressed_parts.append(decision)
+            
+            compressed_parts.append("### Recent Observations")
+            for _, obs in recent_obs:
+                compressed_parts.append(obs)
+            
+            compressed = '\n'.join(compressed_parts)
+            if len(compressed) > target_size:
+                # Final trim if still too long
+                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
+            
+            return compressed
+
+        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
+        asset_store: Dict[str, Dict] = {}
+        decision_history = []  # Track all decisions made
+        
+        # Enhanced planning phase
+        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
+        execution_plan = planner.decompose_task(original_user_prompt, context or "")
+        current_plan_version = 1
+        
+        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
+            "plan_version": current_plan_version,
+            "total_tasks": len(execution_plan.tasks),
+            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
+        })
+        
+        for i, task in enumerate(execution_plan.tasks):
+            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
+            
+        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
+        
+        # Enhanced initial state
+        initial_state_parts = [
+            f"### Original User Request\n{original_user_prompt}",
+            f"### Context\n{context or 'No additional context provided'}",
+            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
+            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
+        ]
+        
+        for i, task in enumerate(execution_plan.tasks): 
+            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
+        
+        if images:
+            initial_state_parts.append(f"### Provided Assets")
+            for img_b64 in images:
+                img_uuid = str(uuid.uuid4())
+                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
+                initial_state_parts.append(f"- Image asset: {img_uuid}")
+        
+        current_scratchpad = "\n".join(initial_state_parts)
+        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
+
+        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
+        completed_tasks, current_task_index = set(), 0
+        plan_revision_count = 0
+        
+        # Main reasoning loop with enhanced decision tracking
+        for i in range(max_reasoning_steps):
+            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
+            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
+            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
+            
+            try:
+                # Enhanced scratchpad management
+                if len(current_scratchpad) > max_scratchpad_size:
+                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
+                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
+                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
+                
+                # Enhanced reasoning prompt with better decision tracking
+                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
+
+=== AVAILABLE ACTIONS ===
+{formatted_tools_list}
+
+=== YOUR COMPLETE ANALYSIS HISTORY ===
+{current_scratchpad}
+=== END ANALYSIS HISTORY ===
+
+=== DECISION GUIDELINES ===
+1. **Review your progress**: Look at what you've already discovered and accomplished
+2. **Consider your current task**: Focus on the next logical step in your plan
+3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
+4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
+5. **Stay focused**: Each action should clearly advance toward the final goal
+
+=== YOUR NEXT DECISION ===
+Choose the single most appropriate action to take right now. Consider:
+- What specific step are you currently working on?
+- What information do you still need?
+- What would be most helpful for the user?
+
+Provide your decision as JSON:
+{{
+    "reasoning": "Explain your current thinking and why this action makes sense now",
+    "action": {{
+        "tool_name": "exact_tool_name",
+        "requires_code_input": false,
+        "requires_image_input": false,
+        "confidence": 0.8
+    }},
+    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
+}}"""
+
+                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
+                decision_data = self.generate_structured_content(
+                    prompt=reasoning_prompt, 
+                    schema={
+                        "reasoning": "string", 
+                        "action": "object",
+                        "plan_status": "string"
+                    }, 
+                    system_prompt=reasoning_system_prompt, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
+                
+                if not (decision_data and isinstance(decision_data.get("action"), dict)):
+                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
+                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
+                    continue
+
+                action = decision_data.get("action", {})
+                reasoning = decision_data.get("reasoning", "No reasoning provided")
+                plan_status = decision_data.get("plan_status", "on_track")
+                tool_name = action.get("tool_name")
+                requires_code = action.get("requires_code_input", False)
+                requires_image = action.get("requires_image_input", False)
+                confidence = action.get("confidence", 0.5)
+                
+                # Track the decision
+                decision_history.append({
+                    "step": i+1,
+                    "tool_name": tool_name,
+                    "reasoning": reasoning,
+                    "confidence": confidence,
+                    "plan_status": plan_status
+                })
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
+                
+                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
+                    "tool_name": tool_name,
+                    "confidence": confidence,
+                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
+                })
+                
+                # Handle plan revision
+                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
+                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
+                    tool_name = "local_tools::revise_plan"  # Force plan revision
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                
+                # Handle clarification request
+                if tool_name == "local_tools::request_clarification":
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                
+                # Handle clarification request
+                if tool_name == "local_tools::request_clarification":
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+
+                # Handle plan revision
+                if tool_name == "local_tools::revise_plan":
+                    plan_revision_count += 1
+                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
+                    
+                    try:
+                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+REASON FOR REVISION: {reasoning}
+
+Create a new plan that reflects your current understanding. Consider:
+1. What have you already accomplished?
+2. What new information have you discovered?
+3. What steps are still needed?
+4. How can you be more efficient?
+
+Provide your revision as JSON:
+{{
+    "revision_reason": "Clear explanation of why the plan needed to change",
+    "new_plan": [
+        {{"step": 1, "description": "First revised step", "status": "pending"}},
+        {{"step": 2, "description": "Second revised step", "status": "pending"}}
+    ],
+    "confidence": 0.8
+}}"""
+
+                        revision_data = self.generate_structured_content(
+                            prompt=revision_prompt,
+                            schema={
+                                "revision_reason": "string",
+                                "new_plan": "array", 
+                                "confidence": "number"
+                            },
+                            temperature=0.3,
+                            **llm_generation_kwargs
+                        )
+                        
+                        if revision_data and revision_data.get("new_plan"):
+                            # Update the plan
+                            current_plan_version += 1
+                            new_tasks = []
+                            for task_data in revision_data["new_plan"]:
+                                task = TaskDecomposition()  # Assuming this class exists
+                                task.description = task_data.get("description", "Undefined step")
+                                task.status = TaskStatus.PENDING  # Reset all to pending
+                                new_tasks.append(task)
+                            
+                            execution_plan.tasks = new_tasks
+                            current_task_index = 0  # Reset to beginning
+                            
+                            # Update scratchpad with new plan
+                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
+                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
+                            current_scratchpad += f"**New Tasks**:\n"
+                            for i, task in enumerate(execution_plan.tasks):
+                                current_scratchpad += f"  {i+1}. {task.description}\n"
+                            
+                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
+                                "plan_version": current_plan_version,
+                                "new_task_count": len(execution_plan.tasks),
+                                "revision_reason": revision_data.get("revision_reason", "")
+                            })
+                            
+                            # Continue with the new plan
+                            continue
+                        else:
+                            raise ValueError("Failed to generate valid plan revision")
+                            
+                    except Exception as e:
+                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
+                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
+
+                # Prepare parameters for tool execution
+                param_assets = {}
+                if requires_code:
+                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
+                    code_prompt = f"""Generate the specific code needed for the current step.
+
+CURRENT CONTEXT:
+{current_scratchpad}
+
+CURRENT TASK: {tool_name}
+USER REQUEST: "{original_user_prompt}"
+
+Generate clean, functional code that addresses the specific requirements. Focus on:
+1. Solving the immediate problem
+2. Being clear and readable
+3. Including necessary imports and dependencies
+4. Adding helpful comments where appropriate
+
+CODE:"""
+
+                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
+                    code_uuid = f"code_asset_{uuid.uuid4()}"
+                    asset_store[code_uuid] = {"type": "code", "content": code_content}
+                    param_assets['code_asset_id'] = code_uuid
+                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    
+                if requires_image:
+                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
+                    if image_assets:
+                        param_assets['image_asset_id'] = image_assets[0]
+                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    else:
+                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
+
+                # Enhanced parameter generation
+                param_prompt = f"""Generate the optimal parameters for this tool execution.
+
+TOOL: {tool_name}
+CURRENT CONTEXT: {current_scratchpad}
+CURRENT REASONING: {reasoning}
+AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
+
+Based on your analysis and the current step you're working on, provide the most appropriate parameters.
+Be specific and purposeful in your parameter choices.
+
+Output format: {{"tool_params": {{...}}}}"""
+
+                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
+                param_data = self.generate_structured_content(
+                    prompt=param_prompt, 
+                    schema={"tool_params": "object"}, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
+                tool_params = param_data.get("tool_params", {}) if param_data else {}
+                
+                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
+                
+                # Hydrate parameters with assets
+                def _hydrate(data: Any, store: Dict) -> Any:
+                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
+                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
+                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
+                    return data
+                    
+                hydrated_params = _hydrate(tool_params, asset_store)
+                
+                # Execute the tool with detailed logging
+                start_time = time.time()
+                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
+                
+                try:
+                    if tool_name in rag_registry:
+                        query = hydrated_params.get("query", "")
+                        if not query:
+                            # Fall back to using reasoning as query
+                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
+                            
+                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        top_k = rag_tool_specs[tool_name]["default_top_k"]
+                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
+                        
+                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
+                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
+                        
+                        docs = []
+                        for d in raw_iter or []:
+                            doc_data = {
+                                "text": d.get("text", str(d)), 
+                                "score": d.get("score", 0) * 100, 
+                                "metadata": d.get("metadata", {})
+                            }
+                            docs.append(doc_data)
+                        
+                        kept = [x for x in docs if x['score'] >= min_sim]
+                        tool_result = {
+                            "status": "success", 
+                            "results": kept, 
+                            "total_found": len(docs),
+                            "kept_after_filtering": len(kept),
+                            "query_used": query
+                        }
+                        
+                        sources_this_turn.extend([{
+                            "source": tool_name, 
+                            "metadata": x["metadata"], 
+                            "score": x["score"]
+                        } for x in kept])
+                        
+                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
+                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
+                            "tool_name": tool_name,
+                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
+                        })
+                        
+                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
+                        
+                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
+                            "result_status": tool_result.get("status", "unknown"),
+                            "has_error": "error" in tool_result
+                        })
+                        
+                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
+                        image_prompt = hydrated_params.get("prompt", "")
+                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        # This would call your text-to-image functionality
+                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
+                        if image_result:
+                            image_uuid = f"generated_image_{uuid.uuid4()}"
+                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
+                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
+                        else:
+                            tool_result = {"status": "failure", "error": "Image generation failed"}
+                            
+                    else:
+                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
+                        
+                except Exception as e:
+                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
+                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
+                    tool_result = {"status": "failure", "error": error_msg}
+
+                response_time = time.time() - start_time
+                success = tool_result.get("status") == "success"
+                
+                # Record performance
+                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
+                
+                # Update task status
+                if success and current_task_index < len(execution_plan.tasks):
+                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
+                    completed_tasks.add(current_task_index)
+                    current_task_index += 1
+                
+                # Enhanced observation logging
+                observation_text = json.dumps(tool_result, indent=2)
+                if len(observation_text) > 1000:
+                    # Truncate very long results for scratchpad
+                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
+                    observation_text = json.dumps(truncated_result, indent=2)
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
+                current_scratchpad += f"**Tool Used**: {tool_name}\n"
+                current_scratchpad += f"**Success**: {success}\n"
+                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
+                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
+                
+                # Track tool call
+                tool_calls_this_turn.append({
+                    "name": tool_name,
+                    "params": tool_params,
+                    "result": tool_result,
+                    "response_time": response_time,
+                    "confidence": confidence,
+                    "reasoning": reasoning
+                })
+                
+                if success:
+                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "response_time": response_time,
+                        "confidence": confidence
+                    })
+                else:
+                    error_detail = tool_result.get("error", "No error detail provided.")
+                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "error": error_detail,
+                        "confidence": confidence
+                    })
+                    
+                    # Add failure handling to scratchpad
+                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
+                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
+                
+                # Log current progress
+                completed_count = len(completed_tasks)
+                total_tasks = len(execution_plan.tasks)
+                if total_tasks > 0:
+                    progress = (completed_count / total_tasks) * 100
+                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
+                
+                # Check if all tasks are completed
+                if completed_count >= total_tasks:
+                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                    
+            except Exception as ex:
+                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
+                trace_exception(ex)
+                
+                # Add error to scratchpad for context
+                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
+                
+                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
+
+        # Enhanced self-reflection
+        if enable_self_reflection and len(tool_calls_this_turn) > 0:
+            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
+            try:
+                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+TOOLS USED: {len(tool_calls_this_turn)}
+PLAN REVISIONS: {plan_revision_count}
+
+COMPLETE ANALYSIS:
+{current_scratchpad}
+
+Evaluate your performance on multiple dimensions:
+
+1. **Goal Achievement**: Did you fully address the user's request?
+2. **Process Efficiency**: Was your approach optimal given the available tools?
+3. **Information Quality**: Is the information you gathered accurate and relevant?
+4. **Decision Making**: Were your tool choices and parameters appropriate?
+5. **Adaptability**: How well did you handle unexpected results or plan changes?
+
+Provide your assessment as JSON:
+{{
+    "goal_achieved": true,
+    "effectiveness_score": 0.85,
+    "process_efficiency": 0.8,
+    "information_quality": 0.9,
+    "decision_making": 0.85,
+    "adaptability": 0.7,
+    "overall_confidence": 0.82,
+    "strengths": ["Clear reasoning", "Good tool selection"],
+    "areas_for_improvement": ["Could have been more efficient"],
+    "summary": "Successfully completed the user's request with high quality results",
+    "key_insights": ["Discovered that X was more important than initially thought"]
+}}"""
+
+                reflection_data = self.generate_structured_content(
+                    prompt=reflection_prompt,
+                    schema={
+                        "goal_achieved": "boolean",
+                        "effectiveness_score": "number",
+                        "process_efficiency": "number", 
+                        "information_quality": "number",
+                        "decision_making": "number",
+                        "adaptability": "number",
+                        "overall_confidence": "number",
+                        "strengths": "array",
+                        "areas_for_improvement": "array",
+                        "summary": "string",
+                        "key_insights": "array"
+                    },
+                    temperature=0.3,
+                    **llm_generation_kwargs
+                )
+                
+                if reflection_data:
+                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
+                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
+                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
+                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
+                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
+                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
+                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
+                    
+                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
+                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
+                        "goal_achieved": reflection_data.get('goal_achieved', False),
+                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
+                    })
+                else:
+                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+                    
+            except Exception as e:
+                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+
+        # Enhanced final synthesis
+        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
+        
+        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+CONTEXT: {context or "No additional context"}
+
+COMPLETE ANALYSIS AND WORK:
+{current_scratchpad}
+
+GUIDELINES for your response:
+1. **Be Complete**: Address all aspects of the user's request
+2. **Be Clear**: Organize your response logically and use clear language  
+3. **Be Helpful**: Provide actionable information and insights
+4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
+5. **Be Concise**: While being thorough, avoid unnecessary verbosity
+6. **Cite Sources**: If you used research tools, reference the information appropriately
+
+Your response should feel natural and conversational while being informative and valuable.
+
+FINAL RESPONSE:"""
+        
+        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
+        
+        final_answer_text = self.generate_text(
+            prompt=final_answer_prompt,
+            system_prompt=system_prompt,
+            stream=streaming_callback is not None,
+            streaming_callback=streaming_callback,
+            temperature=final_answer_temperature,
+            **llm_generation_kwargs
+        )
+        
+        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
+            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
+            return {
+                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
+                "error": final_answer_text["error"],
+                "final_scratchpad": current_scratchpad,
+                "tool_calls": tool_calls_this_turn,
+                "sources": sources_this_turn,
+                "decision_history": decision_history
+            }
+        
+        final_answer = self.remove_thinking_blocks(final_answer_text)
+        
+        # Calculate overall performance metrics
+        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
+        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
+        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
+        
+        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
+            "final_answer_length": len(final_answer),
+            "total_tools_used": len(tool_calls_this_turn),
+            "success_rate": success_rate,
+            "overall_confidence": overall_confidence
+        })
+
+        return {
+            "final_answer": final_answer,
+            "final_scratchpad": current_scratchpad,
+            "tool_calls": tool_calls_this_turn,
+            "sources": sources_this_turn,
+            "decision_history": decision_history,
+            "performance_stats": {
+                "total_steps": len(tool_calls_this_turn),
+                "successful_steps": successful_calls,
+                "success_rate": success_rate,
+                "average_confidence": overall_confidence,
+                "plan_revisions": plan_revision_count,
+                "total_reasoning_steps": len(decision_history)
+            },
+            "plan_evolution": {
+                "initial_tasks": len(execution_plan.tasks),
+                "final_version": current_plan_version,
+                "total_revisions": plan_revision_count
+            },
+            "clarification_required": False,
+            "overall_confidence": overall_confidence,
+            "error": None
+        }
+
+
+    def _execute_complex_reasoning_loop(
+        self, prompt, context, system_prompt, reasoning_system_prompt, images,
+        max_reasoning_steps, decision_temperature, final_answer_temperature,
+        streaming_callback, debug, enable_self_reflection, all_visible_tools,
+        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
+    ) -> Dict[str, Any]:
+        
+        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
+        
+        def _get_friendly_action_description(tool_name, requires_code, requires_image):
+            descriptions = {
+                "local_tools::final_answer": "📋 Preparing final answer",
+                "local_tools::request_clarification": "❓ Requesting clarification",
+                "local_tools::generate_image": "🎨 Creating image",
+                "local_tools::revise_plan": "📝 Revising execution plan"
+            }
+            if tool_name in descriptions:
+                return descriptions[tool_name]
+            if "research::" in tool_name: 
+                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
+            if requires_code: 
+                return "💻 Processing code"
+            if requires_image: 
+                return "🖼️ Analyzing images"
+            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
+
+        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
+            """Enhanced scratchpad compression that preserves key decisions and recent context"""
+            if len(scratchpad) <= target_size:
+                return scratchpad
+            
+            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
+            
+            # Extract key components
+            lines = scratchpad.split('\n')
+            plan_section = []
+            decisions = []
+            recent_observations = []
+            
+            current_section = None
+            for i, line in enumerate(lines):
+                if "### Execution Plan" in line or "### Updated Plan" in line:
+                    current_section = "plan"
+                elif "### Step" in line and ("Thought" in line or "Decision" in line):
+                    current_section = "decision"
+                elif "### Step" in line and "Observation" in line:
+                    current_section = "observation"
+                elif line.startswith("###"):
+                    current_section = None
+                
+                if current_section == "plan" and line.strip():
+                    plan_section.append(line)
+                elif current_section == "decision" and line.strip():
+                    decisions.append((i, line))
+                elif current_section == "observation" and line.strip():
+                    recent_observations.append((i, line))
+            
+            # Keep most recent items and important decisions
+            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
+            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
+            
+            compressed_parts = [
+                f"### Original Request\n{original_request}",
+                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
+                f"### Recent Key Decisions"
+            ]
+            
+            for _, decision in recent_decisions:
+                compressed_parts.append(decision)
+            
+            compressed_parts.append("### Recent Observations")
+            for _, obs in recent_obs:
+                compressed_parts.append(obs)
+            
+            compressed = '\n'.join(compressed_parts)
+            if len(compressed) > target_size:
+                # Final trim if still too long
+                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
+            
+            return compressed
+
+        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
+        asset_store: Dict[str, Dict] = {}
+        decision_history = []  # Track all decisions made
+        
+        # Enhanced planning phase
+        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
+        execution_plan = planner.decompose_task(original_user_prompt, context or "")
+        current_plan_version = 1
+        
+        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
+            "plan_version": current_plan_version,
+            "total_tasks": len(execution_plan.tasks),
+            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
+        })
+        
+        for i, task in enumerate(execution_plan.tasks):
+            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
+            
+        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
+        
+        # Enhanced initial state
+        initial_state_parts = [
+            f"### Original User Request\n{original_user_prompt}",
+            f"### Context\n{context or 'No additional context provided'}",
+            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
+            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
+        ]
+        
+        for i, task in enumerate(execution_plan.tasks): 
+            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
+        
+        if images:
+            initial_state_parts.append(f"### Provided Assets")
+            for img_b64 in images:
+                img_uuid = str(uuid.uuid4())
+                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
+                initial_state_parts.append(f"- Image asset: {img_uuid}")
+        
+        current_scratchpad = "\n".join(initial_state_parts)
+        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
+
+        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
+        completed_tasks, current_task_index = set(), 0
+        plan_revision_count = 0
+        
+        # Main reasoning loop with enhanced decision tracking
+        for i in range(max_reasoning_steps):
+            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
+            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
+            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
+            
+            try:
+                # Enhanced scratchpad management
+                if len(current_scratchpad) > max_scratchpad_size:
+                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
+                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
+                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
+                
+                # Enhanced reasoning prompt with better decision tracking
+                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
+
+=== AVAILABLE ACTIONS ===
+{formatted_tools_list}
+
+=== YOUR COMPLETE ANALYSIS HISTORY ===
+{current_scratchpad}
+=== END ANALYSIS HISTORY ===
+
+=== DECISION GUIDELINES ===
+1. **Review your progress**: Look at what you've already discovered and accomplished
+2. **Consider your current task**: Focus on the next logical step in your plan
+3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
+4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
+5. **Stay focused**: Each action should clearly advance toward the final goal
+
+=== YOUR NEXT DECISION ===
+Choose the single most appropriate action to take right now. Consider:
+- What specific step are you currently working on?
+- What information do you still need?
+- What would be most helpful for the user?
+
+Provide your decision as JSON:
+{{
+    "reasoning": "Explain your current thinking and why this action makes sense now",
+    "action": {{
+        "tool_name": "exact_tool_name",
+        "requires_code_input": false,
+        "requires_image_input": false,
+        "confidence": 0.8
+    }},
+    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
+}}"""
+
+                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
+                decision_data = self.generate_structured_content(
+                    prompt=reasoning_prompt, 
+                    schema={
+                        "reasoning": "string", 
+                        "action": "object",
+                        "plan_status": "string"
+                    }, 
+                    system_prompt=reasoning_system_prompt, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
+                
+                if not (decision_data and isinstance(decision_data.get("action"), dict)):
+                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
+                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
+                    continue
+
+                action = decision_data.get("action", {})
+                reasoning = decision_data.get("reasoning", "No reasoning provided")
+                plan_status = decision_data.get("plan_status", "on_track")
+                tool_name = action.get("tool_name")
+                requires_code = action.get("requires_code_input", False)
+                requires_image = action.get("requires_image_input", False)
+                confidence = action.get("confidence", 0.5)
+                
+                # Track the decision
+                decision_history.append({
+                    "step": i+1,
+                    "tool_name": tool_name,
+                    "reasoning": reasoning,
+                    "confidence": confidence,
+                    "plan_status": plan_status
+                })
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
+                
+                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
+                    "tool_name": tool_name,
+                    "confidence": confidence,
+                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
+                })
+                
+                # Handle plan revision
+                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
+                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
+                    tool_name = "local_tools::revise_plan"  # Force plan revision
+                
+                # Handle final answer
+                if tool_name == "local_tools::final_answer": 
+                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
+                    break
+                
+                # Handle clarification request
+                if tool_name == "local_tools::request_clarification":
+                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
+
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+Generate a clear, specific question that will help you proceed effectively:"""
+                    
+                    question = self.generate_text(clarification_prompt, temperature=0.3)
+                    question = self.remove_thinking_blocks(question)
+                    
+                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
+                    return {
+                        "final_answer": question, 
+                        "clarification_required": True, 
+                        "final_scratchpad": current_scratchpad,
+                        "tool_calls": tool_calls_this_turn, 
+                        "sources": sources_this_turn, 
+                        "error": None,
+                        "decision_history": decision_history
+                    }
+
+                # Handle plan revision
+                if tool_name == "local_tools::revise_plan":
+                    plan_revision_count += 1
+                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
+                    
+                    try:
+                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+CURRENT ANALYSIS:
+{current_scratchpad}
+
+REASON FOR REVISION: {reasoning}
+
+Create a new plan that reflects your current understanding. Consider:
+1. What have you already accomplished?
+2. What new information have you discovered?
+3. What steps are still needed?
+4. How can you be more efficient?
+
+Provide your revision as JSON:
+{{
+    "revision_reason": "Clear explanation of why the plan needed to change",
+    "new_plan": [
+        {{"step": 1, "description": "First revised step", "status": "pending"}},
+        {{"step": 2, "description": "Second revised step", "status": "pending"}}
+    ],
+    "confidence": 0.8
+}}"""
+
+                        revision_data = self.generate_structured_content(
+                            prompt=revision_prompt,
+                            schema={
+                                "revision_reason": "string",
+                                "new_plan": "array", 
+                                "confidence": "number"
+                            },
+                            temperature=0.3,
+                            **llm_generation_kwargs
+                        )
+                        
+                        if revision_data and revision_data.get("new_plan"):
+                            # Update the plan
+                            current_plan_version += 1
+                            new_tasks = []
+                            for task_data in revision_data["new_plan"]:
+                                task = TaskDecomposition()  # Assuming this class exists
+                                task.description = task_data.get("description", "Undefined step")
+                                task.status = TaskStatus.PENDING  # Reset all to pending
+                                new_tasks.append(task)
+                            
+                            execution_plan.tasks = new_tasks
+                            current_task_index = 0  # Reset to beginning
+                            
+                            # Update scratchpad with new plan
+                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
+                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
+                            current_scratchpad += f"**New Tasks**:\n"
+                            for i, task in enumerate(execution_plan.tasks):
+                                current_scratchpad += f"  {i+1}. {task.description}\n"
+                            
+                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
+                                "plan_version": current_plan_version,
+                                "new_task_count": len(execution_plan.tasks),
+                                "revision_reason": revision_data.get("revision_reason", "")
+                            })
+                            
+                            # Continue with the new plan
+                            continue
+                        else:
+                            raise ValueError("Failed to generate valid plan revision")
+                            
+                    except Exception as e:
+                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
+                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
+
+                # Prepare parameters for tool execution
+                param_assets = {}
+                if requires_code:
+                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
+                    code_prompt = f"""Generate the specific code needed for the current step.
+
+CURRENT CONTEXT:
+{current_scratchpad}
+
+CURRENT TASK: {tool_name}
+USER REQUEST: "{original_user_prompt}"
+
+Generate clean, functional code that addresses the specific requirements. Focus on:
+1. Solving the immediate problem
+2. Being clear and readable
+3. Including necessary imports and dependencies
+4. Adding helpful comments where appropriate
+
+CODE:"""
+
+                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
+                    code_uuid = f"code_asset_{uuid.uuid4()}"
+                    asset_store[code_uuid] = {"type": "code", "content": code_content}
+                    param_assets['code_asset_id'] = code_uuid
+                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    
+                if requires_image:
+                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
+                    if image_assets:
+                        param_assets['image_asset_id'] = image_assets[0]
+                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
+                    else:
+                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
+
+                # Enhanced parameter generation
+                param_prompt = f"""Generate the optimal parameters for this tool execution.
+
+TOOL: {tool_name}
+CURRENT CONTEXT: {current_scratchpad}
+CURRENT REASONING: {reasoning}
+AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
+
+Based on your analysis and the current step you're working on, provide the most appropriate parameters.
+Be specific and purposeful in your parameter choices.
 
-                param_prompt = f"""Fill the parameters for the tool: '{tool_name}'. Available assets: {json.dumps(param_assets)}.
-SCRATCHPAD:\n{current_scratchpad}\n
-Output only: {{"tool_params": {{...}}}}"""
-                param_data = self.generate_structured_content(prompt=param_prompt, schema={"tool_params": "object"}, temperature=decision_temperature, **llm_generation_kwargs)
+Output format: {{"tool_params": {{...}}}}"""
+
+                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
+                param_data = self.generate_structured_content(
+                    prompt=param_prompt, 
+                    schema={"tool_params": "object"}, 
+                    temperature=decision_temperature, 
+                    **llm_generation_kwargs
+                )
                 tool_params = param_data.get("tool_params", {}) if param_data else {}
                 
+                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
+                
+                # Hydrate parameters with assets
                 def _hydrate(data: Any, store: Dict) -> Any:
                     if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
                     if isinstance(data, list): return [_hydrate(item, store) for item in data]
                     if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
                     return data
+                    
                 hydrated_params = _hydrate(tool_params, asset_store)
                 
-                start_time, tool_result = time.time(), {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
+                # Execute the tool with detailed logging
+                start_time = time.time()
+                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
+                
                 try:
                     if tool_name in rag_registry:
                         query = hydrated_params.get("query", "")
-                        top_k, min_sim = rag_tool_specs[tool_name]["default_top_k"], rag_tool_specs[tool_name]["default_min_sim"]
+                        if not query:
+                            # Fall back to using reasoning as query
+                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
+                            
+                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        top_k = rag_tool_specs[tool_name]["default_top_k"]
+                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
+                        
                         raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
                         raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
-                        docs = [{"text": d.get("text", str(d)), "score": d.get("score", 0)*100, "metadata": d.get("metadata", {})} for d in raw_iter or []]
+                        
+                        docs = []
+                        for d in raw_iter or []:
+                            doc_data = {
+                                "text": d.get("text", str(d)), 
+                                "score": d.get("score", 0) * 100, 
+                                "metadata": d.get("metadata", {})
+                            }
+                            docs.append(doc_data)
+                        
                         kept = [x for x in docs if x['score'] >= min_sim]
-                        tool_result = {"status": "success", "results": kept, "dropped": len(docs) - len(kept)}
-                        sources_this_turn.extend([{"source": tool_name, "metadata": x["metadata"], "score": x["score"]} for x in kept])
-                    elif hasattr(self, "mcp"):
+                        tool_result = {
+                            "status": "success", 
+                            "results": kept, 
+                            "total_found": len(docs),
+                            "kept_after_filtering": len(kept),
+                            "query_used": query
+                        }
+                        
+                        sources_this_turn.extend([{
+                            "source": tool_name, 
+                            "metadata": x["metadata"], 
+                            "score": x["score"]
+                        } for x in kept])
+                        
+                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
+                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
+                            "tool_name": tool_name,
+                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
+                        })
+                        
                         tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
+                        
+                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
+                            "result_status": tool_result.get("status", "unknown"),
+                            "has_error": "error" in tool_result
+                        })
+                        
+                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
+                        image_prompt = hydrated_params.get("prompt", "")
+                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
+                        
+                        # This would call your text-to-image functionality
+                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
+                        if image_result:
+                            image_uuid = f"generated_image_{uuid.uuid4()}"
+                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
+                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
+                        else:
+                            tool_result = {"status": "failure", "error": "Image generation failed"}
+                            
+                    else:
+                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
+                        
                 except Exception as e:
-                    error_msg = f"Exception during '{tool_name}' execution: {e}"
+                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
                     log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
                     tool_result = {"status": "failure", "error": error_msg}
 
                 response_time = time.time() - start_time
                 success = tool_result.get("status") == "success"
-                performance_tracker.record_tool_usage(tool_name, success, 0.8, response_time, tool_result.get("error"))
                 
+                # Record performance
+                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
+                
+                # Update task status
                 if success and current_task_index < len(execution_plan.tasks):
                     execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
+                    completed_tasks.add(current_task_index)
                     current_task_index += 1
                 
-                observation_text = f"```json\n{json.dumps(tool_result, indent=2)}\n```"
-                tool_calls_this_turn.append({"name": tool_name, "params": tool_params, "result": tool_result, "response_time": response_time})
-                current_scratchpad += f"\n\n### Step {i+1}: Observation\n- Action: `{tool_name}`\n- Result:\n{observation_text}"
+                # Enhanced observation logging
+                observation_text = json.dumps(tool_result, indent=2)
+                if len(observation_text) > 1000:
+                    # Truncate very long results for scratchpad
+                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
+                    observation_text = json.dumps(truncated_result, indent=2)
+                
+                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
+                current_scratchpad += f"**Tool Used**: {tool_name}\n"
+                current_scratchpad += f"**Success**: {success}\n"
+                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
+                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
+                
+                # Track tool call
+                tool_calls_this_turn.append({
+                    "name": tool_name,
+                    "params": tool_params,
+                    "result": tool_result,
+                    "response_time": response_time,
+                    "confidence": confidence,
+                    "reasoning": reasoning
+                })
                 
                 if success:
-                    log_event_fn(f"✅ Step completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
+                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "response_time": response_time,
+                        "confidence": confidence
+                    })
                 else:
                     error_detail = tool_result.get("error", "No error detail provided.")
-                    log_event_fn(f"Tool reported failure: {error_detail}", MSG_TYPE.MSG_TYPE_WARNING)
-                    log_event_fn(f"⚠️ Step completed with issues", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={"error": error_detail})
+                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
+                        "tool_name": tool_name,
+                        "error": error_detail,
+                        "confidence": confidence
+                    })
+                    
+                    # Add failure handling to scratchpad
+                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
+                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
+                
+                # Log current progress
+                completed_count = len(completed_tasks)
+                total_tasks = len(execution_plan.tasks)
+                if total_tasks > 0:
+                    progress = (completed_count / total_tasks) * 100
+                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
                 
-                if len(completed_tasks) == len(execution_plan.tasks): break
+                # Check if all tasks are completed
+                if completed_count >= total_tasks:
+                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
+                    break
                     
             except Exception as ex:
-                log_event_fn(f"An unexpected error occurred in reasoning loop: {ex}", MSG_TYPE.MSG_TYPE_EXCEPTION, event_id=reasoning_step_id)
+                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
                 trace_exception(ex)
-                log_event_fn("⚠️ Encountered an issue, adjusting approach...", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
+                
+                # Add error to scratchpad for context
+                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
+                
+                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
 
-        if enable_self_reflection and len(tool_calls_this_turn) > 1:
-            reflection_id = log_event_fn("🤔 Reviewing my work...", MSG_TYPE.MSG_TYPE_STEP_START)
+        # Enhanced self-reflection
+        if enable_self_reflection and len(tool_calls_this_turn) > 0:
+            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
             try:
-                reflection_prompt = f"""Review the user request and your work. Was the goal achieved effectively?
-REQUEST: "{original_user_prompt}"
-SCRATCHPAD:\n{current_scratchpad}\n
-JSON assessment: {{"goal_achieved": true, "effectiveness_score": 0.8, "summary": "..."}}"""
-                reflection_data = self.generate_structured_content(prompt=reflection_prompt, schema={"goal_achieved": "boolean", "effectiveness_score": "number", "summary": "string"}, temperature=0.3, **llm_generation_kwargs)
-                if reflection_data: current_scratchpad += f"\n\n### Self-Reflection\n- Goal Achieved: {reflection_data.get('goal_achieved')}\n- Effectiveness: {reflection_data.get('effectiveness_score')}"
-                log_event_fn("✅ Quality check completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id)
+                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+TOOLS USED: {len(tool_calls_this_turn)}
+PLAN REVISIONS: {plan_revision_count}
+
+COMPLETE ANALYSIS:
+{current_scratchpad}
+
+Evaluate your performance on multiple dimensions:
+
+1. **Goal Achievement**: Did you fully address the user's request?
+2. **Process Efficiency**: Was your approach optimal given the available tools?
+3. **Information Quality**: Is the information you gathered accurate and relevant?
+4. **Decision Making**: Were your tool choices and parameters appropriate?
+5. **Adaptability**: How well did you handle unexpected results or plan changes?
+
+Provide your assessment as JSON:
+{{
+    "goal_achieved": true,
+    "effectiveness_score": 0.85,
+    "process_efficiency": 0.8,
+    "information_quality": 0.9,
+    "decision_making": 0.85,
+    "adaptability": 0.7,
+    "overall_confidence": 0.82,
+    "strengths": ["Clear reasoning", "Good tool selection"],
+    "areas_for_improvement": ["Could have been more efficient"],
+    "summary": "Successfully completed the user's request with high quality results",
+    "key_insights": ["Discovered that X was more important than initially thought"]
+}}"""
+
+                reflection_data = self.generate_structured_content(
+                    prompt=reflection_prompt,
+                    schema={
+                        "goal_achieved": "boolean",
+                        "effectiveness_score": "number",
+                        "process_efficiency": "number", 
+                        "information_quality": "number",
+                        "decision_making": "number",
+                        "adaptability": "number",
+                        "overall_confidence": "number",
+                        "strengths": "array",
+                        "areas_for_improvement": "array",
+                        "summary": "string",
+                        "key_insights": "array"
+                    },
+                    temperature=0.3,
+                    **llm_generation_kwargs
+                )
+                
+                if reflection_data:
+                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
+                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
+                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
+                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
+                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
+                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
+                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
+                    
+                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
+                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
+                        "goal_achieved": reflection_data.get('goal_achieved', False),
+                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
+                    })
+                else:
+                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+                    
             except Exception as e:
-                log_event_fn(f"Self-review failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
+                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
 
-        synthesis_id = log_event_fn("📝 Preparing your complete answer...", MSG_TYPE.MSG_TYPE_STEP_START)
-        final_answer_prompt = f"""Synthesize a comprehensive, user-friendly final answer based on your complete analysis.
-USER REQUEST: "{original_user_prompt}"
-FULL SCRATCHPAD:\n{current_scratchpad}\n---
-FINAL ANSWER:"""
+        # Enhanced final synthesis
+        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
+        
+        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
+
+ORIGINAL REQUEST: "{original_user_prompt}"
+CONTEXT: {context or "No additional context"}
+
+COMPLETE ANALYSIS AND WORK:
+{current_scratchpad}
+
+GUIDELINES for your response:
+1. **Be Complete**: Address all aspects of the user's request
+2. **Be Clear**: Organize your response logically and use clear language  
+3. **Be Helpful**: Provide actionable information and insights
+4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
+5. **Be Concise**: While being thorough, avoid unnecessary verbosity
+6. **Cite Sources**: If you used research tools, reference the information appropriately
+
+Your response should feel natural and conversational while being informative and valuable.
+
+FINAL RESPONSE:"""
+        
+        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
+        
+        final_answer_text = self.generate_text(
+            prompt=final_answer_prompt,
+            system_prompt=system_prompt,
+            stream=streaming_callback is not None,
+            streaming_callback=streaming_callback,
+            temperature=final_answer_temperature,
+            **llm_generation_kwargs
+        )
         
-        final_answer_text = self.generate_text(prompt=final_answer_prompt, system_prompt=system_prompt, stream=streaming_callback is not None, streaming_callback=streaming_callback, temperature=final_answer_temperature, **llm_generation_kwargs)
         if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
-            return {"final_answer": "", "error": final_answer_text["error"], "final_scratchpad": current_scratchpad}
+            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
+            return {
+                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
+                "error": final_answer_text["error"],
+                "final_scratchpad": current_scratchpad,
+                "tool_calls": tool_calls_this_turn,
+                "sources": sources_this_turn,
+                "decision_history": decision_history
+            }
         
         final_answer = self.remove_thinking_blocks(final_answer_text)
-        log_event_fn("✅ Answer ready!", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id)
+        
+        # Calculate overall performance metrics
+        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
+        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
+        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
+        
+        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
+            "final_answer_length": len(final_answer),
+            "total_tools_used": len(tool_calls_this_turn),
+            "success_rate": success_rate,
+            "overall_confidence": overall_confidence
+        })
 
-        overall_confidence = sum(c.get('confidence', 0.5) for c in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
         return {
-            "final_answer": final_answer, "final_scratchpad": current_scratchpad,
-            "tool_calls": tool_calls_this_turn, "sources": sources_this_turn,
-            "performance_stats": {"total_steps": len(tool_calls_this_turn), "average_confidence": overall_confidence},
-            "clarification_required": False, "overall_confidence": overall_confidence, "error": None
+            "final_answer": final_answer,
+            "final_scratchpad": current_scratchpad,
+            "tool_calls": tool_calls_this_turn,
+            "sources": sources_this_turn,
+            "decision_history": decision_history,
+            "performance_stats": {
+                "total_steps": len(tool_calls_this_turn),
+                "successful_steps": successful_calls,
+                "success_rate": success_rate,
+                "average_confidence": overall_confidence,
+                "plan_revisions": plan_revision_count,
+                "total_reasoning_steps": len(decision_history)
+            },
+            "plan_evolution": {
+                "initial_tasks": len(execution_plan.tasks),
+                "final_version": current_plan_version,
+                "total_revisions": plan_revision_count
+            },
+            "clarification_required": False,
+            "overall_confidence": overall_confidence,
+            "error": None
         }
 
 
@@ -1831,23 +3921,35 @@ FINAL ANSWER:"""
                         repeat_penalty:float|None=None,
                         repeat_last_n:int|None=None,
                         callback=None,
-                        debug:bool=False ):
+                        debug:bool=False,
+                        override_all_prompts:bool=False ):
         """
         Generates a single code block based on a prompt.
         Uses the underlying LLM binding via `generate_text`.
         Handles potential continuation if the code block is incomplete.
+        
+        Args:
+            override_all_prompts: If True, uses only the provided prompt and system_prompt 
+                                without any internal prompt engineering or modifications.
         """
-        if not  system_prompt:
-            system_prompt = f"""Act as a code generation assistant that generates code from user prompt."""
+        
+        # Use original prompts without modification if override is enabled
+        if override_all_prompts:
+            final_system_prompt = system_prompt if system_prompt else ""
+            final_prompt = prompt
+        else:
+            # Original prompt engineering logic
+            if not system_prompt:
+                system_prompt = f"""Act as a code generation assistant that generates code from user prompt."""
 
-        if template and template !="{}":
-            if language in ["json","yaml","xml"]:
-                system_prompt += f"\nMake sure the generated context follows the following schema:\n```{language}\n{template}\n```\n"
-            else:
-                system_prompt += f"\nHere is a template of the answer:\n```{language}\n{template}\n```\n"
-            
-            if code_tag_format=="markdown":
-                system_prompt += f"""You must answer with the code placed inside the markdown code tag:
+            if template and template !="{}":
+                if language in ["json","yaml","xml"]:
+                    system_prompt += f"\nMake sure the generated context follows the following schema:\n```{language}\n{template}\n```\n"
+                else:
+                    system_prompt += f"\nHere is a template of the answer:\n```{language}\n{template}\n```\n"
+                
+                if code_tag_format=="markdown":
+                    system_prompt += f"""You must answer with the code placed inside the markdown code tag:
 ```{language}
 ```
 """
@@ -1856,14 +3958,17 @@ FINAL ANSWER:"""
 <code language="{language}">
 </code>
 """
-        system_prompt += f"""You must return a single code tag.
+            system_prompt += f"""You must return a single code tag.
 Do not split the code in multiple tags.
 {self.ai_full_header}"""
+            
+            final_system_prompt = system_prompt
+            final_prompt = prompt
 
         response = self.generate_text(
-            prompt,
+            final_prompt,
             images=images,
-            system_prompt=system_prompt,
+            system_prompt=final_system_prompt,
             n_predict=n_predict,
             temperature=temperature,
             top_k=top_k,
@@ -1874,8 +3979,8 @@ Do not split the code in multiple tags.
             )
 
         if isinstance(response, dict) and not response.get("status", True):
-             ASCIIColors.error(f"Code generation failed: {response.get('error')}")
-             return None
+            ASCIIColors.error(f"Code generation failed: {response.get('error')}")
+            return None
 
         codes = self.extract_code_blocks(response, format=code_tag_format)
         code_content = None
@@ -1885,46 +3990,425 @@ Do not split the code in multiple tags.
             code_content = last_code["content"]
 
             # Handle incomplete code block continuation (simple approach)
-            max_retries = 3 # Limit continuation attempts
-            retries = 0
-            while not last_code["is_complete"] and retries < max_retries:
-                retries += 1
-                ASCIIColors.info(f"Code block seems incomplete. Attempting continuation ({retries}/{max_retries})...")
-                continuation_prompt = f"{prompt}\n\nAssistant:\n{code_content}\n\n{self.user_full_header}The previous code block was incomplete. Continue the code exactly from where it left off. Do not repeat the previous part. Only provide the continuation inside a single {code_tag_format} code tag.\n{self.ai_full_header}"
-
-                continuation_response = self.generate_text(
-                    continuation_prompt,
-                    images=images, # Resend images if needed for context
-                    n_predict=n_predict, # Allow space for continuation
-                    temperature=temperature, # Use same parameters
+            # Skip continuation logic if override_all_prompts is True to respect user's intent
+            if not override_all_prompts:
+                max_retries = 3 # Limit continuation attempts
+                retries = 0
+                while not last_code["is_complete"] and retries < max_retries:
+                    retries += 1
+                    ASCIIColors.info(f"Code block seems incomplete. Attempting continuation ({retries}/{max_retries})...")
+                    continuation_prompt = f"{prompt}\n\nAssistant:\n{code_content}\n\n{self.user_full_header}The previous code block was incomplete. Continue the code exactly from where it left off. Do not repeat the previous part. Only provide the continuation inside a single {code_tag_format} code tag.\n{self.ai_full_header}"
+
+                    continuation_response = self.generate_text(
+                        continuation_prompt,
+                        images=images, # Resend images if needed for context
+                        n_predict=n_predict, # Allow space for continuation
+                        temperature=temperature, # Use same parameters
+                        top_k=top_k,
+                        top_p=top_p,
+                        repeat_penalty=repeat_penalty,
+                        repeat_last_n=repeat_last_n,
+                        streaming_callback=callback
+                    )
+
+                    if isinstance(continuation_response, dict) and not continuation_response.get("status", True):
+                        ASCIIColors.warning(f"Continuation attempt failed: {continuation_response.get('error')}")
+                        break # Stop trying if generation fails
+
+                    continuation_codes = self.extract_code_blocks(continuation_response, format=code_tag_format)
+
+                    if continuation_codes:
+                        new_code_part = continuation_codes[0]["content"]
+                        code_content += "\n" + new_code_part # Append continuation
+                        last_code["is_complete"] = continuation_codes[0]["is_complete"] # Update completeness
+                        if last_code["is_complete"]:
+                            ASCIIColors.info("Code block continuation successful.")
+                            break # Exit loop if complete
+                    else:
+                        ASCIIColors.warning("Continuation response contained no code block.")
+                        break # Stop if no code block found in continuation
+
+                if not last_code["is_complete"]:
+                    ASCIIColors.warning("Code block remained incomplete after multiple attempts.")
+
+        return code_content # Return the (potentially completed) code content or None
+
+
+    def update_code(
+        self,
+        original_code: str,
+        modification_prompt: str,
+        language: str = "python",
+        images=[],
+        system_prompt: str | None = None,
+        patch_format: str = "unified",  # "unified" or "simple"
+        n_predict: int | None = None,
+        temperature: float | None = None,
+        top_k: int | None = None,
+        top_p: float | None = None,
+        repeat_penalty: float | None = None,
+        repeat_last_n: int | None = None,
+        callback=None,
+        debug: bool = False,
+        max_retries: int = 3
+    ):
+        """
+        Updates existing code based on a modification prompt by generating and applying patches.
+        Designed to work reliably even with smaller LLMs.
+        
+        Args:
+            original_code: The original code to be modified
+            modification_prompt: Description of changes to apply to the code
+            language: Programming language of the code
+            patch_format: "unified" for diff-style patches or "simple" for line-based replacements
+            max_retries: Maximum number of attempts if patch generation/application fails
+            
+        Returns:
+            dict: Contains 'success' (bool), 'updated_code' (str or None), 'patch' (str or None), 
+                and 'error' (str or None)
+        """
+        
+        if not original_code or not original_code.strip():
+            return {
+                "success": False,
+                "updated_code": None,
+                "patch": None,
+                "error": "Original code is empty"
+            }
+        
+        # Choose patch format based on LLM capabilities
+        if patch_format == "simple":
+            # Simple format for smaller LLMs - just old/new line pairs
+            patch_system_prompt = f"""You are a code modification assistant.
+You will receive {language} code and a modification request.
+Generate a patch using this EXACT format:
+
+PATCH_START
+REPLACE_LINE: <line_number>
+OLD: <exact_old_line>
+NEW: <new_line>
+REPLACE_LINE: <another_line_number>
+OLD: <exact_old_line>
+NEW: <new_line>
+PATCH_END
+
+For adding lines:
+ADD_AFTER: <line_number>
+NEW: <line_to_add>
+
+For removing lines:
+REMOVE_LINE: <line_number>
+
+Rules:
+- Line numbers start at 1
+- Match OLD lines EXACTLY including whitespace
+- Only include lines that need changes
+- Keep changes minimal and focused"""
+
+        else:  # unified diff format
+            patch_system_prompt = f"""You are a code modification assistant.
+You will receive {language} code and a modification request.
+Generate a unified diff patch showing the changes.
+
+Format your response as:
+```diff
+@@ -start_line,count +start_line,count @@
+ context_line (unchanged)
+-removed_line
++added_line
+ context_line (unchanged)
+```
+
+Rules:
+- Use standard unified diff format
+- Include 1-2 lines of context around changes
+- Be precise with line numbers
+- Keep changes minimal"""
+
+        if system_prompt:
+            patch_system_prompt = system_prompt + "\n\n" + patch_system_prompt
+        
+        # Prepare the prompt with line numbers for reference
+        numbered_code = "\n".join(
+            f"{i+1:4d}: {line}" 
+            for i, line in enumerate(original_code.split("\n"))
+        )
+        
+        patch_prompt = f"""Original {language} code (with line numbers for reference):
+```{language}
+{numbered_code}
+```
+
+Modification request: {modification_prompt}
+
+Generate a patch to apply these changes. Follow the format specified in your instructions exactly."""
+
+        retry_count = 0
+        last_error = None
+        
+        while retry_count < max_retries:
+            try:
+                if debug:
+                    ASCIIColors.info(f"Attempting to generate patch (attempt {retry_count + 1}/{max_retries})")
+                
+                # Generate the patch
+                response = self.generate_text(
+                    patch_prompt,
+                    images=images,
+                    system_prompt=patch_system_prompt,
+                    n_predict=n_predict or 2000,  # Usually patches are not too long
+                    temperature=temperature or 0.3,  # Lower temperature for more deterministic patches
                     top_k=top_k,
                     top_p=top_p,
                     repeat_penalty=repeat_penalty,
                     repeat_last_n=repeat_last_n,
                     streaming_callback=callback
                 )
+                
+                if isinstance(response, dict) and not response.get("status", True):
+                    raise Exception(f"Patch generation failed: {response.get('error')}")
+                
+                # Extract and apply the patch
+                if patch_format == "simple":
+                    updated_code, patch_text = self._apply_simple_patch(original_code, response, debug)
+                else:
+                    updated_code, patch_text = self._apply_unified_patch(original_code, response, debug)
+                
+                if updated_code:
+                    if debug:
+                        ASCIIColors.success("Code successfully updated")
+                    return {
+                        "success": True,
+                        "updated_code": updated_code,
+                        "patch": patch_text,
+                        "error": None
+                    }
+                else:
+                    raise Exception("Failed to apply patch - no valid changes found")
+                    
+            except Exception as e:
+                last_error = str(e)
+                if debug:
+                    ASCIIColors.warning(f"Attempt {retry_count + 1} failed: {last_error}")
+                
+                retry_count += 1
+                
+                # Try simpler approach on retry
+                if retry_count < max_retries and patch_format == "unified":
+                    patch_format = "simple"
+                    if debug:
+                        ASCIIColors.info("Switching to simple patch format for next attempt")
+        
+        # All retries exhausted
+        return {
+            "success": False,
+            "updated_code": None,
+            "patch": None,
+            "error": f"Failed after {max_retries} attempts. Last error: {last_error}"
+        }
 
-                if isinstance(continuation_response, dict) and not continuation_response.get("status", True):
-                    ASCIIColors.warning(f"Continuation attempt failed: {continuation_response.get('error')}")
-                    break # Stop trying if generation fails
 
-                continuation_codes = self.extract_code_blocks(continuation_response, format=code_tag_format)
+    def _apply_simple_patch(self, original_code: str, patch_response: str, debug: bool = False):
+        """
+        Apply a simple line-based patch to the original code.
+        
+        Returns:
+            tuple: (updated_code or None, patch_text or None)
+        """
+        try:
+            lines = original_code.split("\n")
+            patch_lines = []
+            
+            # Extract patch content
+            if "PATCH_START" in patch_response and "PATCH_END" in patch_response:
+                start_idx = patch_response.index("PATCH_START")
+                end_idx = patch_response.index("PATCH_END")
+                patch_content = patch_response[start_idx + len("PATCH_START"):end_idx].strip()
+            else:
+                # Try to extract patch instructions even without markers
+                patch_content = patch_response
+            
+            # Track modifications to apply them in reverse order (to preserve line numbers)
+            modifications = []
+            
+            for line in patch_content.split("\n"):
+                line = line.strip()
+                if not line:
+                    continue
+                    
+                patch_lines.append(line)
+                
+                if line.startswith("REPLACE_LINE:"):
+                    try:
+                        line_num = int(line.split(":")[1].strip()) - 1  # Convert to 0-based
+                        # Look for OLD and NEW in next lines
+                        idx = patch_content.index(line)
+                        remaining = patch_content[idx:].split("\n")
+                        
+                        old_line = None
+                        new_line = None
+                        
+                        for next_line in remaining[1:]:
+                            if next_line.strip().startswith("OLD:"):
+                                old_line = next_line[next_line.index("OLD:") + 4:].strip()
+                            elif next_line.strip().startswith("NEW:"):
+                                new_line = next_line[next_line.index("NEW:") + 4:].strip()
+                                break
+                        
+                        if old_line is not None and new_line is not None:
+                            modifications.append(("replace", line_num, old_line, new_line))
+                            
+                    except (ValueError, IndexError) as e:
+                        if debug:
+                            ASCIIColors.warning(f"Failed to parse REPLACE_LINE: {e}")
+                            
+                elif line.startswith("ADD_AFTER:"):
+                    try:
+                        line_num = int(line.split(":")[1].strip()) - 1
+                        # Look for NEW in next line
+                        idx = patch_content.index(line)
+                        remaining = patch_content[idx:].split("\n")
+                        
+                        for next_line in remaining[1:]:
+                            if next_line.strip().startswith("NEW:"):
+                                new_line = next_line[next_line.index("NEW:") + 4:].strip()
+                                modifications.append(("add_after", line_num, None, new_line))
+                                break
+                                
+                    except (ValueError, IndexError) as e:
+                        if debug:
+                            ASCIIColors.warning(f"Failed to parse ADD_AFTER: {e}")
+                            
+                elif line.startswith("REMOVE_LINE:"):
+                    try:
+                        line_num = int(line.split(":")[1].strip()) - 1
+                        modifications.append(("remove", line_num, None, None))
+                        
+                    except (ValueError, IndexError) as e:
+                        if debug:
+                            ASCIIColors.warning(f"Failed to parse REMOVE_LINE: {e}")
+            
+            if not modifications:
+                return None, None
+            
+            # Sort modifications by line number in reverse order
+            modifications.sort(key=lambda x: x[1], reverse=True)
+            
+            # Apply modifications
+            for mod_type, line_num, old_line, new_line in modifications:
+                if line_num < 0 or line_num >= len(lines):
+                    if debug:
+                        ASCIIColors.warning(f"Line number {line_num + 1} out of range")
+                    continue
+                    
+                if mod_type == "replace":
+                    # Verify old line matches (with some flexibility for whitespace)
+                    if old_line and lines[line_num].strip() == old_line.strip():
+                        # Preserve original indentation
+                        indent = len(lines[line_num]) - len(lines[line_num].lstrip())
+                        lines[line_num] = " " * indent + new_line.lstrip()
+                    elif debug:
+                        ASCIIColors.warning(f"Line {line_num + 1} doesn't match expected content")
+                        
+                elif mod_type == "add_after":
+                    # Get indentation from the reference line
+                    indent = len(lines[line_num]) - len(lines[line_num].lstrip())
+                    lines.insert(line_num + 1, " " * indent + new_line.lstrip())
+                    
+                elif mod_type == "remove":
+                    del lines[line_num]
+            
+            updated_code = "\n".join(lines)
+            patch_text = "\n".join(patch_lines)
+            
+            return updated_code, patch_text
+            
+        except Exception as e:
+            if debug:
+                ASCIIColors.error(f"Error applying simple patch: {e}")
+            return None, None
+
 
-                if continuation_codes:
-                    new_code_part = continuation_codes[0]["content"]
-                    code_content += "\n" + new_code_part # Append continuation
-                    last_code["is_complete"] = continuation_codes[0]["is_complete"] # Update completeness
-                    if last_code["is_complete"]:
-                        ASCIIColors.info("Code block continuation successful.")
-                        break # Exit loop if complete
+    def _apply_unified_patch(self, original_code: str, patch_response: str, debug: bool = False):
+        """
+        Apply a unified diff patch to the original code.
+        
+        Returns:
+            tuple: (updated_code or None, patch_text or None)
+        """
+        try:
+            import re
+            
+            lines = original_code.split("\n")
+            
+            # Extract diff content
+            diff_pattern = r'```diff\n(.*?)\n```'
+            diff_match = re.search(diff_pattern, patch_response, re.DOTALL)
+            
+            if diff_match:
+                patch_text = diff_match.group(1)
+            else:
+                # Try to find diff content without code blocks
+                if "@@" in patch_response:
+                    patch_text = patch_response
                 else:
-                     ASCIIColors.warning("Continuation response contained no code block.")
-                     break # Stop if no code block found in continuation
-
-            if not last_code["is_complete"]:
-                ASCIIColors.warning("Code block remained incomplete after multiple attempts.")
+                    return None, None
+            
+            # Parse and apply hunks
+            hunk_pattern = r'@@ -(\d+),?(\d*) \+(\d+),?(\d*) @@'
+            hunks = re.finditer(hunk_pattern, patch_text)
+            
+            changes = []
+            for hunk in hunks:
+                old_start = int(hunk.group(1)) - 1  # Convert to 0-based
+                old_count = int(hunk.group(2)) if hunk.group(2) else 1
+                new_start = int(hunk.group(3)) - 1
+                new_count = int(hunk.group(4)) if hunk.group(4) else 1
+                
+                # Extract hunk content
+                hunk_start = hunk.end()
+                next_hunk = re.search(hunk_pattern, patch_text[hunk_start:])
+                hunk_end = hunk_start + next_hunk.start() if next_hunk else len(patch_text)
+                
+                hunk_lines = patch_text[hunk_start:hunk_end].strip().split("\n")
+                
+                # Process hunk lines
+                for line in hunk_lines:
+                    if not line:
+                        continue
+                        
+                    if line.startswith("-"):
+                        changes.append(("remove", old_start, line[1:].strip()))
+                    elif line.startswith("+"):
+                        changes.append(("add", old_start, line[1:].strip()))
+                    # Context lines (starting with space) are ignored
+            
+            # Apply changes (in reverse order to maintain line numbers)
+            changes.sort(key=lambda x: x[1], reverse=True)
+            
+            for change_type, line_num, content in changes:
+                if line_num < 0 or line_num > len(lines):
+                    continue
+                    
+                if change_type == "remove":
+                    if line_num < len(lines) and lines[line_num].strip() == content:
+                        del lines[line_num]
+                elif change_type == "add":
+                    # Preserve indentation from nearby lines
+                    indent = 0
+                    if line_num > 0:
+                        indent = len(lines[line_num - 1]) - len(lines[line_num - 1].lstrip())
+                    lines.insert(line_num, " " * indent + content)
+            
+            updated_code = "\n".join(lines)
+            
+            return updated_code, patch_text
+            
+        except Exception as e:
+            if debug:
+                ASCIIColors.error(f"Error applying unified patch: {e}")
+            return None, None
 
-        return code_content # Return the (potentially completed) code content or None
 
     def generate_structured_content(
         self,
@@ -1933,8 +4417,16 @@ Do not split the code in multiple tags.
         schema=None,
         system_prompt=None,
         max_retries=1,
+        use_override=False,
         **kwargs
     ):
+        """
+        Enhanced structured content generation with optional prompt override.
+        
+        Args:
+            use_override: If True, uses override_all_prompts=True in generate_code
+                        and relies entirely on provided system_prompt and prompt.
+        """
         import json
         images = [] if images is None else images
         schema = {} if schema is None else schema
@@ -1954,21 +4446,15 @@ Do not split the code in multiple tags.
         else:
             raise TypeError("schema must be a dict or a JSON string.")
 
-        # --- FIX STARTS HERE ---
         # Heuristic to detect if the schema is a properties-only dictionary
-        # and needs to be wrapped in a root object to be a valid schema.
-        # This handles cases where the user provides `{"field1": {...}, "field2": {...}}`
-        # instead of `{"type": "object", "properties": {"field1": ...}}`.
         if "type" not in schema_obj and "properties" not in schema_obj and all(isinstance(v, dict) for v in schema_obj.values()):
             if kwargs.get("debug"):
                 ASCIIColors.info("Schema appears to be a properties-only dictionary; wrapping it in a root object.")
             schema_obj = {
                 "type": "object",
                 "properties": schema_obj,
-                # Assume all top-level keys are required when wrapping
                 "required": list(schema_obj.keys())
             }
-        # --- FIX ENDS HERE ---
 
         def _instance_skeleton(s):
             if not isinstance(s, dict):
@@ -1978,7 +4464,6 @@ Do not split the code in multiple tags.
             if "enum" in s and isinstance(s["enum"], list) and s["enum"]:
                 return s["enum"][0]
             
-            # Handle default values
             if "default" in s:
                 return s["default"]
 
@@ -1992,22 +4477,19 @@ Do not split the code in multiple tags.
             if t == "boolean":
                 return False
             if t == "array":
-                # Generate one minimal item if schema is provided
                 items = s.get("items", {})
                 min_items = s.get("minItems", 0)
-                # Let's generate at least one item for the example if possible
                 num_items = max(min_items, 1) if items and not min_items == 0 else min_items
                 return [_instance_skeleton(items) for _ in range(num_items)]
             if t == "object":
                 props = s.get("properties", {})
-                # Use required fields, otherwise fall back to all properties for the skeleton
                 req = s.get("required", list(props.keys()))
                 out = {}
                 for k in req:
                     if k in props:
                         out[k] = _instance_skeleton(props[k])
                     else:
-                        out[k] = None # Should not happen if schema is well-formed
+                        out[k] = None
                 return out
             if "oneOf" in s and isinstance(s["oneOf"], list) and s["oneOf"]:
                 return _instance_skeleton(s["oneOf"][0])
@@ -2022,42 +4504,56 @@ Do not split the code in multiple tags.
                 return merged if merged else {}
             return {}
 
-        # Now derive strings from the (potentially corrected) schema_obj
         schema_str = json.dumps(schema_obj, indent=2, ensure_ascii=False)
         example_obj = _instance_skeleton(schema_obj)
         example_str = json.dumps(example_obj, indent=2, ensure_ascii=False)
 
-        base_system = (
-            "Your objective is to generate a JSON object that satisfies the user's request and conforms to the provided schema.\n"
-            "Rules:\n"
-            "1) The schema is reference ONLY. Do not include the schema in the output.\n"
-            "2) Output exactly ONE valid JSON object.\n"
-            "3) Wrap the JSON object inside a single ```json code block.\n"
-            "4) Do not output explanations or text outside the JSON.\n"
-            "5) Use 2 spaces for indentation. Do not use tabs.\n"
-            "6) Only include fields allowed by the schema and ensure all required fields are present.\n"
-            "7) For enums, choose a valid value from the list.\n\n"
-            "Schema (reference only):\n"
-            f"```json\n{schema_str}\n```\n\n"
-            "Correct example of output format (structure only, values are illustrative):\n"
-            f"```json\n{example_str}\n```"
-        )
-        full_system_prompt = f"{system_prompt}\n\n{base_system}" if system_prompt else base_system
+        if use_override:
+            # Use provided system_prompt and prompt as-is
+            final_system_prompt = system_prompt if system_prompt else ""
+            final_prompt = prompt
+            override_prompts = True
+        else:
+            # Use original prompt engineering
+            base_system = (
+                "Your objective is to generate a JSON object that satisfies the user's request and conforms to the provided schema.\n"
+                "Rules:\n"
+                "1) The schema is reference ONLY. Do not include the schema in the output.\n"
+                "2) Output exactly ONE valid JSON object.\n"
+                "3) Wrap the JSON object inside a single ```json code block.\n"
+                "4) Do not output explanations or text outside the JSON.\n"
+                "5) Use 2 spaces for indentation. Do not use tabs.\n"
+                "6) Only include fields allowed by the schema and ensure all required fields are present.\n"
+                "7) For enums, choose a valid value from the list.\n\n"
+                "Schema (reference only):\n"
+                f"```json\n{schema_str}\n```\n\n"
+                "Correct example of output format (structure only, values are illustrative):\n"
+                f"```json\n{example_str}\n```"
+            )
+            final_system_prompt = f"{system_prompt}\n\n{base_system}" if system_prompt else base_system
+            final_prompt = prompt
+            override_prompts = False
 
         if kwargs.get("debug"):
             ASCIIColors.info("Generating structured content...")
 
         last_error = None
         for attempt in range(max_retries + 1):
+            retry_system_prompt = final_system_prompt
+            if attempt > 0 and not use_override:
+                retry_system_prompt = f"{final_system_prompt}\n\nPrevious attempt failed validation: {last_error}\nReturn a corrected JSON instance that strictly satisfies the schema."
+            
             json_string = self.generate_code(
-                prompt=prompt,
+                prompt=final_prompt,
                 images=images,
-                system_prompt=full_system_prompt if attempt == 0 else f"{full_system_prompt}\n\nPrevious attempt failed validation: {last_error}\nReturn a corrected JSON instance that strictly satisfies the schema.",
-                template=example_str,
+                system_prompt=retry_system_prompt,
+                template=example_str if not use_override else None,
                 language="json",
                 code_tag_format="markdown",
+                override_all_prompts=override_prompts,
                 **kwargs
             )
+            
             if not json_string:
                 last_error = "LLM returned an empty response."
                 if kwargs.get("debug"): ASCIIColors.warning(last_error)
@@ -2083,19 +4579,213 @@ Do not split the code in multiple tags.
                         if kwargs.get("debug"): ASCIIColors.warning(last_error)
                         if attempt < max_retries:
                             continue
-                        # Return the invalid object after last retry if validation fails
                         return parsed_json
                 return parsed_json
             except Exception as e:
                 trace_exception(e)
                 ASCIIColors.error(f"Unexpected error during JSON processing: {e}")
                 last_error = f"An unexpected error occurred: {e}"
-                # Do not retry on unexpected errors, break the loop
                 break
         
         ASCIIColors.error(f"Failed to generate valid structured content after {max_retries + 1} attempts. Last error: {last_error}")
         return None
 
+    def generate_structured_content_pydantic(
+        self,
+        prompt,
+        pydantic_model,
+        images=None,
+        system_prompt=None,
+        max_retries=1,
+        use_override=False,
+        **kwargs
+    ):
+        """
+        Generate structured content using Pydantic models for validation.
+        
+        Args:
+            prompt: The user prompt
+            pydantic_model: A Pydantic BaseModel class or instance
+            images: Optional images for context
+            system_prompt: Optional system prompt
+            max_retries: Number of retry attempts
+            use_override: If True, uses override_all_prompts=True in generate_code
+            **kwargs: Additional arguments passed to generate_code
+            
+        Returns:
+            Validated Pydantic model instance or None if generation failed
+        """
+        import json
+        from typing import get_type_hints, get_origin, get_args
+        
+        try:
+            from pydantic import BaseModel, ValidationError
+            from pydantic.fields import FieldInfo
+        except ImportError:
+            ASCIIColors.error("Pydantic is required for this method. Please install it with: pip install pydantic")
+            return None
+        
+        images = [] if images is None else images
+        
+        # Handle both class and instance
+        if isinstance(pydantic_model, type) and issubclass(pydantic_model, BaseModel):
+            model_class = pydantic_model
+        elif isinstance(pydantic_model, BaseModel):
+            model_class = type(pydantic_model)
+        else:
+            raise TypeError("pydantic_model must be a Pydantic BaseModel class or instance")
+        
+        def _get_pydantic_schema_info(model_cls):
+            """Extract schema information from Pydantic model."""
+            schema = model_cls.model_json_schema()
+            
+            # Create example instance
+            try:
+                # Try to create with defaults first
+                example_instance = model_cls()
+                example_dict = example_instance.model_dump()
+            except:
+                # If that fails, create minimal example based on schema
+                example_dict = _create_example_from_schema(schema)
+            
+            return schema, example_dict
+        
+        def _create_example_from_schema(schema):
+            """Create example data from JSON schema."""
+            if schema.get("type") == "object":
+                properties = schema.get("properties", {})
+                required = schema.get("required", [])
+                example = {}
+                
+                for field_name, field_schema in properties.items():
+                    if field_name in required or "default" in field_schema:
+                        example[field_name] = _get_example_value(field_schema)
+                
+                return example
+            return {}
+        
+        def _get_example_value(field_schema):
+            """Get example value for a field based on its schema."""
+            if "default" in field_schema:
+                return field_schema["default"]
+            if "const" in field_schema:
+                return field_schema["const"]
+            if "enum" in field_schema and field_schema["enum"]:
+                return field_schema["enum"][0]
+            
+            field_type = field_schema.get("type")
+            if field_type == "string":
+                return "example"
+            elif field_type == "integer":
+                return 0
+            elif field_type == "number":
+                return 0.0
+            elif field_type == "boolean":
+                return False
+            elif field_type == "array":
+                items_schema = field_schema.get("items", {})
+                return [_get_example_value(items_schema)]
+            elif field_type == "object":
+                return _create_example_from_schema(field_schema)
+            else:
+                return None
+        
+        # Get schema and example from Pydantic model
+        try:
+            schema, example_dict = _get_pydantic_schema_info(model_class)
+            schema_str = json.dumps(schema, indent=2, ensure_ascii=False)
+            example_str = json.dumps(example_dict, indent=2, ensure_ascii=False)
+        except Exception as e:
+            ASCIIColors.error(f"Failed to extract schema from Pydantic model: {e}")
+            return None
+        
+        if use_override:
+            # Use provided system_prompt and prompt as-is
+            final_system_prompt = system_prompt if system_prompt else ""
+            final_prompt = prompt
+            override_prompts = True
+        else:
+            # Enhanced prompt engineering for Pydantic
+            base_system = (
+                "Your objective is to generate a JSON object that satisfies the user's request and conforms to the provided Pydantic model schema.\n"
+                "Rules:\n"
+                "1) The schema is reference ONLY. Do not include the schema in the output.\n"
+                "2) Output exactly ONE valid JSON object that can be parsed by the Pydantic model.\n"
+                "3) Wrap the JSON object inside a single ```json code block.\n"
+                "4) Do not output explanations or text outside the JSON.\n"
+                "5) Use 2 spaces for indentation. Do not use tabs.\n"
+                "6) Respect all field types, constraints, and validation rules.\n"
+                "7) Include all required fields and use appropriate default values where specified.\n"
+                "8) For enums, choose a valid value from the allowed options.\n\n"
+                f"Pydantic Model Schema (reference only):\n"
+                f"```json\n{schema_str}\n```\n\n"
+                "Correct example of output format (structure only, values are illustrative):\n"
+                f"```json\n{example_str}\n```"
+            )
+            final_system_prompt = f"{system_prompt}\n\n{base_system}" if system_prompt else base_system
+            final_prompt = prompt
+            override_prompts = False
+        
+        if kwargs.get("debug"):
+            ASCIIColors.info("Generating Pydantic-structured content...")
+            ASCIIColors.info(f"Using model: {model_class.__name__}")
+        
+        last_error = None
+        for attempt in range(max_retries + 1):
+            retry_system_prompt = final_system_prompt
+            if attempt > 0:
+                retry_system_prompt = f"{final_system_prompt}\n\nPrevious attempt failed Pydantic validation: {last_error}\nReturn a corrected JSON instance that strictly satisfies the Pydantic model."
+            
+            json_string = self.generate_code(
+                prompt=prompt,
+                images=images,
+                system_prompt=retry_system_prompt,
+                language="json",
+                code_tag_format="markdown",
+                override_all_prompts=True,  # Always use override for structured content
+                **kwargs
+            )
+            
+            if not json_string:
+                last_error = "LLM returned an empty response."
+                if kwargs.get("debug"): ASCIIColors.warning(last_error)
+                continue
+            
+            if kwargs.get("debug"):
+                ASCIIColors.info("Parsing generated JSON string...")
+                print(f"--- Raw JSON String ---\n{json_string}\n-----------------------")
+            
+            try:
+                # Parse JSON
+                parsed_json = robust_json_parser(json_string)
+                if parsed_json is None:
+                    last_error = "Failed to robustly parse the generated string into JSON."
+                    if kwargs.get("debug"): ASCIIColors.warning(last_error)
+                    continue
+                
+                # Validate with Pydantic
+                try:
+                    validated_instance = model_class.model_validate(parsed_json)
+                    if kwargs.get("debug"):
+                        ASCIIColors.success("Pydantic validation successful!")
+                    return validated_instance
+                except ValidationError as ve:
+                    last_error = f"Pydantic Validation Error: {ve}"
+                    if kwargs.get("debug"): ASCIIColors.warning(last_error)
+                    if attempt < max_retries:
+                        continue
+                    # Return the raw parsed JSON if validation fails on last attempt
+                    ASCIIColors.warning("Returning unvalidated JSON after final attempt.")
+                    return parsed_json
+                    
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.error(f"Unexpected error during JSON processing: {e}")
+                last_error = f"An unexpected error occurred: {e}"
+                break
+    
+        ASCIIColors.error(f"Failed to generate valid Pydantic-structured content after {max_retries + 1} attempts. Last error: {last_error}")
+        return None
 
     def extract_code_blocks(self, text: str, format: str = "markdown") -> List[dict]:
         """