lollms-client 1.6.1__py3-none-any.whl → 1.6.4__py3-none-any.whl

lollms_client/lollms_core.py

@@ -143,16 +143,21 @@ class LollmsClient():
                 ASCIIColors.warning(f"Failed to create LLM binding: {llm_binding_name}. Available: {available}")
 
         if tts_binding_name:
-            self.tts = self.tts_binding_manager.create_binding(
-                binding_name=tts_binding_name,
-                **{
-                    k: v
-                    for k, v in (tts_binding_config or {}).items()
-                    if k != "binding_name"
-                }
-            )
-            if self.tts is None:
-                ASCIIColors.warning(f"Failed to create TTS binding: {tts_binding_name}. Available: {self.tts_binding_manager.get_available_bindings()}")
+            try:
+                params = {
+                        k: v
+                        for k, v in (tts_binding_config or {}).items()
+                        if k != "binding_name"
+                    }
+                self.tts = self.tts_binding_manager.create_binding(
+                    binding_name=tts_binding_name,
+                    **params
+                )
+                if self.tts is None:
+                    ASCIIColors.warning(f"Failed to create TTS binding: {tts_binding_name}. Available: {self.tts_binding_manager.get_available_bindings()}")
+            except Exception as e:
+                trace_exception(e)
+                ASCIIColors.warning(f"Exception occurred while creating TTS binding: {str(e)}")
 
         if tti_binding_name:
             if tti_binding_config:
@@ -268,8 +273,8 @@ class LollmsClient():
             raise ValueError(f"Failed to update LLM binding: {binding_name}. Available: {available}")
 
     def get_ctx_size(self, model_name:str|None=None):
-        if self.llm:
-            ctx_size = self.llm.get_ctx_size(model_name)
+        if self.llm and self.llm.model_name:
+            ctx_size = self.llm.get_ctx_size(model_name or self.llm.model_name)
             return ctx_size if ctx_size else self.llm.default_ctx_size
         else:
             return None
@@ -674,10 +679,10 @@ class LollmsClient():
         raise RuntimeError("LLM binding not initialized.")
 
 
-    def listModels(self):
+    def list_models(self):
         """Lists models available to the current LLM binding."""
         if self.llm:
-            return self.llm.listModels()
+            return self.llm.list_models()
         raise RuntimeError("LLM binding not initialized.")
 
     # --- Convenience Methods for Lollms LLM Binding Features ---
@@ -2405,1444 +2410,6 @@ GUIDELINES for your response:
 
 Your response should feel natural and conversational while being informative and valuable.
 
-FINAL RESPONSE:"""
-        
-        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
-        
-        final_answer_text = self.generate_text(
-            prompt=final_answer_prompt,
-            system_prompt=system_prompt,
-            stream=streaming_callback is not None,
-            streaming_callback=streaming_callback,
-            temperature=final_answer_temperature,
-            **llm_generation_kwargs
-        )
-        
-        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
-            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
-            return {
-                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
-                "error": final_answer_text["error"],
-                "final_scratchpad": current_scratchpad,
-                "tool_calls": tool_calls_this_turn,
-                "sources": sources_this_turn,
-                "decision_history": decision_history
-            }
-        
-        final_answer = self.remove_thinking_blocks(final_answer_text)
-        
-        # Calculate overall performance metrics
-        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
-        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
-        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
-        
-        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
-            "final_answer_length": len(final_answer),
-            "total_tools_used": len(tool_calls_this_turn),
-            "success_rate": success_rate,
-            "overall_confidence": overall_confidence
-        })
-
-        return {
-            "final_answer": final_answer,
-            "final_scratchpad": current_scratchpad,
-            "tool_calls": tool_calls_this_turn,
-            "sources": sources_this_turn,
-            "decision_history": decision_history,
-            "performance_stats": {
-                "total_steps": len(tool_calls_this_turn),
-                "successful_steps": successful_calls,
-                "success_rate": success_rate,
-                "average_confidence": overall_confidence,
-                "plan_revisions": plan_revision_count,
-                "total_reasoning_steps": len(decision_history)
-            },
-            "plan_evolution": {
-                "initial_tasks": len(execution_plan.tasks),
-                "final_version": current_plan_version,
-                "total_revisions": plan_revision_count
-            },
-            "clarification_required": False,
-            "overall_confidence": overall_confidence,
-            "error": None
-        }
-
-
-    def _execute_complex_reasoning_loop(
-        self, prompt, context, system_prompt, reasoning_system_prompt, images,
-        max_reasoning_steps, decision_temperature, final_answer_temperature,
-        streaming_callback, debug, enable_self_reflection, all_visible_tools,
-        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
-    ) -> Dict[str, Any]:
-        
-        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
-        
-        def _get_friendly_action_description(tool_name, requires_code, requires_image):
-            descriptions = {
-                "local_tools::final_answer": "📋 Preparing final answer",
-                "local_tools::request_clarification": "❓ Requesting clarification",
-                "local_tools::generate_image": "🎨 Creating image",
-                "local_tools::revise_plan": "📝 Revising execution plan"
-            }
-            if tool_name in descriptions:
-                return descriptions[tool_name]
-            if "research::" in tool_name: 
-                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
-            if requires_code: 
-                return "💻 Processing code"
-            if requires_image: 
-                return "🖼️ Analyzing images"
-            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
-
-        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
-            """Enhanced scratchpad compression that preserves key decisions and recent context"""
-            if len(scratchpad) <= target_size:
-                return scratchpad
-            
-            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
-            
-            # Extract key components
-            lines = scratchpad.split('\n')
-            plan_section = []
-            decisions = []
-            recent_observations = []
-            
-            current_section = None
-            for i, line in enumerate(lines):
-                if "### Execution Plan" in line or "### Updated Plan" in line:
-                    current_section = "plan"
-                elif "### Step" in line and ("Thought" in line or "Decision" in line):
-                    current_section = "decision"
-                elif "### Step" in line and "Observation" in line:
-                    current_section = "observation"
-                elif line.startswith("###"):
-                    current_section = None
-                
-                if current_section == "plan" and line.strip():
-                    plan_section.append(line)
-                elif current_section == "decision" and line.strip():
-                    decisions.append((i, line))
-                elif current_section == "observation" and line.strip():
-                    recent_observations.append((i, line))
-            
-            # Keep most recent items and important decisions
-            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
-            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
-            
-            compressed_parts = [
-                f"### Original Request\n{original_request}",
-                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
-                f"### Recent Key Decisions"
-            ]
-            
-            for _, decision in recent_decisions:
-                compressed_parts.append(decision)
-            
-            compressed_parts.append("### Recent Observations")
-            for _, obs in recent_obs:
-                compressed_parts.append(obs)
-            
-            compressed = '\n'.join(compressed_parts)
-            if len(compressed) > target_size:
-                # Final trim if still too long
-                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
-            
-            return compressed
-
-        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
-        asset_store: Dict[str, Dict] = {}
-        decision_history = []  # Track all decisions made
-        
-        # Enhanced planning phase
-        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
-        execution_plan = planner.decompose_task(original_user_prompt, context or "")
-        current_plan_version = 1
-        
-        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
-            "plan_version": current_plan_version,
-            "total_tasks": len(execution_plan.tasks),
-            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
-        })
-        
-        for i, task in enumerate(execution_plan.tasks):
-            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
-            
-        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
-        
-        # Enhanced initial state
-        initial_state_parts = [
-            f"### Original User Request\n{original_user_prompt}",
-            f"### Context\n{context or 'No additional context provided'}",
-            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
-            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
-        ]
-        
-        for i, task in enumerate(execution_plan.tasks): 
-            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
-        
-        if images:
-            initial_state_parts.append(f"### Provided Assets")
-            for img_b64 in images:
-                img_uuid = str(uuid.uuid4())
-                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
-                initial_state_parts.append(f"- Image asset: {img_uuid}")
-        
-        current_scratchpad = "\n".join(initial_state_parts)
-        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
-
-        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
-        completed_tasks, current_task_index = set(), 0
-        plan_revision_count = 0
-        
-        # Main reasoning loop with enhanced decision tracking
-        for i in range(max_reasoning_steps):
-            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
-            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
-            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
-            
-            try:
-                # Enhanced scratchpad management
-                if len(current_scratchpad) > max_scratchpad_size:
-                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
-                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
-                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
-                
-                # Enhanced reasoning prompt with better decision tracking
-                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
-
-=== AVAILABLE ACTIONS ===
-{formatted_tools_list}
-
-=== YOUR COMPLETE ANALYSIS HISTORY ===
-{current_scratchpad}
-=== END ANALYSIS HISTORY ===
-
-=== DECISION GUIDELINES ===
-1. **Review your progress**: Look at what you've already discovered and accomplished
-2. **Consider your current task**: Focus on the next logical step in your plan
-3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
-4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
-5. **Stay focused**: Each action should clearly advance toward the final goal
-
-=== YOUR NEXT DECISION ===
-Choose the single most appropriate action to take right now. Consider:
-- What specific step are you currently working on?
-- What information do you still need?
-- What would be most helpful for the user?
-
-Provide your decision as JSON:
-{{
-    "reasoning": "Explain your current thinking and why this action makes sense now",
-    "action": {{
-        "tool_name": "exact_tool_name",
-        "requires_code_input": false,
-        "requires_image_input": false,
-        "confidence": 0.8
-    }},
-    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
-}}"""
-
-                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
-                decision_data = self.generate_structured_content(
-                    prompt=reasoning_prompt, 
-                    schema={
-                        "reasoning": "string", 
-                        "action": "object",
-                        "plan_status": "string"
-                    }, 
-                    system_prompt=reasoning_system_prompt, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                
-                if not (decision_data and isinstance(decision_data.get("action"), dict)):
-                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
-                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
-                    continue
-
-                action = decision_data.get("action", {})
-                reasoning = decision_data.get("reasoning", "No reasoning provided")
-                plan_status = decision_data.get("plan_status", "on_track")
-                tool_name = action.get("tool_name")
-                requires_code = action.get("requires_code_input", False)
-                requires_image = action.get("requires_image_input", False)
-                confidence = action.get("confidence", 0.5)
-                
-                # Track the decision
-                decision_history.append({
-                    "step": i+1,
-                    "tool_name": tool_name,
-                    "reasoning": reasoning,
-                    "confidence": confidence,
-                    "plan_status": plan_status
-                })
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
-                
-                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
-                    "tool_name": tool_name,
-                    "confidence": confidence,
-                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
-                })
-                
-                # Handle plan revision
-                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
-                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
-                    tool_name = "local_tools::revise_plan"  # Force plan revision
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-
-                # Handle plan revision
-                if tool_name == "local_tools::revise_plan":
-                    plan_revision_count += 1
-                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
-                    
-                    try:
-                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-REASON FOR REVISION: {reasoning}
-
-Create a new plan that reflects your current understanding. Consider:
-1. What have you already accomplished?
-2. What new information have you discovered?
-3. What steps are still needed?
-4. How can you be more efficient?
-
-Provide your revision as JSON:
-{{
-    "revision_reason": "Clear explanation of why the plan needed to change",
-    "new_plan": [
-        {{"step": 1, "description": "First revised step", "status": "pending"}},
-        {{"step": 2, "description": "Second revised step", "status": "pending"}}
-    ],
-    "confidence": 0.8
-}}"""
-
-                        revision_data = self.generate_structured_content(
-                            prompt=revision_prompt,
-                            schema={
-                                "revision_reason": "string",
-                                "new_plan": "array", 
-                                "confidence": "number"
-                            },
-                            temperature=0.3,
-                            **llm_generation_kwargs
-                        )
-                        
-                        if revision_data and revision_data.get("new_plan"):
-                            # Update the plan
-                            current_plan_version += 1
-                            new_tasks = []
-                            for task_data in revision_data["new_plan"]:
-                                task = TaskDecomposition()  # Assuming this class exists
-                                task.description = task_data.get("description", "Undefined step")
-                                task.status = TaskStatus.PENDING  # Reset all to pending
-                                new_tasks.append(task)
-                            
-                            execution_plan.tasks = new_tasks
-                            current_task_index = 0  # Reset to beginning
-                            
-                            # Update scratchpad with new plan
-                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
-                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
-                            current_scratchpad += f"**New Tasks**:\n"
-                            for i, task in enumerate(execution_plan.tasks):
-                                current_scratchpad += f"  {i+1}. {task.description}\n"
-                            
-                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
-                                "plan_version": current_plan_version,
-                                "new_task_count": len(execution_plan.tasks),
-                                "revision_reason": revision_data.get("revision_reason", "")
-                            })
-                            
-                            # Continue with the new plan
-                            continue
-                        else:
-                            raise ValueError("Failed to generate valid plan revision")
-                            
-                    except Exception as e:
-                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
-                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
-
-                # Prepare parameters for tool execution
-                param_assets = {}
-                if requires_code:
-                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
-                    code_prompt = f"""Generate the specific code needed for the current step.
-
-CURRENT CONTEXT:
-{current_scratchpad}
-
-CURRENT TASK: {tool_name}
-USER REQUEST: "{original_user_prompt}"
-
-Generate clean, functional code that addresses the specific requirements. Focus on:
-1. Solving the immediate problem
-2. Being clear and readable
-3. Including necessary imports and dependencies
-4. Adding helpful comments where appropriate
-
-CODE:"""
-
-                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
-                    code_uuid = f"code_asset_{uuid.uuid4()}"
-                    asset_store[code_uuid] = {"type": "code", "content": code_content}
-                    param_assets['code_asset_id'] = code_uuid
-                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    
-                if requires_image:
-                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
-                    if image_assets:
-                        param_assets['image_asset_id'] = image_assets[0]
-                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    else:
-                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
-
-                # Enhanced parameter generation
-                param_prompt = f"""Generate the optimal parameters for this tool execution.
-
-TOOL: {tool_name}
-CURRENT CONTEXT: {current_scratchpad}
-CURRENT REASONING: {reasoning}
-AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
-
-Based on your analysis and the current step you're working on, provide the most appropriate parameters.
-Be specific and purposeful in your parameter choices.
-
-Output format: {{"tool_params": {{...}}}}"""
-
-                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
-                param_data = self.generate_structured_content(
-                    prompt=param_prompt, 
-                    schema={"tool_params": "object"}, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                tool_params = param_data.get("tool_params", {}) if param_data else {}
-                
-                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
-                
-                # Hydrate parameters with assets
-                def _hydrate(data: Any, store: Dict) -> Any:
-                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
-                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
-                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
-                    return data
-                    
-                hydrated_params = _hydrate(tool_params, asset_store)
-                
-                # Execute the tool with detailed logging
-                start_time = time.time()
-                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
-                
-                try:
-                    if tool_name in rag_registry:
-                        query = hydrated_params.get("query", "")
-                        if not query:
-                            # Fall back to using reasoning as query
-                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
-                            
-                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        top_k = rag_tool_specs[tool_name]["default_top_k"]
-                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
-                        
-                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
-                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
-                        
-                        docs = []
-                        for d in raw_iter or []:
-                            doc_data = {
-                                "text": d.get("text", str(d)), 
-                                "score": d.get("score", 0) * 100, 
-                                "metadata": d.get("metadata", {})
-                            }
-                            docs.append(doc_data)
-                        
-                        kept = [x for x in docs if x['score'] >= min_sim]
-                        tool_result = {
-                            "status": "success", 
-                            "results": kept, 
-                            "total_found": len(docs),
-                            "kept_after_filtering": len(kept),
-                            "query_used": query
-                        }
-                        
-                        sources_this_turn.extend([{
-                            "source": tool_name, 
-                            "metadata": x["metadata"], 
-                            "score": x["score"]
-                        } for x in kept])
-                        
-                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
-                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
-                            "tool_name": tool_name,
-                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
-                        })
-                        
-                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
-                        
-                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
-                            "result_status": tool_result.get("status", "unknown"),
-                            "has_error": "error" in tool_result
-                        })
-                        
-                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
-                        image_prompt = hydrated_params.get("prompt", "")
-                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        # This would call your text-to-image functionality
-                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
-                        if image_result:
-                            image_uuid = f"generated_image_{uuid.uuid4()}"
-                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
-                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
-                        else:
-                            tool_result = {"status": "failure", "error": "Image generation failed"}
-                            
-                    else:
-                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
-                        
-                except Exception as e:
-                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
-                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
-                    tool_result = {"status": "failure", "error": error_msg}
-
-                response_time = time.time() - start_time
-                success = tool_result.get("status") == "success"
-                
-                # Record performance
-                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
-                
-                # Update task status
-                if success and current_task_index < len(execution_plan.tasks):
-                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
-                    completed_tasks.add(current_task_index)
-                    current_task_index += 1
-                
-                # Enhanced observation logging
-                observation_text = json.dumps(tool_result, indent=2)
-                if len(observation_text) > 1000:
-                    # Truncate very long results for scratchpad
-                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
-                    observation_text = json.dumps(truncated_result, indent=2)
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
-                current_scratchpad += f"**Tool Used**: {tool_name}\n"
-                current_scratchpad += f"**Success**: {success}\n"
-                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
-                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
-                
-                # Track tool call
-                tool_calls_this_turn.append({
-                    "name": tool_name,
-                    "params": tool_params,
-                    "result": tool_result,
-                    "response_time": response_time,
-                    "confidence": confidence,
-                    "reasoning": reasoning
-                })
-                
-                if success:
-                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "response_time": response_time,
-                        "confidence": confidence
-                    })
-                else:
-                    error_detail = tool_result.get("error", "No error detail provided.")
-                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "error": error_detail,
-                        "confidence": confidence
-                    })
-                    
-                    # Add failure handling to scratchpad
-                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
-                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
-                
-                # Log current progress
-                completed_count = len(completed_tasks)
-                total_tasks = len(execution_plan.tasks)
-                if total_tasks > 0:
-                    progress = (completed_count / total_tasks) * 100
-                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
-                
-                # Check if all tasks are completed
-                if completed_count >= total_tasks:
-                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                    
-            except Exception as ex:
-                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
-                trace_exception(ex)
-                
-                # Add error to scratchpad for context
-                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
-                
-                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
-
-        # Enhanced self-reflection
-        if enable_self_reflection and len(tool_calls_this_turn) > 0:
-            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
-            try:
-                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-TOOLS USED: {len(tool_calls_this_turn)}
-PLAN REVISIONS: {plan_revision_count}
-
-COMPLETE ANALYSIS:
-{current_scratchpad}
-
-Evaluate your performance on multiple dimensions:
-
-1. **Goal Achievement**: Did you fully address the user's request?
-2. **Process Efficiency**: Was your approach optimal given the available tools?
-3. **Information Quality**: Is the information you gathered accurate and relevant?
-4. **Decision Making**: Were your tool choices and parameters appropriate?
-5. **Adaptability**: How well did you handle unexpected results or plan changes?
-
-Provide your assessment as JSON:
-{{
-    "goal_achieved": true,
-    "effectiveness_score": 0.85,
-    "process_efficiency": 0.8,
-    "information_quality": 0.9,
-    "decision_making": 0.85,
-    "adaptability": 0.7,
-    "overall_confidence": 0.82,
-    "strengths": ["Clear reasoning", "Good tool selection"],
-    "areas_for_improvement": ["Could have been more efficient"],
-    "summary": "Successfully completed the user's request with high quality results",
-    "key_insights": ["Discovered that X was more important than initially thought"]
-}}"""
-
-                reflection_data = self.generate_structured_content(
-                    prompt=reflection_prompt,
-                    schema={
-                        "goal_achieved": "boolean",
-                        "effectiveness_score": "number",
-                        "process_efficiency": "number", 
-                        "information_quality": "number",
-                        "decision_making": "number",
-                        "adaptability": "number",
-                        "overall_confidence": "number",
-                        "strengths": "array",
-                        "areas_for_improvement": "array",
-                        "summary": "string",
-                        "key_insights": "array"
-                    },
-                    temperature=0.3,
-                    **llm_generation_kwargs
-                )
-                
-                if reflection_data:
-                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
-                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
-                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
-                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
-                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
-                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
-                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
-                    
-                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
-                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
-                        "goal_achieved": reflection_data.get('goal_achieved', False),
-                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
-                    })
-                else:
-                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-                    
-            except Exception as e:
-                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-
-        # Enhanced final synthesis
-        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
-        
-        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CONTEXT: {context or "No additional context"}
-
-COMPLETE ANALYSIS AND WORK:
-{current_scratchpad}
-
-GUIDELINES for your response:
-1. **Be Complete**: Address all aspects of the user's request
-2. **Be Clear**: Organize your response logically and use clear language  
-3. **Be Helpful**: Provide actionable information and insights
-4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
-5. **Be Concise**: While being thorough, avoid unnecessary verbosity
-6. **Cite Sources**: If you used research tools, reference the information appropriately
-
-Your response should feel natural and conversational while being informative and valuable.
-
-FINAL RESPONSE:"""
-        
-        log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)
-        
-        final_answer_text = self.generate_text(
-            prompt=final_answer_prompt,
-            system_prompt=system_prompt,
-            stream=streaming_callback is not None,
-            streaming_callback=streaming_callback,
-            temperature=final_answer_temperature,
-            **llm_generation_kwargs
-        )
-        
-        if isinstance(final_answer_text, dict) and "error" in final_answer_text: 
-            log_event_fn(f"Final synthesis failed: {final_answer_text['error']}", MSG_TYPE.MSG_TYPE_ERROR, event_id=synthesis_id)
-            return {
-                "final_answer": "I encountered an issue while preparing my final response. Please let me know if you'd like me to try again.",
-                "error": final_answer_text["error"],
-                "final_scratchpad": current_scratchpad,
-                "tool_calls": tool_calls_this_turn,
-                "sources": sources_this_turn,
-                "decision_history": decision_history
-            }
-        
-        final_answer = self.remove_thinking_blocks(final_answer_text)
-        
-        # Calculate overall performance metrics
-        overall_confidence = sum(call.get('confidence', 0.5) for call in tool_calls_this_turn) / max(len(tool_calls_this_turn), 1)
-        successful_calls = sum(1 for call in tool_calls_this_turn if call.get('result', {}).get('status') == 'success')
-        success_rate = successful_calls / max(len(tool_calls_this_turn), 1)
-        
-        log_event_fn("✅ Comprehensive response ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=synthesis_id, meta={
-            "final_answer_length": len(final_answer),
-            "total_tools_used": len(tool_calls_this_turn),
-            "success_rate": success_rate,
-            "overall_confidence": overall_confidence
-        })
-
-        return {
-            "final_answer": final_answer,
-            "final_scratchpad": current_scratchpad,
-            "tool_calls": tool_calls_this_turn,
-            "sources": sources_this_turn,
-            "decision_history": decision_history,
-            "performance_stats": {
-                "total_steps": len(tool_calls_this_turn),
-                "successful_steps": successful_calls,
-                "success_rate": success_rate,
-                "average_confidence": overall_confidence,
-                "plan_revisions": plan_revision_count,
-                "total_reasoning_steps": len(decision_history)
-            },
-            "plan_evolution": {
-                "initial_tasks": len(execution_plan.tasks),
-                "final_version": current_plan_version,
-                "total_revisions": plan_revision_count
-            },
-            "clarification_required": False,
-            "overall_confidence": overall_confidence,
-            "error": None
-        }
-
-
-    def _execute_complex_reasoning_loop(
-        self, prompt, context, system_prompt, reasoning_system_prompt, images,
-        max_reasoning_steps, decision_temperature, final_answer_temperature,
-        streaming_callback, debug, enable_self_reflection, all_visible_tools,
-        rag_registry, rag_tool_specs, log_event_fn, log_prompt_fn, max_scratchpad_size, **llm_generation_kwargs
-    ) -> Dict[str, Any]:
-        
-        planner, memory_manager, performance_tracker = TaskPlanner(self), MemoryManager(), ToolPerformanceTracker()
-        
-        def _get_friendly_action_description(tool_name, requires_code, requires_image):
-            descriptions = {
-                "local_tools::final_answer": "📋 Preparing final answer",
-                "local_tools::request_clarification": "❓ Requesting clarification",
-                "local_tools::generate_image": "🎨 Creating image",
-                "local_tools::revise_plan": "📝 Revising execution plan"
-            }
-            if tool_name in descriptions:
-                return descriptions[tool_name]
-            if "research::" in tool_name: 
-                return f"🔍 Searching {tool_name.split('::')[-1]} knowledge base"
-            if requires_code: 
-                return "💻 Processing code"
-            if requires_image: 
-                return "🖼️ Analyzing images"
-            return f"🔧 Using {tool_name.replace('_', ' ').replace('::', ' - ').title()}"
-
-        def _compress_scratchpad_intelligently(scratchpad: str, original_request: str, target_size: int) -> str:
-            """Enhanced scratchpad compression that preserves key decisions and recent context"""
-            if len(scratchpad) <= target_size:
-                return scratchpad
-            
-            log_event_fn("📝 Compressing scratchpad to maintain focus...", MSG_TYPE.MSG_TYPE_INFO)
-            
-            # Extract key components
-            lines = scratchpad.split('\n')
-            plan_section = []
-            decisions = []
-            recent_observations = []
-            
-            current_section = None
-            for i, line in enumerate(lines):
-                if "### Execution Plan" in line or "### Updated Plan" in line:
-                    current_section = "plan"
-                elif "### Step" in line and ("Thought" in line or "Decision" in line):
-                    current_section = "decision"
-                elif "### Step" in line and "Observation" in line:
-                    current_section = "observation"
-                elif line.startswith("###"):
-                    current_section = None
-                
-                if current_section == "plan" and line.strip():
-                    plan_section.append(line)
-                elif current_section == "decision" and line.strip():
-                    decisions.append((i, line))
-                elif current_section == "observation" and line.strip():
-                    recent_observations.append((i, line))
-            
-            # Keep most recent items and important decisions
-            recent_decisions = decisions[-3:] if len(decisions) > 3 else decisions
-            recent_obs = recent_observations[-5:] if len(recent_observations) > 5 else recent_observations
-            
-            compressed_parts = [
-                f"### Original Request\n{original_request}",
-                f"### Current Plan\n" + '\n'.join(plan_section[-10:]),
-                f"### Recent Key Decisions"
-            ]
-            
-            for _, decision in recent_decisions:
-                compressed_parts.append(decision)
-            
-            compressed_parts.append("### Recent Observations")
-            for _, obs in recent_obs:
-                compressed_parts.append(obs)
-            
-            compressed = '\n'.join(compressed_parts)
-            if len(compressed) > target_size:
-                # Final trim if still too long
-                compressed = compressed[:target_size-100] + "\n...[content compressed for focus]"
-            
-            return compressed
-
-        original_user_prompt, tool_calls_this_turn, sources_this_turn = prompt, [], []
-        asset_store: Dict[str, Dict] = {}
-        decision_history = []  # Track all decisions made
-        
-        # Enhanced planning phase
-        planning_step_id = log_event_fn("📋 Creating adaptive execution plan...", MSG_TYPE.MSG_TYPE_STEP_START)
-        execution_plan = planner.decompose_task(original_user_prompt, context or "")
-        current_plan_version = 1
-        
-        log_event_fn(f"Initial plan created with {len(execution_plan.tasks)} tasks", MSG_TYPE.MSG_TYPE_INFO, meta={
-            "plan_version": current_plan_version,
-            "total_tasks": len(execution_plan.tasks),
-            "estimated_complexity": "medium" if len(execution_plan.tasks) <= 5 else "high"
-        })
-        
-        for i, task in enumerate(execution_plan.tasks):
-            log_event_fn(f"Task {i+1}: {task.description}", MSG_TYPE.MSG_TYPE_INFO)
-            
-        log_event_fn("✅ Adaptive plan ready", MSG_TYPE.MSG_TYPE_STEP_END, event_id=planning_step_id)
-        
-        # Enhanced initial state
-        initial_state_parts = [
-            f"### Original User Request\n{original_user_prompt}",
-            f"### Context\n{context or 'No additional context provided'}",
-            f"### Execution Plan (Version {current_plan_version})\n- Total tasks: {len(execution_plan.tasks)}",
-            f"- Estimated complexity: {'High' if len(execution_plan.tasks) > 5 else 'Medium'}"
-        ]
-        
-        for i, task in enumerate(execution_plan.tasks): 
-            initial_state_parts.append(f"  {i+1}. {task.description} [Status: {task.status.value}]")
-        
-        if images:
-            initial_state_parts.append(f"### Provided Assets")
-            for img_b64 in images:
-                img_uuid = str(uuid.uuid4())
-                asset_store[img_uuid] = {"type": "image", "content": img_b64, "source": "user"}
-                initial_state_parts.append(f"- Image asset: {img_uuid}")
-        
-        current_scratchpad = "\n".join(initial_state_parts)
-        log_event_fn("Initial analysis complete", MSG_TYPE.MSG_TYPE_SCRATCHPAD, meta={"scratchpad_size": len(current_scratchpad)})
-
-        formatted_tools_list = "\n".join([f"**{t['name']}**: {t['description']}" for t in all_visible_tools])
-        completed_tasks, current_task_index = set(), 0
-        plan_revision_count = 0
-        
-        # Main reasoning loop with enhanced decision tracking
-        for i in range(max_reasoning_steps):
-            current_task_desc = execution_plan.tasks[current_task_index].description if current_task_index < len(execution_plan.tasks) else "Finalizing analysis"
-            step_desc = f"🤔 Step {i+1}: {current_task_desc}"
-            reasoning_step_id = log_event_fn(step_desc, MSG_TYPE.MSG_TYPE_STEP_START)
-            
-            try:
-                # Enhanced scratchpad management
-                if len(current_scratchpad) > max_scratchpad_size:
-                    log_event_fn(f"Scratchpad size ({len(current_scratchpad)}) exceeds limit, compressing...", MSG_TYPE.MSG_TYPE_INFO)
-                    current_scratchpad = _compress_scratchpad_intelligently(current_scratchpad, original_user_prompt, max_scratchpad_size // 2)
-                    log_event_fn(f"Scratchpad compressed to {len(current_scratchpad)} characters", MSG_TYPE.MSG_TYPE_INFO)
-                
-                # Enhanced reasoning prompt with better decision tracking
-                reasoning_prompt = f"""You are working on: "{original_user_prompt}"
-
-=== AVAILABLE ACTIONS ===
-{formatted_tools_list}
-
-=== YOUR COMPLETE ANALYSIS HISTORY ===
-{current_scratchpad}
-=== END ANALYSIS HISTORY ===
-
-=== DECISION GUIDELINES ===
-1. **Review your progress**: Look at what you've already discovered and accomplished
-2. **Consider your current task**: Focus on the next logical step in your plan
-3. **Remember your decisions**: If you previously decided to use a tool, follow through unless you have a good reason to change
-4. **Be adaptive**: If you discover new information that changes the situation, consider revising your plan
-5. **Stay focused**: Each action should clearly advance toward the final goal
-
-=== YOUR NEXT DECISION ===
-Choose the single most appropriate action to take right now. Consider:
-- What specific step are you currently working on?
-- What information do you still need?
-- What would be most helpful for the user?
-
-Provide your decision as JSON:
-{{
-    "reasoning": "Explain your current thinking and why this action makes sense now",
-    "action": {{
-        "tool_name": "exact_tool_name",
-        "requires_code_input": false,
-        "requires_image_input": false,
-        "confidence": 0.8
-    }},
-    "plan_status": "on_track" // or "needs_revision" if you want to change the plan
-}}"""
-
-                log_prompt_fn(f"Reasoning Prompt Step {i+1}", reasoning_prompt)
-                decision_data = self.generate_structured_content(
-                    prompt=reasoning_prompt, 
-                    schema={
-                        "reasoning": "string", 
-                        "action": "object",
-                        "plan_status": "string"
-                    }, 
-                    system_prompt=reasoning_system_prompt, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                
-                if not (decision_data and isinstance(decision_data.get("action"), dict)):
-                    log_event_fn("⚠️ Invalid decision format from AI", MSG_TYPE.MSG_TYPE_WARNING, event_id=reasoning_step_id)
-                    current_scratchpad += f"\n\n### Step {i+1}: Decision Error\n- Error: AI produced invalid decision JSON\n- Continuing with fallback approach"
-                    continue
-
-                action = decision_data.get("action", {})
-                reasoning = decision_data.get("reasoning", "No reasoning provided")
-                plan_status = decision_data.get("plan_status", "on_track")
-                tool_name = action.get("tool_name")
-                requires_code = action.get("requires_code_input", False)
-                requires_image = action.get("requires_image_input", False)
-                confidence = action.get("confidence", 0.5)
-                
-                # Track the decision
-                decision_history.append({
-                    "step": i+1,
-                    "tool_name": tool_name,
-                    "reasoning": reasoning,
-                    "confidence": confidence,
-                    "plan_status": plan_status
-                })
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Decision & Reasoning\n**Reasoning**: {reasoning}\n**Chosen Action**: {tool_name}\n**Confidence**: {confidence}\n**Plan Status**: {plan_status}"
-                
-                log_event_fn(_get_friendly_action_description(tool_name, requires_code, requires_image), MSG_TYPE.MSG_TYPE_STEP, meta={
-                    "tool_name": tool_name,
-                    "confidence": confidence,
-                    "reasoning": reasoning[:100] + "..." if len(reasoning) > 100 else reasoning
-                })
-                
-                # Handle plan revision
-                if plan_status == "needs_revision" and tool_name != "local_tools::revise_plan":
-                    log_event_fn("🔄 AI indicates plan needs revision", MSG_TYPE.MSG_TYPE_INFO)
-                    tool_name = "local_tools::revise_plan"  # Force plan revision
-                
-                # Handle final answer
-                if tool_name == "local_tools::final_answer": 
-                    log_event_fn("🎯 Ready to provide final answer", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                
-                # Handle clarification request
-                if tool_name == "local_tools::request_clarification":
-                    clarification_prompt = f"""Based on your analysis, what specific information do you need from the user?
-
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-Generate a clear, specific question that will help you proceed effectively:"""
-                    
-                    question = self.generate_text(clarification_prompt, temperature=0.3)
-                    question = self.remove_thinking_blocks(question)
-                    
-                    log_event_fn("❓ Clarification needed from user", MSG_TYPE.MSG_TYPE_INFO)
-                    return {
-                        "final_answer": question, 
-                        "clarification_required": True, 
-                        "final_scratchpad": current_scratchpad,
-                        "tool_calls": tool_calls_this_turn, 
-                        "sources": sources_this_turn, 
-                        "error": None,
-                        "decision_history": decision_history
-                    }
-
-                # Handle plan revision
-                if tool_name == "local_tools::revise_plan":
-                    plan_revision_count += 1
-                    revision_id = log_event_fn(f"📝 Revising execution plan (revision #{plan_revision_count})", MSG_TYPE.MSG_TYPE_STEP_START)
-                    
-                    try:
-                        revision_prompt = f"""Based on your current analysis and discoveries, create an updated execution plan.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CURRENT ANALYSIS:
-{current_scratchpad}
-
-REASON FOR REVISION: {reasoning}
-
-Create a new plan that reflects your current understanding. Consider:
-1. What have you already accomplished?
-2. What new information have you discovered?
-3. What steps are still needed?
-4. How can you be more efficient?
-
-Provide your revision as JSON:
-{{
-    "revision_reason": "Clear explanation of why the plan needed to change",
-    "new_plan": [
-        {{"step": 1, "description": "First revised step", "status": "pending"}},
-        {{"step": 2, "description": "Second revised step", "status": "pending"}}
-    ],
-    "confidence": 0.8
-}}"""
-
-                        revision_data = self.generate_structured_content(
-                            prompt=revision_prompt,
-                            schema={
-                                "revision_reason": "string",
-                                "new_plan": "array", 
-                                "confidence": "number"
-                            },
-                            temperature=0.3,
-                            **llm_generation_kwargs
-                        )
-                        
-                        if revision_data and revision_data.get("new_plan"):
-                            # Update the plan
-                            current_plan_version += 1
-                            new_tasks = []
-                            for task_data in revision_data["new_plan"]:
-                                task = TaskDecomposition()  # Assuming this class exists
-                                task.description = task_data.get("description", "Undefined step")
-                                task.status = TaskStatus.PENDING  # Reset all to pending
-                                new_tasks.append(task)
-                            
-                            execution_plan.tasks = new_tasks
-                            current_task_index = 0  # Reset to beginning
-                            
-                            # Update scratchpad with new plan
-                            current_scratchpad += f"\n\n### Updated Plan (Version {current_plan_version})\n"
-                            current_scratchpad += f"**Revision Reason**: {revision_data.get('revision_reason', 'Plan needed updating')}\n"
-                            current_scratchpad += f"**New Tasks**:\n"
-                            for i, task in enumerate(execution_plan.tasks):
-                                current_scratchpad += f"  {i+1}. {task.description}\n"
-                            
-                            log_event_fn(f"✅ Plan revised with {len(execution_plan.tasks)} updated tasks", MSG_TYPE.MSG_TYPE_STEP_END, event_id=revision_id, meta={
-                                "plan_version": current_plan_version,
-                                "new_task_count": len(execution_plan.tasks),
-                                "revision_reason": revision_data.get("revision_reason", "")
-                            })
-                            
-                            # Continue with the new plan
-                            continue
-                        else:
-                            raise ValueError("Failed to generate valid plan revision")
-                            
-                    except Exception as e:
-                        log_event_fn(f"Plan revision failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=revision_id)
-                        current_scratchpad += f"\n**Plan Revision Failed**: {str(e)}\nContinuing with original plan."
-
-                # Prepare parameters for tool execution
-                param_assets = {}
-                if requires_code:
-                    log_event_fn("💻 Generating code for task", MSG_TYPE.MSG_TYPE_INFO)
-                    code_prompt = f"""Generate the specific code needed for the current step.
-
-CURRENT CONTEXT:
-{current_scratchpad}
-
-CURRENT TASK: {tool_name}
-USER REQUEST: "{original_user_prompt}"
-
-Generate clean, functional code that addresses the specific requirements. Focus on:
-1. Solving the immediate problem
-2. Being clear and readable
-3. Including necessary imports and dependencies
-4. Adding helpful comments where appropriate
-
-CODE:"""
-
-                    code_content = self.generate_code(prompt=code_prompt, **llm_generation_kwargs)
-                    code_uuid = f"code_asset_{uuid.uuid4()}"
-                    asset_store[code_uuid] = {"type": "code", "content": code_content}
-                    param_assets['code_asset_id'] = code_uuid
-                    log_event_fn(f"Code asset created: {code_uuid[:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    
-                if requires_image:
-                    image_assets = [asset_id for asset_id, asset in asset_store.items() if asset['type'] == 'image' and asset.get('source') == 'user']
-                    if image_assets:
-                        param_assets['image_asset_id'] = image_assets[0]
-                        log_event_fn(f"Using image asset: {image_assets[0][:8]}...", MSG_TYPE.MSG_TYPE_INFO)
-                    else:
-                        log_event_fn("⚠️ Image required but none available", MSG_TYPE.MSG_TYPE_WARNING)
-
-                # Enhanced parameter generation
-                param_prompt = f"""Generate the optimal parameters for this tool execution.
-
-TOOL: {tool_name}
-CURRENT CONTEXT: {current_scratchpad}
-CURRENT REASONING: {reasoning}
-AVAILABLE ASSETS: {json.dumps(param_assets) if param_assets else "None"}
-
-Based on your analysis and the current step you're working on, provide the most appropriate parameters.
-Be specific and purposeful in your parameter choices.
-
-Output format: {{"tool_params": {{...}}}}"""
-
-                log_prompt_fn(f"Parameter Generation Step {i+1}", param_prompt)
-                param_data = self.generate_structured_content(
-                    prompt=param_prompt, 
-                    schema={"tool_params": "object"}, 
-                    temperature=decision_temperature, 
-                    **llm_generation_kwargs
-                )
-                tool_params = param_data.get("tool_params", {}) if param_data else {}
-                
-                current_scratchpad += f"\n**Parameters Generated**: {json.dumps(tool_params, indent=2)}"
-                
-                # Hydrate parameters with assets
-                def _hydrate(data: Any, store: Dict) -> Any:
-                    if isinstance(data, dict): return {k: _hydrate(v, store) for k, v in data.items()}
-                    if isinstance(data, list): return [_hydrate(item, store) for item in data]
-                    if isinstance(data, str) and "asset_" in data and data in store: return store[data].get("content", data)
-                    return data
-                    
-                hydrated_params = _hydrate(tool_params, asset_store)
-                
-                # Execute the tool with detailed logging
-                start_time = time.time()
-                tool_result = {"status": "failure", "error": f"Tool '{tool_name}' failed to execute."}
-                
-                try:
-                    if tool_name in rag_registry:
-                        query = hydrated_params.get("query", "")
-                        if not query:
-                            # Fall back to using reasoning as query
-                            query = reasoning[:200] + "..." if len(reasoning) > 200 else reasoning
-                            
-                        log_event_fn(f"🔍 Searching knowledge base with query: '{query[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        top_k = rag_tool_specs[tool_name]["default_top_k"]
-                        min_sim = rag_tool_specs[tool_name]["default_min_sim"]
-                        
-                        raw_results = rag_registry[tool_name](query=query, rag_top_k=top_k)
-                        raw_iter = raw_results["results"] if isinstance(raw_results, dict) and "results" in raw_results else raw_results
-                        
-                        docs = []
-                        for d in raw_iter or []:
-                            doc_data = {
-                                "text": d.get("text", str(d)), 
-                                "score": d.get("score", 0) * 100, 
-                                "metadata": d.get("metadata", {})
-                            }
-                            docs.append(doc_data)
-                        
-                        kept = [x for x in docs if x['score'] >= min_sim]
-                        tool_result = {
-                            "status": "success", 
-                            "results": kept, 
-                            "total_found": len(docs),
-                            "kept_after_filtering": len(kept),
-                            "query_used": query
-                        }
-                        
-                        sources_this_turn.extend([{
-                            "source": tool_name, 
-                            "metadata": x["metadata"], 
-                            "score": x["score"]
-                        } for x in kept])
-                        
-                        log_event_fn(f"📚 Retrieved {len(kept)} relevant documents (from {len(docs)} total)", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                    elif hasattr(self, "mcp") and "local_tools" not in tool_name:
-                        log_event_fn(f"🔧 Executing MCP tool: {tool_name}", MSG_TYPE.MSG_TYPE_TOOL_CALL, meta={
-                            "tool_name": tool_name,
-                            "params": {k: str(v)[:100] for k, v in hydrated_params.items()}  # Truncate for logging
-                        })
-                        
-                        tool_result = self.mcp.execute_tool(tool_name, hydrated_params, lollms_client_instance=self)
-                        
-                        log_event_fn(f"Tool execution completed", MSG_TYPE.MSG_TYPE_TOOL_OUTPUT, meta={
-                            "result_status": tool_result.get("status", "unknown"),
-                            "has_error": "error" in tool_result
-                        })
-                        
-                    elif tool_name == "local_tools::generate_image" and hasattr(self, "tti"):
-                        image_prompt = hydrated_params.get("prompt", "")
-                        log_event_fn(f"🎨 Generating image with prompt: '{image_prompt[:50]}...'", MSG_TYPE.MSG_TYPE_INFO)
-                        
-                        # This would call your text-to-image functionality
-                        image_result = self.tti.generate_image(image_prompt)  # Assuming this method exists
-                        if image_result:
-                            image_uuid = f"generated_image_{uuid.uuid4()}"
-                            asset_store[image_uuid] = {"type": "image", "content": image_result, "source": "generated"}
-                            tool_result = {"status": "success", "image_id": image_uuid, "prompt_used": image_prompt}
-                        else:
-                            tool_result = {"status": "failure", "error": "Image generation failed"}
-                            
-                    else:
-                        tool_result = {"status": "failure", "error": f"Tool '{tool_name}' is not available or supported in this context."}
-                        
-                except Exception as e:
-                    error_msg = f"Exception during '{tool_name}' execution: {str(e)}"
-                    log_event_fn(error_msg, MSG_TYPE.MSG_TYPE_EXCEPTION)
-                    tool_result = {"status": "failure", "error": error_msg}
-
-                response_time = time.time() - start_time
-                success = tool_result.get("status") == "success"
-                
-                # Record performance
-                performance_tracker.record_tool_usage(tool_name, success, confidence, response_time, tool_result.get("error"))
-                
-                # Update task status
-                if success and current_task_index < len(execution_plan.tasks):
-                    execution_plan.tasks[current_task_index].status = TaskStatus.COMPLETED
-                    completed_tasks.add(current_task_index)
-                    current_task_index += 1
-                
-                # Enhanced observation logging
-                observation_text = json.dumps(tool_result, indent=2)
-                if len(observation_text) > 1000:
-                    # Truncate very long results for scratchpad
-                    truncated_result = {k: (str(v)[:200] + "..." if len(str(v)) > 200 else v) for k, v in tool_result.items()}
-                    observation_text = json.dumps(truncated_result, indent=2)
-                
-                current_scratchpad += f"\n\n### Step {i+1}: Execution & Observation\n"
-                current_scratchpad += f"**Tool Used**: {tool_name}\n"
-                current_scratchpad += f"**Success**: {success}\n"
-                current_scratchpad += f"**Response Time**: {response_time:.2f}s\n"
-                current_scratchpad += f"**Result**:\n```json\n{observation_text}\n```"
-                
-                # Track tool call
-                tool_calls_this_turn.append({
-                    "name": tool_name,
-                    "params": tool_params,
-                    "result": tool_result,
-                    "response_time": response_time,
-                    "confidence": confidence,
-                    "reasoning": reasoning
-                })
-                
-                if success:
-                    log_event_fn(f"✅ Step {i+1} completed successfully", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "response_time": response_time,
-                        "confidence": confidence
-                    })
-                else:
-                    error_detail = tool_result.get("error", "No error detail provided.")
-                    log_event_fn(f"⚠️ Step {i+1} completed with issues: {error_detail}", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id, meta={
-                        "tool_name": tool_name,
-                        "error": error_detail,
-                        "confidence": confidence
-                    })
-                    
-                    # Add failure handling to scratchpad
-                    current_scratchpad += f"\n**Failure Analysis**: {error_detail}"
-                    current_scratchpad += f"\n**Next Steps**: Consider alternative approaches or tools"
-                
-                # Log current progress
-                completed_count = len(completed_tasks)
-                total_tasks = len(execution_plan.tasks)
-                if total_tasks > 0:
-                    progress = (completed_count / total_tasks) * 100
-                    log_event_fn(f"Progress: {completed_count}/{total_tasks} tasks completed ({progress:.1f}%)", MSG_TYPE.MSG_TYPE_STEP_PROGRESS, meta={"progress": progress})
-                
-                # Check if all tasks are completed
-                if completed_count >= total_tasks:
-                    log_event_fn("🎯 All planned tasks completed", MSG_TYPE.MSG_TYPE_INFO)
-                    break
-                    
-            except Exception as ex:
-                log_event_fn(f"💥 Unexpected error in reasoning step {i+1}: {str(ex)}", MSG_TYPE.MSG_TYPE_ERROR, event_id=reasoning_step_id)
-                trace_exception(ex)
-                
-                # Add error to scratchpad for context
-                current_scratchpad += f"\n\n### Step {i+1}: Unexpected Error\n**Error**: {str(ex)}\n**Recovery**: Continuing with adjusted approach"
-                
-                log_event_fn("🔄 Recovering and continuing with next step", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reasoning_step_id)
-
-        # Enhanced self-reflection
-        if enable_self_reflection and len(tool_calls_this_turn) > 0:
-            reflection_id = log_event_fn("🤔 Conducting comprehensive self-assessment...", MSG_TYPE.MSG_TYPE_STEP_START)
-            try:
-                reflection_prompt = f"""Conduct a thorough review of your work and assess the quality of your response to the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-TOOLS USED: {len(tool_calls_this_turn)}
-PLAN REVISIONS: {plan_revision_count}
-
-COMPLETE ANALYSIS:
-{current_scratchpad}
-
-Evaluate your performance on multiple dimensions:
-
-1. **Goal Achievement**: Did you fully address the user's request?
-2. **Process Efficiency**: Was your approach optimal given the available tools?
-3. **Information Quality**: Is the information you gathered accurate and relevant?
-4. **Decision Making**: Were your tool choices and parameters appropriate?
-5. **Adaptability**: How well did you handle unexpected results or plan changes?
-
-Provide your assessment as JSON:
-{{
-    "goal_achieved": true,
-    "effectiveness_score": 0.85,
-    "process_efficiency": 0.8,
-    "information_quality": 0.9,
-    "decision_making": 0.85,
-    "adaptability": 0.7,
-    "overall_confidence": 0.82,
-    "strengths": ["Clear reasoning", "Good tool selection"],
-    "areas_for_improvement": ["Could have been more efficient"],
-    "summary": "Successfully completed the user's request with high quality results",
-    "key_insights": ["Discovered that X was more important than initially thought"]
-}}"""
-
-                reflection_data = self.generate_structured_content(
-                    prompt=reflection_prompt,
-                    schema={
-                        "goal_achieved": "boolean",
-                        "effectiveness_score": "number",
-                        "process_efficiency": "number", 
-                        "information_quality": "number",
-                        "decision_making": "number",
-                        "adaptability": "number",
-                        "overall_confidence": "number",
-                        "strengths": "array",
-                        "areas_for_improvement": "array",
-                        "summary": "string",
-                        "key_insights": "array"
-                    },
-                    temperature=0.3,
-                    **llm_generation_kwargs
-                )
-                
-                if reflection_data:
-                    current_scratchpad += f"\n\n### Comprehensive Self-Assessment\n"
-                    current_scratchpad += f"**Goal Achieved**: {reflection_data.get('goal_achieved', False)}\n"
-                    current_scratchpad += f"**Overall Confidence**: {reflection_data.get('overall_confidence', 0.5):.2f}\n"
-                    current_scratchpad += f"**Effectiveness Score**: {reflection_data.get('effectiveness_score', 0.5):.2f}\n"
-                    current_scratchpad += f"**Key Strengths**: {', '.join(reflection_data.get('strengths', []))}\n"
-                    current_scratchpad += f"**Improvement Areas**: {', '.join(reflection_data.get('areas_for_improvement', []))}\n"
-                    current_scratchpad += f"**Summary**: {reflection_data.get('summary', '')}\n"
-                    
-                    log_event_fn(f"✅ Self-assessment completed", MSG_TYPE.MSG_TYPE_STEP_END, event_id=reflection_id, meta={
-                        "overall_confidence": reflection_data.get('overall_confidence', 0.5),
-                        "goal_achieved": reflection_data.get('goal_achieved', False),
-                        "effectiveness_score": reflection_data.get('effectiveness_score', 0.5)
-                    })
-                else:
-                    log_event_fn("Self-assessment data generation failed", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-                    
-            except Exception as e:
-                log_event_fn(f"Self-assessment failed: {e}", MSG_TYPE.MSG_TYPE_WARNING, event_id=reflection_id)
-
-        # Enhanced final synthesis
-        synthesis_id = log_event_fn("📝 Synthesizing comprehensive final response...", MSG_TYPE.MSG_TYPE_STEP_START)
-        
-        final_answer_prompt = f"""Create a comprehensive, well-structured final response that fully addresses the user's request.
-
-ORIGINAL REQUEST: "{original_user_prompt}"
-CONTEXT: {context or "No additional context"}
-
-COMPLETE ANALYSIS AND WORK:
-{current_scratchpad}
-
-GUIDELINES for your response:
-1. **Be Complete**: Address all aspects of the user's request
-2. **Be Clear**: Organize your response logically and use clear language  
-3. **Be Helpful**: Provide actionable information and insights
-4. **Be Honest**: If there were limitations or uncertainties, mention them appropriately
-5. **Be Concise**: While being thorough, avoid unnecessary verbosity
-6. **Cite Sources**: If you used research tools, reference the information appropriately
-
-Your response should feel natural and conversational while being informative and valuable.
-
 FINAL RESPONSE:"""
         
         log_prompt_fn("Final Synthesis Prompt", final_answer_prompt)