fastworkflow 2.15.5__py3-none-any.whl → 2.17.13__py3-none-any.whl

fastworkflow/chat_session.py

@@ -8,6 +8,8 @@ import contextlib
 import uuid
 from pathlib import Path
 import os
+import time
+from datetime import datetime
 
 import dspy
 
@@ -16,7 +18,6 @@ from fastworkflow.utils.logging import logger
 from fastworkflow.utils import dspy_utils
 from fastworkflow.model_pipeline_training import CommandRouter
 from fastworkflow.utils.startup_progress import StartupProgress
-from fastworkflow.command_metadata_api import CommandMetadataAPI
 
 
 class SessionStatus(Enum):
@@ -35,7 +36,7 @@ class ChatWorker(Thread):
         """Process messages for the root workflow"""
         try:
             self.chat_session._status = SessionStatus.RUNNING
-            workflow = ChatSession.get_active_workflow()
+            workflow = self.chat_session.get_active_workflow()
             logger.debug(f"Started root workflow {workflow.id}")
             
             # Run the workflow loop
@@ -44,39 +45,34 @@ class ChatWorker(Thread):
         finally:
             self.chat_session._status = SessionStatus.STOPPED
             # Ensure workflow is popped if thread terminates unexpectedly
-            if ChatSession.get_active_workflow() is not None:
-                ChatSession.pop_active_workflow()
+            if self.chat_session.get_active_workflow() is not None:
+                self.chat_session.pop_active_workflow()
 
 class ChatSession:
-    _workflow_stack_lock = Lock()
-    _workflow_stack: ClassVar[deque[fastworkflow.Workflow]] = deque()  # Stack of workflow objects
-    
-    @classmethod
-    def get_active_workflow(cls) -> Optional[fastworkflow.Workflow]:
+    def get_active_workflow(self) -> Optional[fastworkflow.Workflow]:
         """Get the currently active workflow (top of stack)"""
-        with cls._workflow_stack_lock:
-            return cls._workflow_stack[-1] if cls._workflow_stack else None
+        with self._workflow_stack_lock:
+            return self._workflow_stack[-1] if self._workflow_stack else None
     
-    @classmethod
-    def push_active_workflow(cls, workflow: fastworkflow.Workflow) -> None:
-        with cls._workflow_stack_lock:
-            cls._workflow_stack.append(workflow)
-            logger.debug(f"Workflow stack: {[w.id for w in cls._workflow_stack]}")
+    def push_active_workflow(self, workflow: fastworkflow.Workflow) -> None:
+        """Push a workflow onto this session's stack"""
+        with self._workflow_stack_lock:
+            self._workflow_stack.append(workflow)
+            logger.debug(f"Workflow stack: {[w.id for w in self._workflow_stack]}")
     
-    @classmethod
-    def pop_active_workflow(cls) -> Optional[fastworkflow.Workflow]:
-        with cls._workflow_stack_lock:
-            if not cls._workflow_stack:
+    def pop_active_workflow(self) -> Optional[fastworkflow.Workflow]:
+        """Pop a workflow from this session's stack"""
+        with self._workflow_stack_lock:
+            if not self._workflow_stack:
                 return None
-            workflow = cls._workflow_stack.pop()
-            logger.debug(f"Workflow stack after pop: {[w.id for w in cls._workflow_stack]}")
+            workflow = self._workflow_stack.pop()
+            logger.debug(f"Workflow stack after pop: {[w.id for w in self._workflow_stack]}")
             return workflow
 
-    @classmethod
-    def clear_workflow_stack(cls) -> None:
-        """Clear the entire workflow stack"""
-        with cls._workflow_stack_lock:
-            cls._workflow_stack.clear()
+    def clear_workflow_stack(self) -> None:
+        """Clear the entire workflow stack for this session"""
+        with self._workflow_stack_lock:
+            self._workflow_stack.clear()
             logger.debug("Workflow stack cleared")
 
     def stop_workflow(self) -> None:
@@ -95,7 +91,7 @@ class ChatSession:
                 logger.warning("Chat worker thread did not terminate within timeout")
         
         # Clear the workflow stack
-        ChatSession.clear_workflow_stack()
+        self.clear_workflow_stack()
         
         # Reset status to stopped
         self._status = SessionStatus.STOPPED
@@ -116,6 +112,10 @@ class ChatSession:
         A chat session can run multiple workflows that share the same message queues.
         Use start_workflow() to start a specific workflow within this session.
         """
+        # Create instance-level workflow stack (supports nested workflows within this session)
+        self._workflow_stack: deque[fastworkflow.Workflow] = deque()
+        self._workflow_stack_lock = Lock()
+        
         # Create queues for user messages and command outputs
         self._user_message_queue = Queue()
         self._command_output_queue = Queue()
@@ -123,7 +123,7 @@ class ChatSession:
         self._status = SessionStatus.STOPPED
         self._chat_worker = None
 
-        self._conversation_history = dspy.History(messages=[])
+        self._conversation_history: dspy.History = dspy.History(messages=[])
         
         # Import here to avoid circular imports
         from fastworkflow.command_executor import CommandExecutor
@@ -134,7 +134,8 @@ class ChatSession:
         
         # Initialize agent-related attributes
         self._run_as_agent = run_as_agent
-        self._workflow_tool_agent = None            
+        self._workflow_tool_agent = None
+        self._intent_clarification_agent = None            
         
         # Create the command metadata extraction workflow with a unique ID
         self._cme_workflow = fastworkflow.Workflow.create(
@@ -145,6 +146,10 @@ class ChatSession:
             }
         )
 
+        # this intializes the conversation traces file name also
+        # which is necessary when starting a brand new chat session
+        self.clear_conversation_history()
+
     def start_workflow(self,
         workflow_folderpath: str, 
         workflow_id_str: Optional[str] = None, 
@@ -193,7 +198,7 @@ class ChatSession:
 
         # Check if we need to stop the current workflow
         # Stop if this is a new root workflow (no parent, keep_alive=True)
-        current_workflow = ChatSession.get_active_workflow()
+        current_workflow = self.get_active_workflow()
         if (current_workflow and 
             parent_workflow_id is None and 
             self._keep_alive):
@@ -240,13 +245,13 @@ class ChatSession:
         self._status = SessionStatus.STARTING
         
         # Push this workflow as active
-        ChatSession.push_active_workflow(workflow)
+        self.push_active_workflow(workflow)
         
         # Initialize workflow tool agent if in agent mode
         # This must happen after pushing the workflow to the stack
         # so that get_active_workflow() returns the correct workflow
         if self._run_as_agent:
-            self._initialize_workflow_tool_agent()
+            self._initialize_agent_functionality()
         
         command_output = None
         if self._keep_alive:
@@ -260,19 +265,32 @@ class ChatSession:
 
         return command_output
 
-    def _initialize_workflow_tool_agent(self):
+    def _initialize_agent_functionality(self):
         """
         Initialize the workflow tool agent for agent mode.
         This agent handles individual tool selection and execution.
         """
-        if not self._workflow_tool_agent:
-            # Initialize the workflow tool agent
-            from fastworkflow.mcp_server import FastWorkflowMCPServer
-            from fastworkflow.workflow_agent import initialize_workflow_tool_agent
-            
-            mcp_server = FastWorkflowMCPServer(self)
-            self._workflow_tool_agent = initialize_workflow_tool_agent(mcp_server)
-    
+        self._cme_workflow.context["run_as_agent"] = True
+        self._current_workflow.context["run_as_agent"] = True
+
+        # Initialize the workflow tool agent
+        from fastworkflow.workflow_agent import initialize_workflow_tool_agent
+        self._workflow_tool_agent = initialize_workflow_tool_agent(self)
+
+        # Initialize the intent clarification agent
+        from fastworkflow.intent_clarification_agent import initialize_intent_clarification_agent
+        self._intent_clarification_agent = initialize_intent_clarification_agent(self)
+
+    @property
+    def workflow_tool_agent(self):
+        """Get the workflow tool agent for agent mode."""
+        return self._workflow_tool_agent
+
+    @property
+    def intent_clarification_agent(self):
+        """Get the intent clarification agent for agent mode."""
+        return self._intent_clarification_agent
+
     @property
     def cme_workflow(self) -> fastworkflow.Workflow:
         """Get the command metadata extraction workflow."""
@@ -297,19 +315,35 @@ class ChatSession:
 
     @property
     def workflow_is_complete(self) -> bool:
-        workflow = ChatSession.get_active_workflow()
+        workflow = self.get_active_workflow()
         return workflow.is_complete if workflow else True
     
     @workflow_is_complete.setter
     def workflow_is_complete(self, value: bool) -> None:
-        if workflow := ChatSession.get_active_workflow():
+        if workflow := self.get_active_workflow():
             workflow.is_complete = value
     
     @property
     def conversation_history(self) -> dspy.History:
         """Return the conversation history."""
         return self._conversation_history
-       
+
+    # def clear_conversation_history(self, trace_filename_suffix: Optional[str] = None) -> None:
+    def clear_conversation_history(self) -> None:
+        """
+        Clear the conversation history.
+        This resets the conversation history to an empty state.
+        """
+        self._conversation_history = dspy.History(messages=[])
+        # Filename for conversation traces
+        # if trace_filename_suffix:
+        #     self._conversation_traces_file_name: str = (
+        #         f"conversation_traces_{trace_filename_suffix}"
+        #     )
+        # else:
+        #     self._conversation_traces_file_name: str = (
+        #         f"conversation_traces_{datetime.now().strftime('%m_%d_%Y:%H_%M_%S')}.jsonl"
+        #     )
 
     def _run_workflow_loop(self) -> Optional[fastworkflow.CommandOutput]:
         """
@@ -319,12 +353,12 @@ class ChatSession:
         - All outputs (success or failure) are sent to queue during processing
         """
         last_output = None
-        workflow = ChatSession.get_active_workflow()
+        workflow = self.get_active_workflow()
 
         try:
             # Handle startup command/action
             if self._startup_command:
-                if self._run_as_agent:
+                if self._run_as_agent and not self._startup_command.startswith('/'):
                     # In agent mode, use workflow tool agent for processing
                     last_output = self._process_agent_message(self._startup_command)
                 else:
@@ -337,15 +371,24 @@ class ChatSession:
             ) and self._status != SessionStatus.STOPPING:
                 try:
                     message = self.user_message_queue.get()
-                    
-                    # Route based on mode and message type
-                    if self._run_as_agent:
-                        # In agent mode, use workflow tool agent for processing
-                        last_output = self._process_agent_message(message)
-                    elif self._is_mcp_tool_call(message):
-                        last_output = self._process_mcp_tool_call(message)
+
+                    # Handle Action objects directly
+                    if isinstance(message, fastworkflow.Action):
+                        last_output = self._process_action(message)
                     else:
-                        last_output = self._process_message(message)
+                        if ((
+                                "NLU_Pipeline_Stage" not in self._cme_workflow.context or
+                                self._cme_workflow.context["NLU_Pipeline_Stage"] == fastworkflow.NLUPipelineStage.INTENT_DETECTION) and
+                            message.startswith('/')
+                        ):
+                            self._cme_workflow.context["is_assistant_mode_command"] = True
+
+                        # Route based on mode and message type
+                        if self._run_as_agent and "is_assistant_mode_command" not in self._cme_workflow.context:
+                            # In agent mode, use workflow tool agent for processing
+                            last_output = self._process_agent_message(message)
+                        else:
+                            last_output = self._process_message(message)
                         
                 except Empty:
                     continue
@@ -356,91 +399,110 @@ class ChatSession:
 
         finally:
             self._status = SessionStatus.STOPPED
-            ChatSession.pop_active_workflow()
+            self.pop_active_workflow()
             logger.debug(f"Workflow {workflow.id if workflow else 'unknown'} completed")
 
         return None
     
-    def _is_mcp_tool_call(self, message: str) -> bool:
-        """Detect if message is an MCP tool call JSON"""
-        try:
-            data = json.loads(message)
-            return data.get("type") == "mcp_tool_call"
-        except (json.JSONDecodeError, AttributeError):
-            return False
+    # def _is_mcp_tool_call(self, message: str) -> bool:
+    #     """Detect if message is an MCP tool call JSON"""
+    #     try:
+    #         data = json.loads(message)
+    #         return data.get("type") == "mcp_tool_call"
+    #     except (json.JSONDecodeError, AttributeError):
+    #         return False
     
-    def _process_mcp_tool_call(self, message: str) -> fastworkflow.CommandOutput:
-        # sourcery skip: class-extract-method, extract-method
-        """Process an MCP tool call message"""
-        workflow = ChatSession.get_active_workflow()
+    # def _process_mcp_tool_call(self, message: str) -> fastworkflow.CommandOutput:
+    #     # sourcery skip: class-extract-method, extract-method
+    #     """Process an MCP tool call message"""
+    #     workflow = self.get_active_workflow()
         
-        try:
-            # Parse JSON message
-            data = json.loads(message)
-            tool_call_data = data["tool_call"]
+    #     try:
+    #         # Parse JSON message
+    #         data = json.loads(message)
+    #         tool_call_data = data["tool_call"]
             
-            # Create MCPToolCall object
-            tool_call = fastworkflow.MCPToolCall(
-                name=tool_call_data["name"],
-                arguments=tool_call_data["arguments"]
-            )
+    #         # Create MCPToolCall object
+    #         tool_call = fastworkflow.MCPToolCall(
+    #             name=tool_call_data["name"],
+    #             arguments=tool_call_data["arguments"]
+    #         )
             
-            # Execute via command executor
-            mcp_result = self._CommandExecutor.perform_mcp_tool_call(
-                workflow, 
-                tool_call, 
-                command_context=workflow.current_command_context_name
-            )
+    #         # Execute via command executor
+    #         mcp_result = self._CommandExecutor.perform_mcp_tool_call(
+    #             workflow, 
+    #             tool_call, 
+    #             command_context=workflow.current_command_context_name
+    #         )
             
-            # Convert MCPToolResult back to CommandOutput for consistency
-            command_output = self._convert_mcp_result_to_command_output(mcp_result)
+    #         # Convert MCPToolResult back to CommandOutput for consistency
+    #         command_output = self._convert_mcp_result_to_command_output(mcp_result)
             
-            # Put in output queue if needed
-            if (not command_output.success or self._keep_alive) and self.command_output_queue:
-                self.command_output_queue.put(command_output)
+    #         # Put in output queue if needed
+    #         if (not command_output.success or self._keep_alive) and self.command_output_queue:
+    #             self.command_output_queue.put(command_output)
 
-            # Flush on successful or failed tool call – state may have changed.
-            if workflow := ChatSession.get_active_workflow():
-                workflow.flush()
+    #         # Flush on successful or failed tool call – state may have changed.
+    #         if workflow := self.get_active_workflow():
+    #             workflow.flush()
             
-            return command_output
+    #         return command_output
             
-        except Exception as e:
-            logger.error(f"Error processing MCP tool call: {e}. Tool call content: {message}")
-            return self._process_message(message)  # process as a message
+    #     except Exception as e:
+    #         logger.error(f"Error processing MCP tool call: {e}. Tool call content: {message}")
+    #         return self._process_message(message)  # process as a message
     
-    def _convert_mcp_result_to_command_output(self, mcp_result: fastworkflow.MCPToolResult) -> fastworkflow.CommandOutput:
-        """Convert MCPToolResult to CommandOutput for compatibility"""
-        command_response = fastworkflow.CommandResponse(
-            response=mcp_result.content[0].text if mcp_result.content else "No response",
-            success=not mcp_result.isError
-        )
+    # def _convert_mcp_result_to_command_output(self, mcp_result: fastworkflow.MCPToolResult) -> fastworkflow.CommandOutput:
+    #     """Convert MCPToolResult to CommandOutput for compatibility"""
+    #     command_response = fastworkflow.CommandResponse(
+    #         response=mcp_result.content[0].text if mcp_result.content else "No response",
+    #         success=not mcp_result.isError
+    #     )
         
-        command_output = fastworkflow.CommandOutput(command_responses=[command_response])
-        command_output._mcp_source = mcp_result  # Mark for special formatting
-        return command_output
+    #     command_output = fastworkflow.CommandOutput(command_responses=[command_response])
+    #     command_output._mcp_source = mcp_result  # Mark for special formatting
+    #     return command_output
     
     def _process_agent_message(self, message: str) -> fastworkflow.CommandOutput:
+        # sourcery skip: class-extract-method
         """Process a message in agent mode using workflow tool agent"""
         # The agent processes the user's message and may make multiple tool calls
         # to the workflow internally (directly via CommandExecutor)
 
         # Ensure any prior action log is removed before a fresh agent run
-        if os.path.exists("action.json"):
-            os.remove("action.json")
+        if os.path.exists("action.jsonl"):
+            os.remove("action.jsonl")
 
-        refined_message = f'messsage\n{self._think_and_plan(message, self.conversation_history)}'
+        refined_user_query = self._refine_user_query(message, self.conversation_history)
+
+        from fastworkflow.workflow_agent import build_query_with_next_steps
+        command_info_and_refined_message_with_todolist = build_query_with_next_steps(
+            refined_user_query, 
+            self
+        )
+
+        # Get available commands for current context and pass to agent.
+        # The CommandsSystemPreludeAdapter will inject these commands into the system 
+        # message, keeping them out of the trajectory to avoid token bloat while still 
+        # providing context-specific command info.
+        from fastworkflow.workflow_agent import _what_can_i_do
+        available_commands = _what_can_i_do(self)
 
         lm = dspy_utils.get_lm("LLM_AGENT", "LITELLM_API_KEY_AGENT")
         from dspy.utils.exceptions import AdapterParseError
+        from fastworkflow.utils.chat_adapter import CommandsSystemPreludeAdapter
+        
+        # Use CommandsSystemPreludeAdapter specifically for workflow agent calls
+        agent_adapter = CommandsSystemPreludeAdapter()
+        
         # Retry logic for AdapterParseError
         max_retries = 2
         for attempt in range(max_retries):
             try:
-                with dspy.context(lm=lm, adapter=dspy.ChatAdapter()):
+                with dspy.context(lm=lm, adapter=agent_adapter):
                     agent_result = self._workflow_tool_agent(
-                        user_query=refined_message,
-                        conversation_history=self.conversation_history
+                        user_query=command_info_and_refined_message_with_todolist,
+                        available_commands=available_commands
                     )
                 break  # Success, exit retry loop
             except AdapterParseError as _:
@@ -460,33 +522,232 @@ class ChatSession:
         # Create CommandOutput with the agent's response
         command_response = fastworkflow.CommandResponse(response=result_text)
 
-        user_instructions_summary = message
+        conversation_traces = None
+        conversation_summary = message
         # Attach actions captured during agent execution as artifacts if available
-        if os.path.exists("action.json"):
-            with open("action.json", "r", encoding="utf-8") as f:
+        if os.path.exists("action.jsonl"):
+            with open("action.jsonl", "r", encoding="utf-8") as f:
                 actions = [json.loads(line) for line in f if line.strip()]
-            user_instructions_summary = self._extract_user_instructions(message, actions)
-            command_response.artifacts["user_instructions_summary"] = user_instructions_summary
+            conversation_summary, conversation_traces = self._extract_conversation_summary(message, actions, result_text)
+            command_response.artifacts["conversation_summary"] = conversation_summary
 
         self.conversation_history.messages.append(
-            {"user_instructions": user_instructions_summary, 
-             "agent_response": result_text}
+            {
+                "conversation summary": conversation_summary,
+                "conversation_traces": conversation_traces,
+                "feedback": None  # Initialize feedback slot for this turn
+            }
         )
 
         command_output = fastworkflow.CommandOutput(
             command_responses=[command_response]
         )
+        command_output.workflow_name = self._current_workflow.folderpath.split('/')[-1]
 
         # Put output in queue (following same pattern as _process_message)
         if (not command_output.success or self._keep_alive) and \
-                self.command_output_queue:
+                    self.command_output_queue:
             self.command_output_queue.put(command_output)
 
         # Persist workflow state changes
-        if workflow := ChatSession.get_active_workflow():
+        if workflow := self.get_active_workflow():
+            workflow.flush()
+
+        return command_output
+
+    def _process_message(self, message: str) -> fastworkflow.CommandOutput:
+        """Process a single message"""
+        # Pre-execution trace
+        if self.command_trace_queue:
+            self.command_trace_queue.put(fastworkflow.CommandTraceEvent(
+                direction=fastworkflow.CommandTraceEventDirection.AGENT_TO_WORKFLOW,
+                raw_command=message,
+                command_name=None,
+                parameters=None,
+                response_text=None,
+                success=None,
+                timestamp_ms=int(time.time() * 1000),
+            ))
+        
+        # Execute command
+        command_output = self._CommandExecutor.invoke_command(self, message)
+
+        # Extract response text and parameters for traces
+        response_text = ""
+        if command_output.command_responses:
+            response_text = command_output.command_responses[0].response or ""
+        
+        # Convert parameters to dict if it's a Pydantic model or other complex object
+        params = command_output.command_parameters or {}
+        if hasattr(params, 'model_dump'):
+            params_dict = params.model_dump()
+        elif hasattr(params, 'dict'):
+            params_dict = params.dict()
+        else:
+            params_dict = params
+        
+        # Post-execution trace
+        if self.command_trace_queue:
+            self.command_trace_queue.put(fastworkflow.CommandTraceEvent(
+                direction=fastworkflow.CommandTraceEventDirection.WORKFLOW_TO_AGENT,
+                raw_command=None,
+                command_name=command_output.command_name or "",
+                parameters=params_dict,
+                response_text=response_text,
+                success=bool(command_output.success),
+                timestamp_ms=int(time.time() * 1000),
+            ))
+        
+        # Record assistant mode trace to action.jsonl (similar to agent mode in workflow_agent.py)
+        # This ensures assistant commands are captured even when interspersed with agent commands
+        record = {
+            "command": message,
+            "command_name": command_output.command_name or "",
+            "parameters": params_dict,
+            "response": response_text
+        }
+
+        self.conversation_history.messages.append(
+            {
+                "conversation summary": "assistant_mode_command",
+                "conversation_traces": json.dumps(record),
+                "feedback": None  # Initialize feedback slot for this turn
+            }
+        )
+
+        if (not command_output.success or self._keep_alive) and \
+            self.command_output_queue:
+            self.command_output_queue.put(command_output)
+
+        # Persist workflow state changes lazily accumulated during message processing.
+        if workflow := self.get_active_workflow():
+            workflow.flush()
+
+        return command_output
+
+    def _process_action(self, action: fastworkflow.Action) -> fastworkflow.CommandOutput:
+        """Process a startup action"""
+        workflow = self.get_active_workflow()
+        
+        # Serialize action parameters for trace
+        params = action.parameters or {}
+        if hasattr(params, 'model_dump'):
+            params_dict = params.model_dump()
+        elif hasattr(params, 'dict'):
+            params_dict = params.dict()
+        else:
+            params_dict = params
+        
+        # Pre-execution trace: serialize action as raw_command
+        raw_command = f"{action.command_name} {json.dumps(params_dict)}"
+        if self.command_trace_queue:
+            self.command_trace_queue.put(fastworkflow.CommandTraceEvent(
+                direction=fastworkflow.CommandTraceEventDirection.AGENT_TO_WORKFLOW,
+                raw_command=raw_command,
+                command_name=None,
+                parameters=None,
+                response_text=None,
+                success=None,
+                timestamp_ms=int(time.time() * 1000),
+            ))
+        
+        # Execute the action
+        command_output = self._CommandExecutor.perform_action(workflow, action)
+
+        # Extract response text for post-execution trace
+        response_text = ""
+        if command_output.command_responses:
+            response_text = command_output.command_responses[0].response or ""
+        
+        # Post-execution trace
+        if self.command_trace_queue:
+            self.command_trace_queue.put(fastworkflow.CommandTraceEvent(
+                direction=fastworkflow.CommandTraceEventDirection.WORKFLOW_TO_AGENT,
+                raw_command=None,
+                command_name=command_output.command_name,
+                parameters=params_dict,
+                response_text=response_text,
+                success=bool(command_output.success),
+                timestamp_ms=int(time.time() * 1000),
+            ))
+        
+        # Record action trace to action.jsonl
+        record = {
+            "command": "process_action",
+            "command_name": action.command_name,
+            "parameters": params_dict,
+            "response": response_text
+        }
+
+        self.conversation_history.messages.append(
+            {
+                "conversation summary": "process_action command",
+                "conversation_traces": json.dumps(record),
+                "feedback": None  # Initialize feedback slot for this turn
+            }
+        )
+
+        if (not command_output.success or self._keep_alive) and \
+            self.command_output_queue:
+            self.command_output_queue.put(command_output)
+
+        # Flush any pending workflow updates triggered by this startup action.
+        if workflow:
             workflow.flush()
 
         return command_output
+
+    def _refine_user_query(self, user_query: str, conversation_history: dspy.History) -> str:
+        """
+        Refine user query using conversation history. 
+        Return the refined user query
+        """
+        if conversation_history.messages:
+            messages = []
+            for conv_dict in conversation_history.messages[-5:]:
+                messages.extend([
+                    f'{k}: {v}' for k, v in conv_dict.items()
+                ])
+            messages.append(f'new_user_query: {user_query}')
+            return '\n'.join(messages)
+
+        return user_query    
+
+    def _extract_conversation_summary(self, 
+        user_query: str, workflow_actions: list[dict[str, str]], final_agent_response: str) -> str:
+        """
+        Summarizes conversation based on original user query, workflow actions and agent response.
+        Returns the conversation summary and the log entry
+        """
+        # Lets log everything to a file called action_log.jsonl, if it exists
+        conversation_traces = {
+            "user_query": user_query,
+            "agent_workflow_interactions": workflow_actions,
+            "final_agent_response": final_agent_response
+        }
+        # with open(self._conversation_traces_file_name, "a", encoding="utf-8") as f:
+        #     f.write(json.dumps(log_entry) + "\n")
+
+        class ConversationSummarySignature(dspy.Signature):
+            """
+            A summary of conversation
+            Omit descriptions of action sequences 
+            Capture relevant facts and parameter values from user query, workflow actions and agent response
+            """
+            user_query: str = dspy.InputField()
+            workflow_actions: list[dict[str, str]] = dspy.InputField()
+            final_agent_response: str = dspy.InputField()
+            conversation_summary: str = dspy.OutputField(desc="A multiline paragraph summary")
+
+        planner_lm = dspy_utils.get_lm("LLM_PLANNER", "LITELLM_API_KEY_PLANNER")
+        with dspy.context(lm=planner_lm):
+            cs_func = dspy.ChainOfThought(ConversationSummarySignature)
+            prediction = cs_func(
+                user_query=user_query, 
+                workflow_actions=workflow_actions, 
+                final_agent_response=final_agent_response)
+            return prediction.conversation_summary, json.dumps(conversation_traces)
+
     
     def profile_invoke_command(self, message: str):
         """
@@ -576,91 +837,3 @@ class ChatSession:
         print(f"Detailed report saved to {os.path.abspath(report_file)}")
         
         return result
-
-    def _process_message(self, message: str) -> fastworkflow.CommandOutput:
-        """Process a single message"""
-        # Use our specialized profiling method
-        # command_output = self.profile_invoke_command(message)
-        
-        command_output = self._CommandExecutor.invoke_command(self, message)
-        if (not command_output.success or self._keep_alive) and \
-            self.command_output_queue:
-            self.command_output_queue.put(command_output)
-
-        # Persist workflow state changes lazily accumulated during message processing.
-        if workflow := ChatSession.get_active_workflow():
-            workflow.flush()
-
-        return command_output
-
-    def _process_action(self, action: fastworkflow.Action) -> fastworkflow.CommandOutput:
-        """Process a startup action"""
-        workflow = ChatSession.get_active_workflow()
-        command_output = self._CommandExecutor.perform_action(workflow, action)
-        if (not command_output.success or self._keep_alive) and \
-            self.command_output_queue:
-            self.command_output_queue.put(command_output)
-
-        # Flush any pending workflow updates triggered by this startup action.
-        if workflow:
-            workflow.flush()
-
-        return command_output
-
-    def _think_and_plan(self, user_query: str, conversation_history: dspy.History) -> str:
-        """
-        Returns a refined plan by breaking down a user_query into simpler tasks.
-        """
-        class TaskPlannerSignature(dspy.Signature):
-            """
-            Break down a user_query into simpler tasks based only on available commands and conversation_history.
-            If user_query is simple, return a single todo that is the user_query as-is
-            """
-            user_query: str = dspy.InputField()
-            conversation_history: dspy.History = dspy.InputField()
-            available_commands: list[str] = dspy.InputField()
-            todo_list: list[str] = dspy.OutputField(desc="task descriptions as short sentences")
-
-        current_workflow = ChatSession.get_active_workflow()
-        available_commands = CommandMetadataAPI.get_command_display_text(
-            subject_workflow_path=current_workflow.folderpath,
-            cme_workflow_path=fastworkflow.get_internal_workflow_path("command_metadata_extraction"),
-            active_context_name=current_workflow.current_command_context_name,
-        )
-
-        planner_lm = dspy_utils.get_lm("LLM_PLANNER", "LITELLM_API_KEY_PLANNER")
-        with dspy.context(lm=planner_lm):
-            task_planner_func = dspy.ChainOfThought(TaskPlannerSignature)
-            prediction = task_planner_func(
-                user_query=user_query,
-                conversation_history=conversation_history, 
-                available_commands=available_commands)
-
-            if not prediction.todo_list or (len(prediction.todo_list) == 1 and prediction.todo_list[0] == user_query):
-                return user_query
-
-            steps_list = '\n'.join([f'{i + 1}. {task}' for i, task in enumerate(prediction.todo_list)])
-            return f"{user_query}\nNext steps:\n{steps_list}"
-
-
-    def _extract_user_instructions(self, 
-        user_query: str, workflow_actions: list[dict[str, str]]) -> str:
-        """
-        Summarizes user instructions based on original user query and subsequent user feedback in workflow actions.
-        """
-        class UserInstructionCompilerSignature(dspy.Signature):
-            """
-            Concise summary of user instructions based on their commands to the workflow. 
-            Include parameter values passed in commands in the summary.
-            """
-            commands_list: list[str] = dspy.InputField()
-            user_instructions_summary: str = dspy.OutputField(desc="A single paragraph summary")
-
-        commands_list: list[str] = [user_query]
-        commands_list.extend([wf_action['command'] for wf_action in workflow_actions if 'command' in wf_action])
-
-        planner_lm = dspy_utils.get_lm("LLM_PLANNER", "LITELLM_API_KEY_PLANNER")
-        with dspy.context(lm=planner_lm):
-            uic_func = dspy.ChainOfThought(UserInstructionCompilerSignature)
-            prediction = uic_func(commands_list=commands_list)
-            return prediction.user_instructions_summary