khoj 1.42.9.dev26__py3-none-any.whl → 1.42.10__py3-none-any.whl

khoj/processor/conversation/google/utils.py

@@ -1,10 +1,9 @@
-import json
 import logging
 import os
 import random
 from copy import deepcopy
 from time import perf_counter
-from typing import AsyncGenerator, AsyncIterator, Dict, List
+from typing import AsyncGenerator, AsyncIterator, Dict
 
 import httpx
 from google import genai
@@ -23,13 +22,11 @@ from tenacity import (
 
 from khoj.processor.conversation.utils import (
     ResponseWithThought,
-    ToolCall,
     commit_conversation_trace,
     get_image_from_base64,
     get_image_from_url,
 )
 from khoj.utils.helpers import (
-    ToolDefinition,
     get_chat_usage_metrics,
     get_gemini_client,
     is_none_or_empty,
@@ -102,29 +99,26 @@ def gemini_completion_with_backoff(
     temperature=1.2,
     api_key=None,
     api_base_url: str = None,
-    model_kwargs={},
+    model_kwargs=None,
     deepthought=False,
     tracer={},
-) -> ResponseWithThought:
+) -> str:
     client = gemini_clients.get(api_key)
     if not client:
         client = get_gemini_client(api_key, api_base_url)
         gemini_clients[api_key] = client
 
     formatted_messages, system_instruction = format_messages_for_gemini(messages, system_prompt)
-    raw_content, response_text, response_thoughts = [], "", None
+    response_thoughts: str | None = None
 
-    # Configure structured output
-    tools = None
+    # format model response schema
     response_schema = None
-    if model_kwargs.get("tools"):
-        tools = to_gemini_tools(model_kwargs["tools"])
-    elif model_kwargs.get("response_schema"):
+    if model_kwargs and model_kwargs.get("response_schema"):
         response_schema = clean_response_schema(model_kwargs["response_schema"])
 
     thinking_config = None
     if deepthought and is_reasoning_model(model_name):
-        thinking_config = gtypes.ThinkingConfig(thinking_budget=MAX_REASONING_TOKENS_GEMINI, include_thoughts=True)
+        thinking_config = gtypes.ThinkingConfig(thinking_budget=MAX_REASONING_TOKENS_GEMINI)
 
     max_output_tokens = MAX_OUTPUT_TOKENS_FOR_STANDARD_GEMINI
     if is_reasoning_model(model_name):
@@ -137,9 +131,8 @@ def gemini_completion_with_backoff(
         thinking_config=thinking_config,
         max_output_tokens=max_output_tokens,
         safety_settings=SAFETY_SETTINGS,
-        response_mime_type=model_kwargs.get("response_mime_type", "text/plain"),
+        response_mime_type=model_kwargs.get("response_mime_type", "text/plain") if model_kwargs else "text/plain",
         response_schema=response_schema,
-        tools=tools,
         seed=seed,
         top_p=0.95,
         http_options=gtypes.HttpOptions(client_args={"timeout": httpx.Timeout(30.0, read=60.0)}),
@@ -148,25 +141,7 @@ def gemini_completion_with_backoff(
     try:
         # Generate the response
         response = client.models.generate_content(model=model_name, config=config, contents=formatted_messages)
-        if (
-            not response.candidates
-            or not response.candidates[0].content
-            or response.candidates[0].content.parts is None
-        ):
-            raise ValueError(f"Failed to get response from model.")
-        raw_content = [part.model_dump() for part in response.candidates[0].content.parts]
-        if response.function_calls:
-            function_calls = [
-                ToolCall(name=function_call.name, args=function_call.args, id=function_call.id).__dict__
-                for function_call in response.function_calls
-            ]
-            response_text = json.dumps(function_calls)
-        else:
-            # If no function calls, use the text response
-            response_text = response.text
-        response_thoughts = "\n".join(
-            [part.text for part in response.candidates[0].content.parts if part.thought and isinstance(part.text, str)]
-        )
+        response_text = response.text
     except gerrors.ClientError as e:
         response = None
         response_text, _ = handle_gemini_response(e.args)
@@ -180,14 +155,8 @@ def gemini_completion_with_backoff(
     input_tokens = response.usage_metadata.prompt_token_count or 0 if response else 0
     output_tokens = response.usage_metadata.candidates_token_count or 0 if response else 0
     thought_tokens = response.usage_metadata.thoughts_token_count or 0 if response else 0
-    cache_read_tokens = response.usage_metadata.cached_content_token_count or 0 if response else 0
     tracer["usage"] = get_chat_usage_metrics(
-        model_name,
-        input_tokens,
-        output_tokens,
-        cache_read_tokens=cache_read_tokens,
-        thought_tokens=thought_tokens,
-        usage=tracer.get("usage"),
+        model_name, input_tokens, output_tokens, thought_tokens=thought_tokens, usage=tracer.get("usage")
     )
 
     # Validate the response. If empty, raise an error to retry.
@@ -201,7 +170,7 @@ def gemini_completion_with_backoff(
     if is_promptrace_enabled():
         commit_conversation_trace(messages, response_text, tracer)
 
-    return ResponseWithThought(text=response_text, thought=response_thoughts, raw_content=raw_content)
+    return response_text
 
 
 @retry(
@@ -269,7 +238,7 @@ async def gemini_chat_completion_with_backoff(
         # handle safety, rate-limit, other finish reasons
         stop_message, stopped = handle_gemini_response(chunk.candidates, chunk.prompt_feedback)
         if stopped:
-            yield ResponseWithThought(text=stop_message)
+            yield ResponseWithThought(response=stop_message)
             logger.warning(
                 f"LLM Response Prevented for {model_name}: {stop_message}.\n"
                 + f"Last Message by {messages[-1].role}: {messages[-1].content}"
@@ -282,7 +251,7 @@ async def gemini_chat_completion_with_backoff(
                 yield ResponseWithThought(thought=part.text)
             elif part.text:
                 aggregated_response += part.text
-                yield ResponseWithThought(text=part.text)
+                yield ResponseWithThought(response=part.text)
     # Calculate cost of chat
     input_tokens = final_chunk.usage_metadata.prompt_token_count or 0 if final_chunk else 0
     output_tokens = final_chunk.usage_metadata.candidates_token_count or 0 if final_chunk else 0
@@ -381,24 +350,8 @@ def format_messages_for_gemini(
     system_prompt = None if is_none_or_empty(system_prompt) else system_prompt
 
     for message in messages:
-        if message.role == "assistant":
-            message.role = "model"
-
-        # Handle tool call and tool result message types from additional_kwargs
-        message_type = message.additional_kwargs.get("message_type")
-        if message_type == "tool_call":
-            pass
-        elif message_type == "tool_result":
-            # Convert tool_result to Gemini function response format
-            # Need to find the corresponding function call from previous messages
-            tool_result_msg_content = []
-            for part in message.content:
-                tool_result_msg_content.append(
-                    gtypes.Part.from_function_response(name=part["name"], response={"result": part["content"]})
-                )
-            message.content = tool_result_msg_content
         # Convert message content to string list from chatml dictionary list
-        elif isinstance(message.content, list):
+        if isinstance(message.content, list):
             # Convert image_urls to PIL.Image and place them at beginning of list (better for Gemini)
             message_content = []
             for item in sorted(message.content, key=lambda x: 0 if x["type"] == "image_url" else 1):
@@ -418,13 +371,16 @@ def format_messages_for_gemini(
                 messages.remove(message)
                 continue
             message.content = message_content
-        elif isinstance(message.content, str) and message.content.strip():
+        elif isinstance(message.content, str):
             message.content = [gtypes.Part.from_text(text=message.content)]
         else:
             logger.error(f"Dropping invalid type: {type(message.content)} of message content: {message.content}")
             messages.remove(message)
             continue
 
+        if message.role == "assistant":
+            message.role = "model"
+
     if len(messages) == 1:
         messages[0].role = "user"
 
@@ -452,21 +408,3 @@ def is_reasoning_model(model_name: str) -> bool:
     Check if the model is a reasoning model.
     """
     return model_name.startswith("gemini-2.5")
-
-
-def to_gemini_tools(tools: List[ToolDefinition]) -> List[gtypes.ToolDict] | None:
-    "Transform tool definitions from standard format to Gemini format."
-    gemini_tools = [
-        gtypes.ToolDict(
-            function_declarations=[
-                gtypes.FunctionDeclarationDict(
-                    name=tool.name,
-                    description=tool.description,
-                    parameters=tool.schema,
-                )
-                for tool in tools
-            ]
-        )
-    ]
-
-    return gemini_tools or None

khoj/processor/conversation/offline/chat_model.py

@@ -145,12 +145,12 @@ async def converse_offline(
                 aggregated_response += response_delta
                 # Put chunk into the asyncio queue (non-blocking)
                 try:
-                    queue.put_nowait(ResponseWithThought(text=response_delta))
+                    queue.put_nowait(ResponseWithThought(response=response_delta))
                 except asyncio.QueueFull:
                     # Should not happen with default queue size unless consumer is very slow
                     logger.warning("Asyncio queue full during offline LLM streaming.")
                     # Potentially block here or handle differently if needed
-                    asyncio.run(queue.put(ResponseWithThought(text=response_delta)))
+                    asyncio.run(queue.put(ResponseWithThought(response=response_delta)))
 
             # Log the time taken to stream the entire response
             logger.info(f"Chat streaming took: {perf_counter() - start_time:.3f} seconds")
@@ -221,4 +221,4 @@ def send_message_to_model_offline(
     if is_promptrace_enabled():
         commit_conversation_trace(messages, response_text, tracer)
 
-    return ResponseWithThought(text=response_text)
+    return response_text

khoj/processor/conversation/openai/gpt.py

@@ -1,24 +1,25 @@
 import logging
 from datetime import datetime
-from typing import Any, AsyncGenerator, Dict, List, Optional
+from typing import AsyncGenerator, Dict, List, Optional
+
+from openai.lib._pydantic import _ensure_strict_json_schema
+from pydantic import BaseModel
 
 from khoj.database.models import Agent, ChatMessageModel, ChatModel
 from khoj.processor.conversation import prompts
 from khoj.processor.conversation.openai.utils import (
     chat_completion_with_backoff,
-    clean_response_schema,
     completion_with_backoff,
-    get_structured_output_support,
-    to_openai_tools,
+    get_openai_api_json_support,
 )
 from khoj.processor.conversation.utils import (
+    JsonSupport,
     OperatorRun,
     ResponseWithThought,
-    StructuredOutputSupport,
     generate_chatml_messages_with_context,
     messages_to_print,
 )
-from khoj.utils.helpers import ToolDefinition, is_none_or_empty, truncate_code_context
+from khoj.utils.helpers import is_none_or_empty, truncate_code_context
 from khoj.utils.rawconfig import FileAttachment, LocationData
 from khoj.utils.yaml import yaml_dump
 
@@ -31,7 +32,6 @@ def send_message_to_model(
     model,
     response_type="text",
     response_schema=None,
-    tools: list[ToolDefinition] = None,
     deepthought=False,
     api_base_url=None,
     tracer: dict = {},
@@ -40,11 +40,9 @@ def send_message_to_model(
     Send message to model
     """
 
-    model_kwargs: Dict[str, Any] = {}
-    json_support = get_structured_output_support(model, api_base_url)
-    if tools and json_support == StructuredOutputSupport.TOOL:
-        model_kwargs["tools"] = to_openai_tools(tools)
-    elif response_schema and json_support >= StructuredOutputSupport.SCHEMA:
+    model_kwargs = {}
+    json_support = get_openai_api_json_support(model, api_base_url)
+    if response_schema and json_support == JsonSupport.SCHEMA:
         # Drop unsupported fields from schema passed to OpenAI APi
         cleaned_response_schema = clean_response_schema(response_schema)
         model_kwargs["response_format"] = {
@@ -55,7 +53,7 @@ def send_message_to_model(
                 "strict": True,
             },
         }
-    elif response_type == "json_object" and json_support == StructuredOutputSupport.OBJECT:
+    elif response_type == "json_object" and json_support == JsonSupport.OBJECT:
         model_kwargs["response_format"] = {"type": response_type}
 
     # Get Response from GPT
@@ -173,3 +171,30 @@ async def converse_openai(
         tracer=tracer,
     ):
         yield chunk
+
+
+def clean_response_schema(schema: BaseModel | dict) -> dict:
+    """
+    Format response schema to be compatible with OpenAI API.
+
+    Clean the response schema by removing unsupported fields.
+    """
+    # Normalize schema to OpenAI compatible JSON schema format
+    schema_json = schema if isinstance(schema, dict) else schema.model_json_schema()
+    schema_json = _ensure_strict_json_schema(schema_json, path=(), root=schema_json)
+
+    # Recursively drop unsupported fields from schema passed to OpenAI API
+    # See https://platform.openai.com/docs/guides/structured-outputs#supported-schemas
+    fields_to_exclude = ["minItems", "maxItems"]
+    if isinstance(schema_json, dict) and isinstance(schema_json.get("properties"), dict):
+        for _, prop_value in schema_json["properties"].items():
+            if isinstance(prop_value, dict):
+                # Remove specified fields from direct properties
+                for field in fields_to_exclude:
+                    prop_value.pop(field, None)
+            # Recursively remove specified fields from child properties
+            if "items" in prop_value and isinstance(prop_value["items"], dict):
+                clean_response_schema(prop_value["items"])
+
+    # Return cleaned schema
+    return schema_json

khoj/processor/conversation/openai/utils.py

@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 from copy import deepcopy
@@ -10,7 +9,6 @@ from urllib.parse import urlparse
 import httpx
 import openai
 from langchain_core.messages.chat import ChatMessage
-from openai.lib._pydantic import _ensure_strict_json_schema
 from openai.lib.streaming.chat import (
     ChatCompletionStream,
     ChatCompletionStreamEvent,
@@ -22,7 +20,6 @@ from openai.types.chat.chat_completion_chunk import (
     Choice,
     ChoiceDelta,
 )
-from pydantic import BaseModel
 from tenacity import (
     before_sleep_log,
     retry,
@@ -33,13 +30,11 @@ from tenacity import (
 )
 
 from khoj.processor.conversation.utils import (
+    JsonSupport,
     ResponseWithThought,
-    StructuredOutputSupport,
-    ToolCall,
     commit_conversation_trace,
 )
 from khoj.utils.helpers import (
-    ToolDefinition,
     convert_image_data_uri,
     get_chat_usage_metrics,
     get_openai_async_client,
@@ -77,7 +72,7 @@ def completion_with_backoff(
     deepthought: bool = False,
     model_kwargs: dict = {},
     tracer: dict = {},
-) -> ResponseWithThought:
+) -> str:
     client_key = f"{openai_api_key}--{api_base_url}"
     client = openai_clients.get(client_key)
     if not client:
@@ -122,9 +117,6 @@ def completion_with_backoff(
     if os.getenv("KHOJ_LLM_SEED"):
         model_kwargs["seed"] = int(os.getenv("KHOJ_LLM_SEED"))
 
-    tool_ids = []
-    tool_calls: list[ToolCall] = []
-    thoughts = ""
     aggregated_response = ""
     if stream:
         with client.beta.chat.completions.stream(
@@ -138,16 +130,7 @@ def completion_with_backoff(
                 if chunk.type == "content.delta":
                     aggregated_response += chunk.delta
                 elif chunk.type == "thought.delta":
-                    thoughts += chunk.delta
-                elif chunk.type == "chunk" and chunk.chunk.choices and chunk.chunk.choices[0].delta.tool_calls:
-                    tool_ids += [tool_call.id for tool_call in chunk.chunk.choices[0].delta.tool_calls]
-                elif chunk.type == "tool_calls.function.arguments.done":
-                    tool_calls += [ToolCall(name=chunk.name, args=json.loads(chunk.arguments), id=None)]
-        if tool_calls:
-            tool_calls = [
-                ToolCall(name=chunk.name, args=chunk.args, id=tool_id) for chunk, tool_id in zip(tool_calls, tool_ids)
-            ]
-            aggregated_response = json.dumps([tool_call.__dict__ for tool_call in tool_calls])
+                    pass
     else:
         # Non-streaming chat completion
         chunk = client.beta.chat.completions.parse(
@@ -181,7 +164,7 @@ def completion_with_backoff(
     if is_promptrace_enabled():
         commit_conversation_trace(messages, aggregated_response, tracer)
 
-    return ResponseWithThought(text=aggregated_response, thought=thoughts)
+    return aggregated_response
 
 
 @retry(
@@ -207,7 +190,6 @@ async def chat_completion_with_backoff(
     deepthought=False,
     model_kwargs: dict = {},
     tracer: dict = {},
-    tools=None,
 ) -> AsyncGenerator[ResponseWithThought, None]:
     client_key = f"{openai_api_key}--{api_base_url}"
     client = openai_async_clients.get(client_key)
@@ -276,8 +258,6 @@ async def chat_completion_with_backoff(
     read_timeout = 300 if is_local_api(api_base_url) else 60
     if os.getenv("KHOJ_LLM_SEED"):
         model_kwargs["seed"] = int(os.getenv("KHOJ_LLM_SEED"))
-    if tools:
-        model_kwargs["tools"] = tools
 
     aggregated_response = ""
     final_chunk = None
@@ -297,7 +277,7 @@ async def chat_completion_with_backoff(
             raise ValueError("No response by model.")
         aggregated_response = response.choices[0].message.content
         final_chunk = response
-        yield ResponseWithThought(text=aggregated_response)
+        yield ResponseWithThought(response=aggregated_response)
     else:
         async for chunk in stream_processor(response):
             # Log the time taken to start response
@@ -313,8 +293,8 @@ async def chat_completion_with_backoff(
             response_chunk: ResponseWithThought = None
             response_delta = chunk.choices[0].delta
             if response_delta.content:
-                response_chunk = ResponseWithThought(text=response_delta.content)
-                aggregated_response += response_chunk.text
+                response_chunk = ResponseWithThought(response=response_delta.content)
+                aggregated_response += response_chunk.response
             elif response_delta.thought:
                 response_chunk = ResponseWithThought(thought=response_delta.thought)
             if response_chunk:
@@ -347,16 +327,16 @@ async def chat_completion_with_backoff(
         commit_conversation_trace(messages, aggregated_response, tracer)
 
 
-def get_structured_output_support(model_name: str, api_base_url: str = None) -> StructuredOutputSupport:
+def get_openai_api_json_support(model_name: str, api_base_url: str = None) -> JsonSupport:
     if model_name.startswith("deepseek-reasoner"):
-        return StructuredOutputSupport.NONE
+        return JsonSupport.NONE
     if api_base_url:
         host = urlparse(api_base_url).hostname
         if host and host.endswith(".ai.azure.com"):
-            return StructuredOutputSupport.OBJECT
+            return JsonSupport.OBJECT
         if host == "api.deepinfra.com":
-            return StructuredOutputSupport.OBJECT
-    return StructuredOutputSupport.TOOL
+            return JsonSupport.OBJECT
+    return JsonSupport.SCHEMA
 
 
 def format_message_for_api(messages: List[ChatMessage], api_base_url: str) -> List[dict]:
@@ -365,43 +345,6 @@ def format_message_for_api(messages: List[ChatMessage], api_base_url: str) -> Li
     """
     formatted_messages = []
     for message in deepcopy(messages):
-        # Handle tool call and tool result message types
-        message_type = message.additional_kwargs.get("message_type")
-        if message_type == "tool_call":
-            # Convert tool_call to OpenAI function call format
-            content = []
-            for part in message.content:
-                content.append(
-                    {
-                        "type": "function",
-                        "id": part.get("id"),
-                        "function": {
-                            "name": part.get("name"),
-                            "arguments": json.dumps(part.get("input", part.get("args", {}))),
-                        },
-                    }
-                )
-            formatted_messages.append(
-                {
-                    "role": "assistant",
-                    "content": None,
-                    "tool_calls": content,
-                }
-            )
-            continue
-        if message_type == "tool_result":
-            # Convert tool_result to OpenAI tool result format
-            # Each part is a result for a tool call
-            for part in message.content:
-                formatted_messages.append(
-                    {
-                        "role": "tool",
-                        "tool_call_id": part.get("id") or part.get("tool_use_id"),
-                        "name": part.get("name"),
-                        "content": part.get("content"),
-                    }
-                )
-            continue
         if isinstance(message.content, list) and not is_openai_api(api_base_url):
             assistant_texts = []
             has_images = False
@@ -765,47 +708,3 @@ def add_qwen_no_think_tag(formatted_messages: List[dict]) -> None:
                     if isinstance(content_part, dict) and content_part.get("type") == "text":
                         content_part["text"] += " /no_think"
                         break
-
-
-def to_openai_tools(tools: List[ToolDefinition]) -> List[Dict] | None:
-    "Transform tool definitions from standard format to OpenAI format."
-    openai_tools = [
-        {
-            "type": "function",
-            "function": {
-                "name": tool.name,
-                "description": tool.description,
-                "parameters": clean_response_schema(tool.schema),
-            },
-        }
-        for tool in tools
-    ]
-
-    return openai_tools or None
-
-
-def clean_response_schema(schema: BaseModel | dict) -> dict:
-    """
-    Format response schema to be compatible with OpenAI API.
-
-    Clean the response schema by removing unsupported fields.
-    """
-    # Normalize schema to OpenAI compatible JSON schema format
-    schema_json = schema if isinstance(schema, dict) else schema.model_json_schema()
-    schema_json = _ensure_strict_json_schema(schema_json, path=(), root=schema_json)
-
-    # Recursively drop unsupported fields from schema passed to OpenAI API
-    # See https://platform.openai.com/docs/guides/structured-outputs#supported-schemas
-    fields_to_exclude = ["minItems", "maxItems"]
-    if isinstance(schema_json, dict) and isinstance(schema_json.get("properties"), dict):
-        for _, prop_value in schema_json["properties"].items():
-            if isinstance(prop_value, dict):
-                # Remove specified fields from direct properties
-                for field in fields_to_exclude:
-                    prop_value.pop(field, None)
-            # Recursively remove specified fields from child properties
-            if "items" in prop_value and isinstance(prop_value["items"], dict):
-                clean_response_schema(prop_value["items"])
-
-    # Return cleaned schema
-    return schema_json

khoj/processor/conversation/prompts.py

@@ -667,37 +667,33 @@ Here's some additional context about you:
 
 plan_function_execution = PromptTemplate.from_template(
     """
-You are Khoj, a smart, creative and meticulous researcher. Use the provided tool AIs to accomplish the task assigned to you.
-Create a multi-step plan and intelligently iterate on the plan to complete the task.
+You are Khoj, a smart, creative and methodical researcher. Use the provided tool AIs to investigate information to answer query.
+Create a multi-step plan and intelligently iterate on the plan based on the retrieved information to find the requested information.
 {personality_context}
 
 # Instructions
-- Provide highly diverse, detailed requests to the tool AIs, one tool AI at a time, to gather information, perform actions etc. Their response will be shown to you in the next iteration.
+- Ask highly diverse, detailed queries to the tool AIs, one tool AI at a time, to discover required information or run calculations. Their response will be shown to you in the next iteration.
 - Break down your research process into independent, self-contained steps that can be executed sequentially using the available tool AIs to answer the user's query. Write your step-by-step plan in the scratchpad.
 - Always ask a new query that was not asked to the tool AI in a previous iteration. Build on the results of the previous iterations.
 - Ensure that all required context is passed to the tool AIs for successful execution. Include any relevant stuff that has previously been attempted. They only know the context provided in your query.
 - Think step by step to come up with creative strategies when the previous iteration did not yield useful results.
-- You are allowed upto {max_iterations} iterations to use the help of the provided tool AIs to accomplish the task assigned to you. Only stop when you have completed the task.
+- You are allowed upto {max_iterations} iterations to use the help of the provided tool AIs to answer the user's question.
+- Stop when you have the required information by returning a JSON object with the "tool" field set to "text" and "query" field empty. E.g., {{"scratchpad": "I have all I need", "tool": "text", "query": ""}}
 
 # Examples
-Assuming you can search the user's files and the internet.
+Assuming you can search the user's notes and the internet.
 - When the user asks for the population of their hometown
-  1. Try look up their hometown in their notes. Ask the semantic search AI to search for their birth certificate, childhood memories, school, resume etc.
-  2. Use the other document retrieval tools to build on the semantic search results, fill in the gaps, add more details or confirm your hypothesis.
-  3. If not found in their notes, try infer their hometown from their online social media profiles. Ask the online search AI to look for {username}'s biography, school, resume on linkedin, facebook, website etc.
-  4. Only then try find the latest population of their hometown by reading official websites with the help of the online search and web page reading AI.
+  1. Try look up their hometown in their notes. Ask the note search AI to search for their birth certificate, childhood memories, school, resume etc.
+  2. If not found in their notes, try infer their hometown from their online social media profiles. Ask the online search AI to look for {username}'s biography, school, resume on linkedin, facebook, website etc.
+  3. Only then try find the latest population of their hometown by reading official websites with the help of the online search and web page reading AI.
 - When the user asks for their computer's specs
-  1. Try find their computer model in their documents.
+  1. Try find their computer model in their notes.
   2. Now find webpages with their computer model's spec online.
   3. Ask the webpage tool AI to extract the required information from the relevant webpages.
 - When the user asks what clothes to carry for their upcoming trip
-  1. Use the semantic search tool to find the itinerary of their upcoming trip in their documents.
+  1. Find the itinerary of their upcoming trip in their notes.
   2. Next find the weather forecast at the destination online.
-  3. Then combine the semantic search, regex search, view file and list files tools to find if all the clothes they own in their files.
-- When the user asks you to summarize their expenses in a particular month
-  1. Combine the semantic search and regex search tool AI to find all transactions in the user's documents for that month.
-  2. Use the view file tool to read the line ranges in the matched files
-  3. Finally summarize the expenses
+  3. Then find if they mentioned what clothes they own in their notes.
 
 # Background Context
 - Current Date: {day_of_week}, {current_date}
@@ -705,9 +701,31 @@ Assuming you can search the user's files and the internet.
 - User Name: {username}
 
 # Available Tool AIs
-You decide which of the tool AIs listed below would you use to accomplish the user assigned task. You **only** have access to the following tool AIs:
+You decide which of the tool AIs listed below would you use to answer the user's question. You **only** have access to the following tool AIs:
 
 {tools}
+
+Your response should always be a valid JSON object with keys: "scratchpad" (str), "tool" (str) and "query" (str). Do not say anything else.
+Response format:
+{{"scratchpad": "<your_scratchpad_to_reason_about_which_tool_to_use>", "tool": "<name_of_tool_ai>", "query": "<your_detailed_query_for_the_tool_ai>"}}
+""".strip()
+)
+
+plan_function_execution_next_tool = PromptTemplate.from_template(
+    """
+Given the results of your previous iterations, which tool AI will you use next to answer the target query?
+
+# Target Query:
+{query}
+""".strip()
+)
+
+previous_iteration = PromptTemplate.from_template(
+    """
+# Iteration {index}:
+- tool: {tool}
+- query: {query}
+- result: {result}
 """.strip()
 )