khoj 2.0.0b13.dev19__py3-none-any.whl → 2.0.0b14__py3-none-any.whl

khoj/processor/conversation/prompts.py

@@ -4,21 +4,27 @@ from langchain_core.prompts import PromptTemplate
 ## --
 personality = PromptTemplate.from_template(
     """
-You are Khoj, a smart, inquisitive and helpful personal assistant.
+You are Khoj, a smart, curious, empathetic and helpful personal assistant.
 Use your general knowledge and past conversation with the user as context to inform your responses.
-You were created by Khoj Inc. with the following capabilities:
 
-- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
-- Users can share files and other information with you using the Khoj Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
-- You *CAN* generate images, look-up real-time information from the internet, set reminders and answer questions based on the user's notes.
+You were created by Khoj Inc. More information about you, the company or Khoj apps can be found at https://khoj.dev.
+
+Today is {day_of_week}, {current_date} in UTC.
+
+# Capabilities
+- Users can share files and other information with you using the Khoj Web, Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
+- You can look up information from the user's notes and documents synced via the Khoj apps.
+- You can generate images, look-up real-time information from the internet, analyze data and answer questions based on the user's notes.
+
+# Style
+- Your responses should be helpful, conversational and tuned to the user's communication style.
 - Make sure to use the specific LaTeX math mode delimiters for your response. LaTex math mode specific delimiters as following
     - inline math mode : \\( and \\)
     - display math mode: insert linebreak after opening $$, \\[ and before closing $$, \\]
-- Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay".
-- Provide inline references to quotes from the user's notes or any web pages you refer to in your responses in markdown format. For example, "The farmer had ten sheep. [1](https://example.com)". *ALWAYS CITE YOUR SOURCES AND PROVIDE REFERENCES*. Add them inline to directly support your claim.
-
-Note: More information about you, the company or Khoj apps can be found at https://khoj.dev.
-Today is {day_of_week}, {current_date} in UTC.
+- Provide inline citations to documents and websites referenced. Add them inline in markdown format to directly support your claim.
+  For example: "The weather today is sunny [1](https://weather.com)."
+- Mention generated assets like images by reference, e.g ![chart](/visualization/image.png). Do not manually output raw, b64 encoded bytes in your response.
+- Do not respond with raw programs or scripts in your final response unless you know the user is a programmer or has explicitly requested code.
 """.strip()
 )
 
@@ -26,18 +32,23 @@ custom_personality = PromptTemplate.from_template(
     """
 You are {name}, a personal agent on Khoj.
 Use your general knowledge and past conversation with the user as context to inform your responses.
-You were created by Khoj Inc. with the following capabilities:
 
-- You *CAN REMEMBER ALL NOTES and PERSONAL INFORMATION FOREVER* that the user ever shares with you.
-- Users can share files and other information with you using the Khoj Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
+You were created on the Khoj platform. More information about you, the company or Khoj apps can be found at https://khoj.dev.
+
+Today is {day_of_week}, {current_date} in UTC.
+
+# Base Capabilities
+- Users can share files and other information with you using the Khoj Web, Desktop, Obsidian or Emacs app. They can also drag and drop their files into the chat window.
+
+# Style
 - Make sure to use the specific LaTeX math mode delimiters for your response. LaTex math mode specific delimiters as following
     - inline math mode : `\\(` and `\\)`
     - display math mode: insert linebreak after opening `$$`, `\\[` and before closing `$$`, `\\]`
-- Sometimes the user will share personal information that needs to be remembered, like an account ID or a residential address. These can be acknowledged with a simple "Got it" or "Okay".
-
-Today is {day_of_week}, {current_date} in UTC.
+- Provide inline citations to documents and websites referenced. Add them inline in markdown format to directly support your claim.
+  For example: "The weather today is sunny [1](https://weather.com)."
+- Mention generated assets like images by reference, e.g ![chart](/visualization/image.png). Do not manually output raw, b64 encoded bytes in your response.
 
-Instructions:\n{bio}
+# Instructions:\n{bio}
 """.strip()
 )
 
@@ -519,12 +530,13 @@ Q: {query}
 
 extract_questions_system_prompt = PromptTemplate.from_template(
     """
-You are Khoj, an extremely smart and helpful document search assistant with only the ability to retrieve information from the user's notes.
-Construct search queries to retrieve relevant information to answer the user's question.
+You are Khoj, an extremely smart and helpful document search assistant with only the ability to use natural language semantic search to retrieve information from the user's notes.
+Construct upto {max_queries} search queries to retrieve relevant information to answer the user's question.
 - You will be provided past questions(User), search queries(Assistant) and answers(A) for context.
-- Add as much context from the previous questions and answers as required into your search queries.
-- Break your search down into multiple search queries from a diverse set of lenses to retrieve all related documents.
-- Add date filters to your search queries from questions and answers when required to retrieve the relevant information.
+- You can use context from previous questions and answers to improve your search queries.
+- Break down your search into multiple search queries from a diverse set of lenses to retrieve all related documents. E.g who, what, where, when, why, how.
+- Add date filters to your search queries when required to retrieve the relevant information. This is the only structured query filter you can use.
+- Output 1 concept per query. Do not use boolean operators (OR/AND) to combine queries. They do not work and degrade search quality.
 - When asked a meta, vague or random questions, search for a variety of broad topics to answer the user's question.
 {personality_context}
 What searches will you perform to answer the users question? Respond with a JSON object with the key "queries" mapping to a list of searches you would perform on the user's knowledge base. Just return the queries and nothing else.
@@ -535,22 +547,27 @@ User's Location: {location}
 
 Here are some examples of how you can construct search queries to answer the user's question:
 
+Illustrate - Using diverse perspectives to retrieve all relevant documents
 User: How was my trip to Cambodia?
 Assistant: {{"queries": ["How was my trip to Cambodia?", "Angkor Wat temple visit", "Flight to Phnom Penh", "Expenses in Cambodia", "Stay in Cambodia"]}}
 A: The trip was amazing. You went to the Angkor Wat temple and it was beautiful.
 
+Illustrate - Combining date filters with natural language queries to retrieve documents in relevant date range
 User: What national parks did I go to last year?
 Assistant: {{"queries": ["National park I visited in {last_new_year} dt>='{last_new_year_date}' dt<'{current_new_year_date}'"]}}
 A: You visited the Grand Canyon and Yellowstone National Park in {last_new_year}.
 
+Illustrate - Using broad topics to answer meta or vague questions
 User: How can you help me?
 Assistant: {{"queries": ["Social relationships", "Physical and mental health", "Education and career", "Personal life goals and habits"]}}
 A: I can help you live healthier and happier across work and personal life
 
+Illustrate - Combining location and date in natural language queries with date filters to retrieve relevant documents
 User: Who all did I meet here yesterday?
 Assistant: {{"queries": ["Met in {location} on {yesterday_date} dt>='{yesterday_date}' dt<'{current_date}'"]}}
 A: Yesterday's note mentions your visit to your local beach with Ram and Shyam.
 
+Illustrate - Combining broad, diverse topics with date filters to answer meta or vague questions
 User: Share some random, interesting experiences from this month
 Assistant: {{"queries": ["Exciting travel adventures from {current_month}", "Fun social events dt>='{current_month}-01' dt<'{current_date}'", "Intense emotional experiences in {current_month}"]}}
 A: You had a great time at the local beach with your friends, attended a music concert and had a deep conversation with your friend, Khalid.
@@ -635,16 +652,17 @@ Here's some additional context about you:
 
 plan_function_execution = PromptTemplate.from_template(
     """
-You are Khoj, a smart, creative and meticulous researcher. Use the provided tool AIs to accomplish the task assigned to you.
+You are Khoj, a smart, creative and meticulous researcher.
 Create a multi-step plan and intelligently iterate on the plan to complete the task.
+Use the help of the provided tool AIs to accomplish the task assigned to you.
 {personality_context}
 
 # Instructions
-- Provide highly diverse, detailed requests to the tool AIs, one tool AI at a time, to gather information, perform actions etc. Their response will be shown to you in the next iteration.
-- Break down your research process into independent, self-contained steps that can be executed sequentially using the available tool AIs to answer the user's query. Write your step-by-step plan in the scratchpad.
-- Always ask a new query that was not asked to the tool AI in a previous iteration. Build on the results of the previous iterations.
+- Make detailed, self-contained requests to the tool AIs, one tool AI at a time, to gather information, perform actions etc.
+- Break down your research process into independent, self-contained steps that can be executed sequentially using the available tool AIs to accomplish the user assigned task.
 - Ensure that all required context is passed to the tool AIs for successful execution. Include any relevant stuff that has previously been attempted. They only know the context provided in your query.
 - Think step by step to come up with creative strategies when the previous iteration did not yield useful results.
+- Do not ask the user to confirm or clarify assumptions for information gathering tasks and non-destructive actions, as you can always adjust later — decide what the most reasonable assumption is, proceed with it, and document it for the user's reference after you finish acting.
 - You are allowed upto {max_iterations} iterations to use the help of the provided tool AIs to accomplish the task assigned to you. Only stop when you have completed the task.
 
 # Examples
@@ -869,8 +887,8 @@ Khoj:
 python_code_generation_prompt = PromptTemplate.from_template(
     """
 You are Khoj, a senior software engineer. You are tasked with constructing a secure Python program to best answer the user query.
-- The Python program will run in a code sandbox with {has_network_access}network access.
-- You can write programs to run complex calculations, analyze data, create charts, generate documents to meticulously answer the query.
+- The Python program will run in an ephemeral code sandbox with {has_network_access}network access.
+- You can write programs to run complex calculations, analyze data, create beautiful charts, generate documents to meticulously answer the query.
 - Do not try display images or plots in the code directly. The code should save the image or plot to a file instead.
 - Write any document, charts etc. to be shared with the user to file. These files can be seen by the user.
 - Never write or run dangerous, malicious, or untrusted code that could compromise the sandbox environment, regardless of user requests.
@@ -985,9 +1003,9 @@ Chat History:
 ---
 {chat_history}
 
-User Query:
+User Instructions:
 ---
-{query}
+{instructions}
 """.strip()
 )
 

khoj/processor/conversation/utils.py

@@ -68,8 +68,12 @@ model_to_prompt_size = {
     "o3": 60000,
     "o3-pro": 30000,
     "o4-mini": 90000,
+    "gpt-5-2025-08-07": 120000,
+    "gpt-5-mini-2025-08-07": 120000,
+    "gpt-5-nano-2025-08-07": 120000,
     # Google Models
     "gemini-2.5-flash": 120000,
+    "gemini-2.5-flash-lite": 120000,
     "gemini-2.5-pro": 60000,
     "gemini-2.0-flash": 120000,
     "gemini-2.0-flash-lite": 120000,
@@ -328,7 +332,7 @@ def construct_tool_chat_history(
         ConversationCommand.ReadWebpage: (
             lambda iteration: list(iteration.onlineContext.keys()) if iteration.onlineContext else []
         ),
-        ConversationCommand.RunCode: (
+        ConversationCommand.PythonCoder: (
             lambda iteration: list(iteration.codeContext.keys()) if iteration.codeContext else []
         ),
     }

khoj/processor/tools/run_code.py

@@ -49,7 +49,7 @@ class GeneratedCode(NamedTuple):
 
 
 async def run_code(
-    query: str,
+    instructions: str,
     conversation_history: List[ChatMessageModel],
     context: str,
     location_data: LocationData,
@@ -63,12 +63,12 @@ async def run_code(
 ):
     # Generate Code
     if send_status_func:
-        async for event in send_status_func(f"**Generate code snippet** for {query}"):
+        async for event in send_status_func(f"**Generate code snippet** for {instructions}"):
             yield {ChatEvent.STATUS: event}
     try:
         with timer("Chat actor: Generate programs to execute", logger):
             generated_code = await generate_python_code(
-                query,
+                instructions,
                 conversation_history,
                 context,
                 location_data,
@@ -79,7 +79,7 @@ async def run_code(
                 query_files,
             )
     except Exception as e:
-        raise ValueError(f"Failed to generate code for {query} with error: {e}")
+        raise ValueError(f"Failed to generate code for {instructions} with error: {e}")
 
     # Prepare Input Data
     input_data = []
@@ -101,21 +101,21 @@ async def run_code(
             code = result.pop("code")
             cleaned_result = truncate_code_context({"cleaned": {"results": result}})["cleaned"]["results"]
             logger.info(f"Executed Code\n----\n{code}\n----\nResult\n----\n{cleaned_result}\n----")
-            yield {query: {"code": code, "results": result}}
+            yield {instructions: {"code": code, "results": result}}
     except asyncio.TimeoutError as e:
         # Call the sandbox_url/stop GET API endpoint to stop the code sandbox
-        error = f"Failed to run code for {query} with Timeout error: {e}"
+        error = f"Failed to run code for {instructions} with Timeout error: {e}"
         try:
             await aiohttp.ClientSession().get(f"{sandbox_url}/stop", timeout=5)
         except Exception as e:
             error += f"\n\nFailed to stop code sandbox with error: {e}"
         raise ValueError(error)
     except Exception as e:
-        raise ValueError(f"Failed to run code for {query} with error: {e}")
+        raise ValueError(f"Failed to run code for {instructions} with error: {e}")
 
 
 async def generate_python_code(
-    q: str,
+    instructions: str,
     chat_history: List[ChatMessageModel],
     context: str,
     location_data: LocationData,
@@ -142,7 +142,7 @@ async def generate_python_code(
     network_access_context = "**NO** " if not is_e2b_code_sandbox_enabled() else ""
 
     code_generation_prompt = prompts.python_code_generation_prompt.format(
-        query=q,
+        instructions=instructions,
         chat_history=chat_history_str,
         context=context,
         has_network_access=network_access_context,
@@ -252,8 +252,12 @@ async def execute_e2b(code: str, input_files: list[dict]) -> dict[str, Any]:
 
         # Identify new files created during execution
         new_files = set(E2bFile(f.name, f.path) for f in await sandbox.files.list("~")) - original_files
+
         # Read newly created files in parallel
-        download_tasks = [sandbox.files.read(f.path, request_timeout=30) for f in new_files]
+        def read_format(f):
+            return "bytes" if Path(f.name).suffix in image_file_ext else "text"
+
+        download_tasks = [sandbox.files.read(f.path, format=read_format(f), request_timeout=30) for f in new_files]
         downloaded_files = await asyncio.gather(*download_tasks)
         for f, content in zip(new_files, downloaded_files):
             if isinstance(content, bytes):
@@ -261,23 +265,12 @@ async def execute_e2b(code: str, input_files: list[dict]) -> dict[str, Any]:
                 b64_data = base64.b64encode(content).decode("utf-8")
             elif Path(f.name).suffix in image_file_ext:
                 # Ignore image files as they are extracted from execution results below for inline display
-                continue
+                b64_data = base64.b64encode(content).decode("utf-8")
             else:
                 # Text files - encode utf-8 string as base64
                 b64_data = content
             output_files.append({"filename": f.name, "b64_data": b64_data})
 
-        # Collect output files from execution results
-        # Repect ordering of output result types to disregard text output associated with images
-        output_result_types = ["png", "jpeg", "svg", "text", "markdown", "json"]
-        for idx, result in enumerate(execution.results):
-            if getattr(result, "chart", None):
-                continue
-            for result_type in output_result_types:
-                if b64_data := getattr(result, result_type, None):
-                    output_files.append({"filename": f"{idx}.{result_type}", "b64_data": b64_data})
-                    break
-
         # collect logs
         success = not execution.error and not execution.logs.stderr
         stdout = "\n".join(execution.logs.stdout)

khoj/routers/api_chat.py

@@ -786,6 +786,9 @@ async def event_generator(
                     if interrupt_query == ChatEvent.END_EVENT.value:
                         cancellation_event.set()
                         logger.debug(f"Chat cancelled by user {user} via interrupt queue.")
+                    elif interrupt_query == ChatEvent.INTERRUPT.value:
+                        cancellation_event.set()
+                        logger.debug("Chat interrupted.")
                     else:
                         # Pass the interrupt query to child tasks
                         logger.info(f"Continuing chat with the new instruction: {interrupt_query}")
@@ -995,7 +998,7 @@ async def event_generator(
             )
         except ValueError as e:
             logger.error(f"Error getting data sources and output format: {e}. Falling back to default.")
-            conversation_commands = [ConversationCommand.General]
+            chosen_io = {"sources": [ConversationCommand.General], "output": ConversationCommand.Text}
 
         conversation_commands = chosen_io.get("sources") + [chosen_io.get("output")]
 
@@ -1523,6 +1526,8 @@ async def chat_ws(
                         ack_type = "interrupt_acknowledged"
                         await websocket.send_text(json.dumps({"type": ack_type}))
                 else:
+                    ack_type = "interrupt_acknowledged"
+                    await websocket.send_text(json.dumps({"type": ack_type}))
                     logger.info(f"No ongoing task to interrupt for user {websocket.scope['user'].object.id}")
                 continue
 
@@ -1556,7 +1561,7 @@ async def chat_ws(
     except WebSocketDisconnect:
         logger.info(f"WebSocket disconnected for user {websocket.scope['user'].object.id}")
         if current_task and not current_task.done():
-            current_task.cancel()
+            interrupt_queue.put_nowait(ChatEvent.INTERRUPT.value)
     except Exception as e:
         logger.error(f"Error in websocket chat: {e}", exc_info=True)
         if current_task and not current_task.done():
@@ -1701,8 +1706,8 @@ async def process_chat_request(
         logger.debug(f"Chat request cancelled for user {websocket.scope['user'].object.id}")
         raise
     except Exception as e:
-        logger.error(f"Error processing chat request: {e}", exc_info=True)
         await websocket.send_text(json.dumps({"error": "Internal server error"}))
+        logger.error(f"Error processing chat request: {e}", exc_info=True)
         raise
 
 

khoj/routers/api_content.py

@@ -570,7 +570,7 @@ async def indexer(
                     file_data.content.decode(file_data.encoding) if file_data.encoding else file_data.content
                 )
             else:
-                logger.warning(f"Skipped indexing unsupported file type sent by {client} client: {file_data.name}")
+                logger.debug(f"Skipped indexing unsupported file type sent by {client} client: {file_data.name}")
 
         indexer_input = IndexerInput(
             org=index_files["org"],

khoj/routers/helpers.py

@@ -1264,6 +1264,7 @@ async def extract_questions(
     location_data: LocationData = None,
     query_images: Optional[List[str]] = None,
     query_files: str = None,
+    max_queries: int = 5,
     tracer: dict = {},
 ):
     """
@@ -1293,14 +1294,20 @@ async def extract_questions(
         location=location,
         username=username,
         personality_context=personality_context,
+        max_queries=max_queries,
     )
 
     prompt = prompts.extract_questions_user_message.format(text=query, chat_history=chat_history_str)
 
     class DocumentQueries(BaseModel):
-        """Choose searches to run on user documents."""
+        """Choose semantic search queries to run on user documents."""
 
-        queries: List[str] = Field(..., min_items=1, description="List of search queries to run on user documents.")
+        queries: List[str] = Field(
+            ...,
+            min_length=1,
+            max_length=max_queries,
+            description="List of semantic search queries to run on user documents.",
+        )
 
     raw_response = await send_message_to_model_wrapper(
         system_message=system_prompt,
@@ -1618,6 +1625,7 @@ async def agenerate_chat_response(
             deepthought = True
 
         chat_model = await ConversationAdapters.aget_valid_chat_model(user, conversation, is_subscribed)
+        max_prompt_size = await ConversationAdapters.aget_max_context_size(chat_model, user)
         vision_available = chat_model.vision_enabled
         if not vision_available and query_images:
             vision_enabled_config = await ConversationAdapters.aget_vision_enabled_config()
@@ -1649,7 +1657,7 @@ async def agenerate_chat_response(
                 model=chat_model_name,
                 api_key=api_key,
                 api_base_url=openai_chat_config.api_base_url,
-                max_prompt_size=chat_model.max_prompt_size,
+                max_prompt_size=max_prompt_size,
                 tokenizer_name=chat_model.tokenizer,
                 agent=agent,
                 vision_available=vision_available,
@@ -1680,7 +1688,7 @@ async def agenerate_chat_response(
                 model=chat_model.name,
                 api_key=api_key,
                 api_base_url=api_base_url,
-                max_prompt_size=chat_model.max_prompt_size,
+                max_prompt_size=max_prompt_size,
                 tokenizer_name=chat_model.tokenizer,
                 agent=agent,
                 vision_available=vision_available,
@@ -1710,7 +1718,7 @@ async def agenerate_chat_response(
                 model=chat_model.name,
                 api_key=api_key,
                 api_base_url=api_base_url,
-                max_prompt_size=chat_model.max_prompt_size,
+                max_prompt_size=max_prompt_size,
                 tokenizer_name=chat_model.tokenizer,
                 agent=agent,
                 vision_available=vision_available,
@@ -2731,7 +2739,9 @@ def configure_content(
 
     try:
         # Initialize Org Notes Search
-        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files["org"]:
+        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Org.value) and files.get(
+            "org"
+        ):
             logger.info("🦄 Setting up search for orgmode notes")
             # Extract Entries, Generate Notes Embeddings
             text_search.setup(
@@ -2746,9 +2756,9 @@ def configure_content(
 
     try:
         # Initialize Markdown Search
-        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files[
+        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Markdown.value) and files.get(
             "markdown"
-        ]:
+        ):
             logger.info("💎 Setting up search for markdown notes")
             # Extract Entries, Generate Markdown Embeddings
             text_search.setup(
@@ -2764,7 +2774,9 @@ def configure_content(
 
     try:
         # Initialize PDF Search
-        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files["pdf"]:
+        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Pdf.value) and files.get(
+            "pdf"
+        ):
             logger.info("🖨️ Setting up search for pdf")
             # Extract Entries, Generate PDF Embeddings
             text_search.setup(
@@ -2780,9 +2792,9 @@ def configure_content(
 
     try:
         # Initialize Plaintext Search
-        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files[
+        if (search_type == state.SearchType.All.value or search_type == state.SearchType.Plaintext.value) and files.get(
             "plaintext"
-        ]:
+        ):
             logger.info("📄 Setting up search for plaintext")
             # Extract Entries, Generate Plaintext Embeddings
             text_search.setup(
@@ -2908,35 +2920,34 @@ async def view_file_content(
         raw_text = file_object.raw_text
 
         # Apply line range filtering if specified
-        if start_line is None and end_line is None:
-            filtered_text = raw_text
-        else:
-            lines = raw_text.split("\n")
-            start_line = start_line or 1
-            end_line = end_line or len(lines)
-
-            # Validate line range
-            if start_line < 1 or end_line < 1 or start_line > end_line:
-                error_msg = f"Invalid line range: {start_line}-{end_line}"
-                logger.warning(error_msg)
-                yield [{"query": query, "file": path, "compiled": error_msg}]
-                return
-            if start_line > len(lines):
-                error_msg = f"Start line {start_line} exceeds total number of lines {len(lines)}"
-                logger.warning(error_msg)
-                yield [{"query": query, "file": path, "compiled": error_msg}]
-                return
+        lines = raw_text.split("\n")
+        start_line = start_line or 1
+        end_line = end_line or len(lines)
 
-            # Convert from 1-based to 0-based indexing and ensure bounds
-            start_idx = max(0, start_line - 1)
-            end_idx = min(len(lines), end_line)
+        # Validate line range
+        if start_line < 1 or end_line < 1 or start_line > end_line:
+            error_msg = f"Invalid line range: {start_line}-{end_line}"
+            logger.warning(error_msg)
+            yield [{"query": query, "file": path, "compiled": error_msg}]
+            return
+        if start_line > len(lines):
+            error_msg = f"Start line {start_line} exceeds total number of lines {len(lines)}"
+            logger.warning(error_msg)
+            yield [{"query": query, "file": path, "compiled": error_msg}]
+            return
+
+        # Convert from 1-based to 0-based indexing and ensure bounds
+        start_idx = max(0, start_line - 1)
+        end_idx = min(len(lines), end_line)
 
-            selected_lines = lines[start_idx:end_idx]
-            filtered_text = "\n".join(selected_lines)
+        # Limit to first 50 lines if more than 50 lines are requested
+        truncation_message = ""
+        if end_idx - start_idx > 50:
+            truncation_message = "\n\n[Truncated after 50 lines! Use narrower line range to view complete section.]"
+            end_idx = start_idx + 50
 
-        # Truncate the text if it's too long
-        if len(filtered_text) > 10000:
-            filtered_text = filtered_text[:10000] + "\n\n[Truncated. Use line numbers to view specific sections.]"
+        selected_lines = lines[start_idx:end_idx]
+        filtered_text = "\n".join(selected_lines) + truncation_message
 
         # Format the result as a document reference
         document_results = [
@@ -2995,7 +3006,7 @@ async def grep_files(
     lines_after = lines_after or 0
 
     try:
-        regex = re.compile(regex_pattern, re.IGNORECASE)
+        regex = re.compile(regex_pattern, re.IGNORECASE | re.MULTILINE)
     except re.error as e:
         yield {
             "query": _generate_query(0, 0, path_prefix, regex_pattern, lines_before, lines_after),
@@ -3005,9 +3016,17 @@ async def grep_files(
         return
 
     try:
-        file_matches = await FileObjectAdapters.aget_file_objects_by_regex(user, regex_pattern, path_prefix)
+        # Make db pushdown filters more permissive by removing line anchors
+        # The precise line-anchored matching will be done in Python stage
+        db_pattern = regex_pattern
+        db_pattern = re.sub(r"\(\?\w*\)", "", db_pattern)  # Remove inline flags like (?i), (?m), (?im)
+        db_pattern = re.sub(r"^\^", "", db_pattern)  # Remove ^ at regex pattern start
+        db_pattern = re.sub(r"\$$", "", db_pattern)  # Remove $ at regex pattern end
+
+        file_matches = await FileObjectAdapters.aget_file_objects_by_regex(user, db_pattern, path_prefix)
 
         line_matches = []
+        line_matches_count = 0
         for file_object in file_matches:
             lines = file_object.raw_text.split("\n")
             matched_line_numbers = []
@@ -3016,6 +3035,7 @@ async def grep_files(
             for i, line in enumerate(lines, 1):
                 if regex.search(line):
                     matched_line_numbers.append(i)
+            line_matches_count += len(matched_line_numbers)
 
             # Build context for each match
             for line_num in matched_line_numbers:
@@ -3032,10 +3052,10 @@ async def grep_files(
 
                     if current_line_num == line_num:
                         # This is the matching line, mark it
-                        context_lines.append(f"{file_object.file_name}:{current_line_num}:> {line_content}")
+                        context_lines.append(f"{file_object.file_name}:{current_line_num}: {line_content}")
                     else:
                         # This is a context line
-                        context_lines.append(f"{file_object.file_name}:{current_line_num}:  {line_content}")
+                        context_lines.append(f"{file_object.file_name}-{current_line_num}-  {line_content}")
 
                 # Add separator between matches if showing context
                 if lines_before > 0 or lines_after > 0:
@@ -3050,7 +3070,7 @@ async def grep_files(
         # Check if no results found
         max_results = 1000
         query = _generate_query(
-            len([m for m in line_matches if ":>" in m]),
+            line_matches_count,
             len(file_matches),
             path_prefix,
             regex_pattern,

khoj/routers/research.py

@@ -100,7 +100,7 @@ async def apick_next_tool(
         ConversationCommand.Notes.value: [tool.value for tool in document_research_tools],
         ConversationCommand.Webpage.value: [ConversationCommand.ReadWebpage.value],
         ConversationCommand.Online.value: [ConversationCommand.SearchWeb.value],
-        ConversationCommand.Code.value: [ConversationCommand.RunCode.value],
+        ConversationCommand.Code.value: [ConversationCommand.PythonCoder.value],
         ConversationCommand.Operator.value: [ConversationCommand.OperateComputer.value],
     }
     for input_tool, research_tools in input_tools_to_research_tools.items():
@@ -197,7 +197,7 @@ async def apick_next_tool(
         if i.warning is None and isinstance(i.query, ToolCall)
     }
     if (parsed_response.name, dict_to_tuple(parsed_response.args)) in previous_tool_query_combinations:
-        warning = "Repeated tool, query combination detected. Skipping iteration. Try something different."
+        warning = f"Repeated tool, query combination detected. You've already called {parsed_response.name} with args: {parsed_response.args}. Try something different."
     # Only send client status updates if we'll execute this iteration and model has thoughts to share.
     elif send_status_func and not is_none_or_empty(response.thought):
         async for event in send_status_func(response.thought):
@@ -412,11 +412,13 @@ async def research(
                 this_iteration.warning = f"Error reading webpages: {e}"
                 logger.error(this_iteration.warning, exc_info=True)
 
-        elif this_iteration.query.name == ConversationCommand.RunCode:
+        elif this_iteration.query.name == ConversationCommand.PythonCoder:
             try:
                 async for result in run_code(
                     **this_iteration.query.args,
-                    conversation_history=construct_tool_chat_history(previous_iterations, ConversationCommand.RunCode),
+                    conversation_history=construct_tool_chat_history(
+                        previous_iterations, ConversationCommand.PythonCoder
+                    ),
                     context="",
                     location_data=location,
                     user=user,
@@ -433,7 +435,7 @@ async def research(
                         this_iteration.codeContext = code_results
                 async for result in send_status_func(f"**Ran code snippets**: {len(this_iteration.codeContext)}"):
                     yield result
-            except ValueError as e:
+            except (ValueError, TypeError) as e:
                 this_iteration.warning = f"Error running code: {e}"
                 logger.warning(this_iteration.warning, exc_info=True)