khoj 1.27.2.dev15__py3-none-any.whl → 1.28.0__py3-none-any.whl

khoj/routers/research.py

@@ -0,0 +1,320 @@
+import json
+import logging
+from datetime import datetime
+from typing import Callable, Dict, List, Optional
+
+import yaml
+from fastapi import Request
+
+from khoj.database.models import Agent, KhojUser
+from khoj.processor.conversation import prompts
+from khoj.processor.conversation.utils import (
+    InformationCollectionIteration,
+    clean_json,
+    construct_iteration_history,
+    construct_tool_chat_history,
+)
+from khoj.processor.tools.online_search import read_webpages, search_online
+from khoj.processor.tools.run_code import run_code
+from khoj.routers.api import extract_references_and_questions
+from khoj.routers.helpers import (
+    ChatEvent,
+    construct_chat_history,
+    extract_relevant_info,
+    generate_summary_from_files,
+    send_message_to_model_wrapper,
+)
+from khoj.utils.helpers import (
+    ConversationCommand,
+    function_calling_description_for_llm,
+    is_none_or_empty,
+    timer,
+)
+from khoj.utils.rawconfig import LocationData
+
+logger = logging.getLogger(__name__)
+
+
+async def apick_next_tool(
+    query: str,
+    conversation_history: dict,
+    user: KhojUser = None,
+    query_images: List[str] = [],
+    location: LocationData = None,
+    user_name: str = None,
+    agent: Agent = None,
+    previous_iterations_history: str = None,
+    max_iterations: int = 5,
+    send_status_func: Optional[Callable] = None,
+    tracer: dict = {},
+):
+    """
+    Given a query, determine which of the available tools the agent should use in order to answer appropriately. One at a time, and it's able to use subsequent iterations to refine the answer.
+    """
+
+    tool_options = dict()
+    tool_options_str = ""
+
+    agent_tools = agent.input_tools if agent else []
+
+    for tool, description in function_calling_description_for_llm.items():
+        tool_options[tool.value] = description
+        if len(agent_tools) == 0 or tool.value in agent_tools:
+            tool_options_str += f'- "{tool.value}": "{description}"\n'
+
+    chat_history = construct_chat_history(conversation_history, agent_name=agent.name if agent else "Khoj")
+
+    if query_images:
+        query = f"[placeholder for user attached images]\n{query}"
+
+    personality_context = (
+        prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
+    )
+
+    # Extract Past User Message and Inferred Questions from Conversation Log
+    today = datetime.today()
+    location_data = f"{location}" if location else "Unknown"
+
+    function_planning_prompt = prompts.plan_function_execution.format(
+        tools=tool_options_str,
+        chat_history=chat_history,
+        personality_context=personality_context,
+        current_date=today.strftime("%Y-%m-%d"),
+        day_of_week=today.strftime("%A"),
+        username=user_name or "Unknown",
+        location=location_data,
+        previous_iterations=previous_iterations_history,
+        max_iterations=max_iterations,
+    )
+
+    with timer("Chat actor: Infer information sources to refer", logger):
+        response = await send_message_to_model_wrapper(
+            query=query,
+            context=function_planning_prompt,
+            response_type="json_object",
+            user=user,
+            query_images=query_images,
+            tracer=tracer,
+        )
+
+    try:
+        response = clean_json(response)
+        response = json.loads(response)
+        selected_tool = response.get("tool", None)
+        generated_query = response.get("query", None)
+        scratchpad = response.get("scratchpad", None)
+        logger.info(f"Response for determining relevant tools: {response}")
+        if send_status_func:
+            determined_tool_message = "**Determined Tool**: "
+            determined_tool_message += f"{selected_tool}({generated_query})." if selected_tool else "respond."
+            determined_tool_message += f"\nReason: {scratchpad}" if scratchpad else ""
+            async for event in send_status_func(f"{scratchpad}"):
+                yield {ChatEvent.STATUS: event}
+
+        yield InformationCollectionIteration(
+            tool=selected_tool,
+            query=generated_query,
+        )
+
+    except Exception as e:
+        logger.error(f"Invalid response for determining relevant tools: {response}. {e}", exc_info=True)
+        yield InformationCollectionIteration(
+            tool=None,
+            query=None,
+        )
+
+
+async def execute_information_collection(
+    request: Request,
+    user: KhojUser,
+    query: str,
+    conversation_id: str,
+    conversation_history: dict,
+    query_images: List[str],
+    agent: Agent = None,
+    send_status_func: Optional[Callable] = None,
+    user_name: str = None,
+    location: LocationData = None,
+    file_filters: List[str] = [],
+    tracer: dict = {},
+):
+    current_iteration = 0
+    MAX_ITERATIONS = 5
+    previous_iterations: List[InformationCollectionIteration] = []
+    while current_iteration < MAX_ITERATIONS:
+        online_results: Dict = dict()
+        code_results: Dict = dict()
+        document_results: List[Dict[str, str]] = []
+        summarize_files: str = ""
+        this_iteration = InformationCollectionIteration(tool=None, query=query)
+        previous_iterations_history = construct_iteration_history(previous_iterations, prompts.previous_iteration)
+
+        async for result in apick_next_tool(
+            query,
+            conversation_history,
+            user,
+            query_images,
+            location,
+            user_name,
+            agent,
+            previous_iterations_history,
+            MAX_ITERATIONS,
+            send_status_func,
+            tracer=tracer,
+        ):
+            if isinstance(result, dict) and ChatEvent.STATUS in result:
+                yield result[ChatEvent.STATUS]
+            elif isinstance(result, InformationCollectionIteration):
+                this_iteration = result
+
+        if this_iteration.tool == ConversationCommand.Notes:
+            this_iteration.context = []
+            document_results = []
+            async for result in extract_references_and_questions(
+                request,
+                construct_tool_chat_history(previous_iterations, ConversationCommand.Notes),
+                this_iteration.query,
+                7,
+                None,
+                conversation_id,
+                [ConversationCommand.Default],
+                location,
+                send_status_func,
+                query_images,
+                agent=agent,
+                tracer=tracer,
+            ):
+                if isinstance(result, dict) and ChatEvent.STATUS in result:
+                    yield result[ChatEvent.STATUS]
+                elif isinstance(result, tuple):
+                    document_results = result[0]
+                    this_iteration.context += document_results
+
+            if not is_none_or_empty(document_results):
+                try:
+                    distinct_files = {d["file"] for d in document_results}
+                    distinct_headings = set([d["compiled"].split("\n")[0] for d in document_results if "compiled" in d])
+                    # Strip only leading # from headings
+                    headings_str = "\n- " + "\n- ".join(distinct_headings).replace("#", "")
+                    async for result in send_status_func(
+                        f"**Found {len(distinct_headings)} Notes Across {len(distinct_files)} Files**: {headings_str}"
+                    ):
+                        yield result
+                except Exception as e:
+                    logger.error(f"Error extracting document references: {e}", exc_info=True)
+
+        elif this_iteration.tool == ConversationCommand.Online:
+            async for result in search_online(
+                this_iteration.query,
+                construct_tool_chat_history(previous_iterations, ConversationCommand.Online),
+                location,
+                user,
+                send_status_func,
+                [],
+                max_webpages_to_read=0,
+                query_images=query_images,
+                agent=agent,
+                tracer=tracer,
+            ):
+                if isinstance(result, dict) and ChatEvent.STATUS in result:
+                    yield result[ChatEvent.STATUS]
+                else:
+                    online_results: Dict[str, Dict] = result  # type: ignore
+                    this_iteration.onlineContext = online_results
+
+        elif this_iteration.tool == ConversationCommand.Webpage:
+            try:
+                async for result in read_webpages(
+                    this_iteration.query,
+                    construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
+                    location,
+                    user,
+                    send_status_func,
+                    query_images=query_images,
+                    agent=agent,
+                    tracer=tracer,
+                ):
+                    if isinstance(result, dict) and ChatEvent.STATUS in result:
+                        yield result[ChatEvent.STATUS]
+                    else:
+                        direct_web_pages: Dict[str, Dict] = result  # type: ignore
+
+                        webpages = []
+                        for web_query in direct_web_pages:
+                            if online_results.get(web_query):
+                                online_results[web_query]["webpages"] = direct_web_pages[web_query]["webpages"]
+                            else:
+                                online_results[web_query] = {"webpages": direct_web_pages[web_query]["webpages"]}
+
+                            for webpage in direct_web_pages[web_query]["webpages"]:
+                                webpages.append(webpage["link"])
+                        this_iteration.onlineContext = online_results
+            except Exception as e:
+                logger.error(f"Error reading webpages: {e}", exc_info=True)
+
+        elif this_iteration.tool == ConversationCommand.Code:
+            try:
+                async for result in run_code(
+                    this_iteration.query,
+                    construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
+                    "",
+                    location,
+                    user,
+                    send_status_func,
+                    query_images=query_images,
+                    agent=agent,
+                    tracer=tracer,
+                ):
+                    if isinstance(result, dict) and ChatEvent.STATUS in result:
+                        yield result[ChatEvent.STATUS]
+                    else:
+                        code_results: Dict[str, Dict] = result  # type: ignore
+                        this_iteration.codeContext = code_results
+                async for result in send_status_func(f"**Ran code snippets**: {len(this_iteration.codeContext)}"):
+                    yield result
+            except ValueError as e:
+                logger.warning(
+                    f"Failed to use code tool: {e}. Attempting to respond without code results",
+                    exc_info=True,
+                )
+
+        elif this_iteration.tool == ConversationCommand.Summarize:
+            try:
+                async for result in generate_summary_from_files(
+                    this_iteration.query,
+                    user,
+                    file_filters,
+                    construct_tool_chat_history(previous_iterations),
+                    query_images=query_images,
+                    agent=agent,
+                    send_status_func=send_status_func,
+                ):
+                    if isinstance(result, dict) and ChatEvent.STATUS in result:
+                        yield result[ChatEvent.STATUS]
+                    else:
+                        summarize_files = result  # type: ignore
+            except Exception as e:
+                logger.error(f"Error generating summary: {e}", exc_info=True)
+
+        else:
+            # No valid tools. This is our exit condition.
+            current_iteration = MAX_ITERATIONS
+
+        current_iteration += 1
+
+        if document_results or online_results or code_results or summarize_files:
+            results_data = f"**Results**:\n"
+            if document_results:
+                results_data += f"**Document References**:\n{yaml.dump(document_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n"
+            if online_results:
+                results_data += f"**Online Results**:\n{yaml.dump(online_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n"
+            if code_results:
+                results_data += f"**Code Results**:\n{yaml.dump(code_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n"
+            if summarize_files:
+                results_data += f"**Summarized Files**:\n{yaml.dump(summarize_files, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n"
+
+            # intermediate_result = await extract_relevant_info(this_iteration.query, results_data, agent)
+            this_iteration.summarizedResult = results_data
+
+        previous_iterations.append(this_iteration)
+        yield this_iteration

khoj/search_filter/date_filter.py

@@ -7,8 +7,6 @@ from math import inf
 from typing import List, Tuple
 
 import dateparser as dtparse
-from dateparser.search import search_dates
-from dateparser_data.settings import default_parsers
 from dateutil.relativedelta import relativedelta
 
 from khoj.search_filter.base_filter import BaseFilter
@@ -23,7 +21,7 @@ class DateFilter(BaseFilter):
     # - dt>="yesterday" dt<"tomorrow"
     # - dt>="last week"
     # - dt:"2 years ago"
-    date_regex = r"dt([:><=]{1,2})[\"'](.*?)[\"']"
+    date_regex = r"dt([:><=]{1,2})[\"'‘’](.*?)[\"'‘’]"
 
     def __init__(self, entry_key="compiled"):
         self.entry_key = entry_key

khoj/search_filter/file_filter.py

@@ -1,11 +1,10 @@
-import fnmatch
 import logging
 import re
 from collections import defaultdict
 from typing import List
 
 from khoj.search_filter.base_filter import BaseFilter
-from khoj.utils.helpers import LRU, timer
+from khoj.utils.helpers import LRU
 
 logger = logging.getLogger(__name__)
 

khoj/search_type/text_search.py

@@ -102,8 +102,8 @@ def load_embeddings(
 
 
 async def query(
-    user: KhojUser,
     raw_query: str,
+    user: KhojUser,
     type: SearchType = SearchType.All,
     question_embedding: Union[torch.Tensor, None] = None,
     max_distance: float = None,
@@ -130,12 +130,12 @@ async def query(
     top_k = 10
     with timer("Search Time", logger, state.device):
         hits = EntryAdapters.search_with_embeddings(
-            user=user,
+            raw_query=raw_query,
             embeddings=question_embedding,
             max_results=top_k,
             file_type_filter=file_type,
-            raw_query=raw_query,
             max_distance=max_distance,
+            user=user,
             agent=agent,
         ).all()
         hits = await sync_to_async(list)(hits)  # type: ignore[call-arg]

khoj/utils/helpers.py

@@ -101,6 +101,15 @@ def merge_dicts(priority_dict: dict, default_dict: dict):
     return merged_dict
 
 
+def fix_json_dict(json_dict: dict) -> dict:
+    for k, v in json_dict.items():
+        if v == "True" or v == "False":
+            json_dict[k] = v == "True"
+        if isinstance(v, dict):
+            json_dict[k] = fix_json_dict(v)
+    return json_dict
+
+
 def get_file_type(file_type: str, file_content: bytes) -> tuple[str, str]:
     "Get file type from file mime type"
 
@@ -313,12 +322,14 @@ class ConversationCommand(str, Enum):
     Help = "help"
     Online = "online"
     Webpage = "webpage"
+    Code = "code"
     Image = "image"
     Text = "text"
     Automation = "automation"
     AutomatedTask = "automated_task"
     Summarize = "summarize"
     Diagram = "diagram"
+    Research = "research"
 
 
 command_descriptions = {
@@ -327,11 +338,13 @@ command_descriptions = {
     ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
     ConversationCommand.Online: "Search for information on the internet.",
     ConversationCommand.Webpage: "Get information from webpage suggested by you.",
+    ConversationCommand.Code: "Run Python code to parse information, run complex calculations, create documents and charts.",
     ConversationCommand.Image: "Generate illustrative, creative images by describing your imagination in words.",
     ConversationCommand.Automation: "Automatically run your query at a specified time or interval.",
     ConversationCommand.Help: "Get help with how to use or setup Khoj from the documentation",
     ConversationCommand.Summarize: "Get help with a question pertaining to an entire document.",
     ConversationCommand.Diagram: "Draw a flowchart, diagram, or any other visual representation best expressed with primitives like lines, rectangles, and text.",
+    ConversationCommand.Research: "Do deep research on a topic. This will take longer than usual, but give a more detailed, comprehensive answer.",
 }
 
 command_descriptions_for_agent = {
@@ -340,6 +353,7 @@ command_descriptions_for_agent = {
     ConversationCommand.Online: "Agent can search the internet for information.",
     ConversationCommand.Webpage: "Agent can read suggested web pages for information.",
     ConversationCommand.Summarize: "Agent can read an entire document. Agents knowledge base must be a single document.",
+    ConversationCommand.Research: "Agent can do deep research on a topic.",
 }
 
 tool_descriptions_for_llm = {
@@ -348,18 +362,26 @@ tool_descriptions_for_llm = {
     ConversationCommand.Notes: "To search the user's personal knowledge base. Especially helpful if the question expects context from the user's notes or documents.",
     ConversationCommand.Online: "To search for the latest, up-to-date information from the internet. Note: **Questions about Khoj should always use this data source**",
     ConversationCommand.Webpage: "To use if the user has directly provided the webpage urls or you are certain of the webpage urls to read.",
+    ConversationCommand.Code: "To run Python code in a Pyodide sandbox with no network access. Helpful when need to parse information, run complex calculations, create documents and charts for user. Matplotlib, bs4, pandas, numpy, etc. are available.",
     ConversationCommand.Summarize: "To retrieve an answer that depends on the entire document or a large text.",
 }
 
+function_calling_description_for_llm = {
+    ConversationCommand.Notes: "To search the user's personal knowledge base. Especially helpful if the question expects context from the user's notes or documents.",
+    ConversationCommand.Online: "To search the internet for information. Useful to get a quick, broad overview from the internet. Provide all relevant context to ensure new searches, not in previous iterations, are performed.",
+    ConversationCommand.Webpage: "To extract information from webpages. Useful for more detailed research from the internet. Usually used when you know the webpage links to refer to. Share the webpage links and information to extract in your query.",
+    ConversationCommand.Code: "To run Python code in a Pyodide sandbox with no network access. Helpful when need to parse information, run complex calculations, create charts for user. Matplotlib, bs4, pandas, numpy, etc. are available.",
+}
+
 mode_descriptions_for_llm = {
-    ConversationCommand.Image: "Use this if you are confident the user is requesting you to create a new picture based on their description.",
+    ConversationCommand.Image: "Use this if you are confident the user is requesting you to create a new picture based on their description. This does not support generating charts or graphs.",
     ConversationCommand.Automation: "Use this if you are confident the user is requesting a response at a scheduled date, time and frequency",
     ConversationCommand.Text: "Use this if a normal text response would be sufficient for accurately responding to the query.",
     ConversationCommand.Diagram: "Use this if the user is requesting a diagram or visual representation that requires primitives like lines, rectangles, and text.",
 }
 
 mode_descriptions_for_agent = {
-    ConversationCommand.Image: "Agent can generate image in response.",
+    ConversationCommand.Image: "Agent can generate images in response. It cannot not use this to generate charts and graphs.",
     ConversationCommand.Automation: "Agent can schedule a task to run at a scheduled date, time and frequency in response.",
     ConversationCommand.Text: "Agent can generate text in response.",
     ConversationCommand.Diagram: "Agent can generate a visual representation that requires primitives like lines, rectangles, and text.",

khoj/utils/yaml.py

@@ -41,3 +41,7 @@ def parse_config_from_string(yaml_config: dict) -> FullConfig:
 def parse_config_from_file(yaml_config_file):
     "Parse and validate config in YML file"
     return parse_config_from_string(load_config_from_file(yaml_config_file))
+
+
+def yaml_dump(data):
+    return yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False)

{khoj-1.27.2.dev15.dist-info → khoj-1.28.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.27.2.dev15
+Version: 1.28.0
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
@@ -36,7 +36,7 @@ Requires-Dist: django==5.0.9
 Requires-Dist: docx2txt==0.8
 Requires-Dist: einops==0.8.0
 Requires-Dist: fastapi>=0.110.0
-Requires-Dist: google-generativeai==0.7.2
+Requires-Dist: google-generativeai==0.8.3
 Requires-Dist: httpx==0.25.0
 Requires-Dist: huggingface-hub>=0.22.2
 Requires-Dist: itsdangerous==2.1.2
@@ -78,6 +78,7 @@ Requires-Dist: black>=23.1.0; extra == 'dev'
 Requires-Dist: boto3>=1.34.57; extra == 'dev'
 Requires-Dist: factory-boy>=3.2.1; extra == 'dev'
 Requires-Dist: freezegun>=1.2.0; extra == 'dev'
+Requires-Dist: gitpython~=3.1.43; extra == 'dev'
 Requires-Dist: google-auth==2.23.3; extra == 'dev'
 Requires-Dist: gunicorn==22.0.0; extra == 'dev'
 Requires-Dist: mypy>=1.0.1; extra == 'dev'