khoj 1.42.9.dev27__py3-none-any.whl → 1.42.10.dev2__py3-none-any.whl

khoj/processor/content/org_mode/orgnode.py

@@ -58,7 +58,7 @@ def makelist_with_filepath(filename):
     return makelist(f, filename)
 
 
-def makelist(file, filename, start_line: int = 1, ancestry_lines: int = 0) -> List["Orgnode"]:
+def makelist(file, filename) -> List["Orgnode"]:
     """
     Read an org-mode file and return a list of Orgnode objects
     created from this file.
@@ -66,7 +66,7 @@ def makelist(file, filename, start_line: int = 1, ancestry_lines: int = 0) -> Li
     ctr = 0
 
     if type(file) == str:
-        f = file.splitlines()
+        f = file.split("\n")
     else:
         f = file
 
@@ -114,23 +114,14 @@ def makelist(file, filename, start_line: int = 1, ancestry_lines: int = 0) -> Li
                     logbook = list()
                 thisNode.properties = property_map
                 nodelist.append(thisNode)
-            # Account for ancestry lines that were prepended when calculating line numbers
-            if ancestry_lines > 0:
-                calculated_line = start_line + ctr - 1 - ancestry_lines
-                if calculated_line <= 0:
-                    calculated_line = 1  # Fallback to line 1 if calculation results in invalid line number
-            else:
-                calculated_line = start_line + ctr - 1
-                if calculated_line <= 0:
-                    calculated_line = ctr  # Use the original behavior if start_line calculation fails
-            property_map = {"LINE": f"file://{normalize_filename(filename)}#line={calculated_line}"}
+            property_map = {"LINE": f"file:{normalize_filename(filename)}::{ctr}"}
             previous_level = level
             previous_heading: str = heading
             level = heading_search.group(1)
             heading = heading_search.group(2)
             bodytext = ""
             tags = list()  # set of all tags in headline
-            tag_search = re.search(r"(.*?)\s+:([a-zA-Z0-9@_].*?):\s*$", heading)
+            tag_search = re.search(r"(.*?)\s*:([a-zA-Z0-9].*?):$", heading)
             if tag_search:
                 heading = tag_search.group(1)
                 parsedtags = tag_search.group(2)
@@ -269,6 +260,14 @@ def makelist(file, filename, start_line: int = 1, ancestry_lines: int = 0) -> Li
         # Prefix filepath/title to ancestors
         n.ancestors = [file_title] + n.ancestors
 
+        # Set SOURCE property to a file+heading based org-mode link to the entry
+        if n.level == 0:
+            n.properties["LINE"] = f"file:{normalize_filename(filename)}::0"
+            n.properties["SOURCE"] = f"[[file:{normalize_filename(filename)}]]"
+        else:
+            escaped_heading = n.heading.replace("[", "\\[").replace("]", "\\]")
+            n.properties["SOURCE"] = f"[[file:{normalize_filename(filename)}::*{escaped_heading}]]"
+
     return nodelist
 
 
@@ -521,11 +520,10 @@ class Orgnode(object):
             n = n + "\n"
 
         # Output Property Drawer
-        if self._properties:
-            n = n + indent + ":PROPERTIES:\n"
-            for key, value in self._properties.items():
-                n = n + indent + f":{key}: {value}\n"
-            n = n + indent + ":END:\n"
+        n = n + indent + ":PROPERTIES:\n"
+        for key, value in self._properties.items():
+            n = n + indent + f":{key}: {value}\n"
+        n = n + indent + ":END:\n"
 
         # Output Body
         if self.hasBody:

khoj/processor/content/text_to_entries.py

@@ -81,35 +81,8 @@ class TextToEntries(ABC):
             chunked_entry_chunks = text_splitter.split_text(entry.compiled)
             corpus_id = uuid.uuid4()
 
-            line_start = None
-            last_offset = 0
-            if entry.uri and entry.uri.startswith("file://"):
-                if "#line=" in entry.uri:
-                    line_start = int(entry.uri.split("#line=", 1)[-1].split("&", 1)[0])
-                else:
-                    line_start = 0
-
             # Create heading prefixed entry from each chunk
             for chunk_index, compiled_entry_chunk in enumerate(chunked_entry_chunks):
-                # set line start in uri of chunked entries
-                entry_uri = entry.uri
-                if line_start is not None:
-                    # Find the chunk in the raw text to get an accurate line number.
-                    # Search for the unmodified chunk from the last offset.
-                    searchable_chunk = compiled_entry_chunk.strip()
-                    if searchable_chunk:
-                        chunk_start_pos_in_raw = entry.raw.find(searchable_chunk, last_offset)
-                        if chunk_start_pos_in_raw != -1:
-                            # Found the chunk. Calculate its line offset from the start of the raw text.
-                            line_offset_in_raw = entry.raw[:chunk_start_pos_in_raw].count("\n")
-                            new_line_num = line_start + line_offset_in_raw
-                            entry_uri = re.sub(r"#line=\d+", f"#line={new_line_num}", entry.uri)
-                            # Update search position for the next chunk to start after the current one.
-                            last_offset = chunk_start_pos_in_raw + len(searchable_chunk)
-                        else:
-                            # Chunk not found in raw text, likely from a heading. Use original line_start.
-                            entry_uri = re.sub(r"#line=\d+", f"#line={line_start}", entry.uri)
-
                 # Prepend heading to all other chunks, the first chunk already has heading from original entry
                 if chunk_index > 0 and entry.heading:
                     # Snip heading to avoid crossing max_tokens limit
@@ -126,7 +99,6 @@ class TextToEntries(ABC):
                 entry.raw = compiled_entry_chunk if raw_is_compiled else TextToEntries.clean_field(entry.raw)
                 entry.heading = TextToEntries.clean_field(entry.heading)
                 entry.file = TextToEntries.clean_field(entry.file)
-                entry_uri = TextToEntries.clean_field(entry_uri)
 
                 chunked_entries.append(
                     Entry(
@@ -135,7 +107,6 @@ class TextToEntries(ABC):
                         heading=entry.heading,
                         file=entry.file,
                         corpus_id=corpus_id,
-                        uri=entry_uri,
                     )
                 )
 
@@ -221,7 +192,6 @@ class TextToEntries(ABC):
                             file_type=file_type,
                             hashed_value=entry_hash,
                             corpus_id=entry.corpus_id,
-                            url=entry.uri,
                             search_model=model,
                             file_object=file_object,
                         )

khoj/processor/conversation/anthropic/anthropic_chat.py

@@ -22,20 +22,12 @@ logger = logging.getLogger(__name__)
 
 
 def anthropic_send_message_to_model(
-    messages,
-    api_key,
-    api_base_url,
-    model,
-    response_type="text",
-    response_schema=None,
-    tools=None,
-    deepthought=False,
-    tracer={},
+    messages, api_key, api_base_url, model, response_type="text", response_schema=None, deepthought=False, tracer={}
 ):
     """
     Send message to model
     """
-    # Get response from model. Don't use response_type because Anthropic doesn't support it.
+    # Get Response from GPT. Don't use response_type because Anthropic doesn't support it.
     return anthropic_completion_with_backoff(
         messages=messages,
         system_prompt="",
@@ -44,7 +36,6 @@ def anthropic_send_message_to_model(
         api_base_url=api_base_url,
         response_type=response_type,
         response_schema=response_schema,
-        tools=tools,
         deepthought=deepthought,
         tracer=tracer,
     )

khoj/processor/conversation/anthropic/utils.py

@@ -1,8 +1,9 @@
 import json
 import logging
 from copy import deepcopy
+from textwrap import dedent
 from time import perf_counter
-from typing import AsyncGenerator, Dict, List
+from typing import AsyncGenerator, Dict, List, Optional, Type
 
 import anthropic
 from langchain_core.messages.chat import ChatMessage
@@ -17,14 +18,11 @@ from tenacity import (
 
 from khoj.processor.conversation.utils import (
     ResponseWithThought,
-    ToolCall,
     commit_conversation_trace,
     get_image_from_base64,
     get_image_from_url,
 )
 from khoj.utils.helpers import (
-    ToolDefinition,
-    create_tool_definition,
     get_anthropic_async_client,
     get_anthropic_client,
     get_chat_usage_metrics,
@@ -59,10 +57,9 @@ def anthropic_completion_with_backoff(
     max_tokens: int | None = None,
     response_type: str = "text",
     response_schema: BaseModel | None = None,
-    tools: List[ToolDefinition] = None,
     deepthought: bool = False,
     tracer: dict = {},
-) -> ResponseWithThought:
+) -> str:
     client = anthropic_clients.get(api_key)
     if not client:
         client = get_anthropic_client(api_key, api_base_url)
@@ -70,26 +67,12 @@ def anthropic_completion_with_backoff(
 
     formatted_messages, system = format_messages_for_anthropic(messages, system_prompt)
 
-    thoughts = ""
     aggregated_response = ""
     final_message = None
     model_kwargs = model_kwargs or dict()
-
-    # Configure structured output
-    if tools:
-        # Convert tools to Anthropic format
-        model_kwargs["tools"] = [
-            anthropic.types.ToolParam(name=tool.name, description=tool.description, input_schema=tool.schema)
-            for tool in tools
-        ]
-        # Cache tool definitions
-        last_tool = model_kwargs["tools"][-1]
-        last_tool["cache_control"] = {"type": "ephemeral"}
-    elif response_schema:
-        tool = create_tool_definition(response_schema)
-        model_kwargs["tools"] = [
-            anthropic.types.ToolParam(name=tool.name, description=tool.description, input_schema=tool.schema)
-        ]
+    if response_schema:
+        tool = create_anthropic_tool_definition(response_schema=response_schema)
+        model_kwargs["tools"] = [tool]
     elif response_type == "json_object" and not (is_reasoning_model(model_name) and deepthought):
         # Prefill model response with '{' to make it output a valid JSON object. Not supported with extended thinking.
         formatted_messages.append(anthropic.types.MessageParam(role="assistant", content="{"))
@@ -113,41 +96,15 @@ def anthropic_completion_with_backoff(
         max_tokens=max_tokens,
         **(model_kwargs),
     ) as stream:
-        for chunk in stream:
-            if chunk.type != "content_block_delta":
-                continue
-            if chunk.delta.type == "thinking_delta":
-                thoughts += chunk.delta.thinking
-            elif chunk.delta.type == "text_delta":
-                aggregated_response += chunk.delta.text
+        for text in stream.text_stream:
+            aggregated_response += text
         final_message = stream.get_final_message()
 
-    # Track raw content of model response to reuse for cache hits in multi-turn chats
-    raw_content = [item.model_dump() for item in final_message.content]
-
-    # Extract all tool calls if tools are enabled
-    if tools:
-        tool_calls = [
-            ToolCall(name=item.name, args=item.input, id=item.id).__dict__
-            for item in final_message.content
-            if item.type == "tool_use"
-        ]
-        if tool_calls:
-            # If there are tool calls, aggregate thoughts and responses into thoughts
-            if thoughts and aggregated_response:
-                # wrap each line of thought in italics
-                thoughts = "\n".join([f"*{line.strip()}*" for line in thoughts.splitlines() if line.strip()])
-                thoughts = f"{thoughts}\n\n{aggregated_response}"
-            else:
-                thoughts = thoughts or aggregated_response
-            # Json dump tool calls into aggregated response
-            aggregated_response = json.dumps(tool_calls)
-    # If response schema is used, return the first tool call's input
-    elif response_schema:
-        for item in final_message.content:
-            if item.type == "tool_use":
-                aggregated_response = json.dumps(item.input)
-                break
+    # Extract first tool call from final message
+    for item in final_message.content:
+        if item.type == "tool_use":
+            aggregated_response = json.dumps(item.input)
+            break
 
     # Calculate cost of chat
     input_tokens = final_message.usage.input_tokens
@@ -169,7 +126,7 @@ def anthropic_completion_with_backoff(
     if is_promptrace_enabled():
         commit_conversation_trace(messages, aggregated_response, tracer)
 
-    return ResponseWithThought(text=aggregated_response, thought=thoughts, raw_content=raw_content)
+    return aggregated_response
 
 
 @retry(
@@ -226,10 +183,10 @@ async def anthropic_chat_completion_with_backoff(
             if chunk.type == "message_delta":
                 if chunk.delta.stop_reason == "refusal":
                     yield ResponseWithThought(
-                        text="...I'm sorry, but my safety filters prevent me from assisting with this query."
+                        response="...I'm sorry, but my safety filters prevent me from assisting with this query."
                     )
                 elif chunk.delta.stop_reason == "max_tokens":
-                    yield ResponseWithThought(text="...I'm sorry, but I've hit my response length limit.")
+                    yield ResponseWithThought(response="...I'm sorry, but I've hit my response length limit.")
                 if chunk.delta.stop_reason in ["refusal", "max_tokens"]:
                     logger.warning(
                         f"LLM Response Prevented for {model_name}: {chunk.delta.stop_reason}.\n"
@@ -242,7 +199,7 @@ async def anthropic_chat_completion_with_backoff(
             # Handle streamed response chunk
             response_chunk: ResponseWithThought = None
             if chunk.delta.type == "text_delta":
-                response_chunk = ResponseWithThought(text=chunk.delta.text)
+                response_chunk = ResponseWithThought(response=chunk.delta.text)
                 aggregated_response += chunk.delta.text
             if chunk.delta.type == "thinking_delta":
                 response_chunk = ResponseWithThought(thought=chunk.delta.thinking)
@@ -275,14 +232,13 @@ async def anthropic_chat_completion_with_backoff(
         commit_conversation_trace(messages, aggregated_response, tracer)
 
 
-def format_messages_for_anthropic(raw_messages: list[ChatMessage], system_prompt: str = None):
+def format_messages_for_anthropic(messages: list[ChatMessage], system_prompt: str = None):
     """
     Format messages for Anthropic
     """
     # Extract system prompt
     system_prompt = system_prompt or ""
-    messages = deepcopy(raw_messages)
-    for message in messages:
+    for message in messages.copy():
         if message.role == "system":
             if isinstance(message.content, list):
                 system_prompt += "\n".join([part["text"] for part in message.content if part["type"] == "text"])
@@ -294,30 +250,15 @@ def format_messages_for_anthropic(raw_messages: list[ChatMessage], system_prompt
     else:
         system = None
 
-    # Anthropic requires the first message to be a user message unless its a tool call
-    message_type = messages[0].additional_kwargs.get("message_type", None)
-    if len(messages) == 1 and message_type != "tool_call":
+    # Anthropic requires the first message to be a 'user' message
+    if len(messages) == 1:
         messages[0].role = "user"
+    elif len(messages) > 1 and messages[0].role == "assistant":
+        messages = messages[1:]
 
+    # Convert image urls to base64 encoded images in Anthropic message format
     for message in messages:
-        # Handle tool call and tool result message types from additional_kwargs
-        message_type = message.additional_kwargs.get("message_type")
-        if message_type == "tool_call":
-            pass
-        elif message_type == "tool_result":
-            # Convert tool_result to Anthropic tool_result format
-            content = []
-            for part in message.content:
-                content.append(
-                    {
-                        "type": "tool_result",
-                        "tool_use_id": part["id"],
-                        "content": part["content"],
-                    }
-                )
-            message.content = content
-        # Convert image urls to base64 encoded images in Anthropic message format
-        elif isinstance(message.content, list):
+        if isinstance(message.content, list):
             content = []
             # Sort the content. Anthropic models prefer that text comes after images.
             message.content.sort(key=lambda x: 0 if x["type"] == "image_url" else 1)
@@ -363,15 +304,18 @@ def format_messages_for_anthropic(raw_messages: list[ChatMessage], system_prompt
                     if isinstance(block, dict) and "cache_control" in block:
                         del block["cache_control"]
 
-        # Add cache control to the last content block of last message.
-        # Caching should improve research efficiency.
-        cache_message = messages[-1]
+        # Add cache control to the last content block of second to last message.
+        # In research mode, this message content is list of iterations, updated after each research iteration.
+        # Caching it should improve research efficiency.
+        cache_message = messages[-2]
         if isinstance(cache_message.content, list) and cache_message.content:
             # Add cache control to the last content block only if it's a text block with non-empty content
             last_block = cache_message.content[-1]
-            if isinstance(last_block, dict) and (
-                (last_block.get("type") == "text" and last_block.get("text", "").strip())
-                or (last_block.get("type") == "tool_result" and last_block.get("content", []))
+            if (
+                isinstance(last_block, dict)
+                and last_block.get("type") == "text"
+                and last_block.get("text")
+                and last_block.get("text").strip()
             ):
                 last_block["cache_control"] = {"type": "ephemeral"}
 
@@ -382,5 +326,74 @@ def format_messages_for_anthropic(raw_messages: list[ChatMessage], system_prompt
     return formatted_messages, system
 
 
+def create_anthropic_tool_definition(
+    response_schema: Type[BaseModel],
+    tool_name: str = None,
+    tool_description: Optional[str] = None,
+) -> anthropic.types.ToolParam:
+    """
+    Converts a response schema BaseModel class into an Anthropic tool definition dictionary.
+
+    This format is expected by Anthropic's API when defining tools the model can use.
+
+    Args:
+        response_schema: The Pydantic BaseModel class to convert.
+                         This class defines the response schema for the tool.
+        tool_name: The name for the Anthropic tool (e.g., "get_weather", "plan_next_step").
+        tool_description: Optional description for the Anthropic tool.
+                           If None, it attempts to use the Pydantic model's docstring.
+                           If that's also missing, a fallback description is generated.
+
+    Returns:
+        An tool definition for Anthropic's API.
+    """
+    model_schema = response_schema.model_json_schema()
+
+    name = tool_name or response_schema.__name__.lower()
+    description = tool_description
+    if description is None:
+        docstring = response_schema.__doc__
+        if docstring:
+            description = dedent(docstring).strip()
+        else:
+            # Fallback description if no explicit one or docstring is provided
+            description = f"Tool named '{name}' accepts specified parameters."
+
+    # Process properties to inline enums and remove $defs dependency
+    processed_properties = {}
+    original_properties = model_schema.get("properties", {})
+    defs = model_schema.get("$defs", {})
+
+    for prop_name, prop_schema in original_properties.items():
+        current_prop_schema = deepcopy(prop_schema)  # Work on a copy
+        # Check for enums defined directly in the property for simpler direct enum definitions.
+        if "$ref" in current_prop_schema:
+            ref_path = current_prop_schema["$ref"]
+            if ref_path.startswith("#/$defs/"):
+                def_name = ref_path.split("/")[-1]
+                if def_name in defs and "enum" in defs[def_name]:
+                    enum_def = defs[def_name]
+                    current_prop_schema["enum"] = enum_def["enum"]
+                    current_prop_schema["type"] = enum_def.get("type", "string")
+                    if "description" not in current_prop_schema and "description" in enum_def:
+                        current_prop_schema["description"] = enum_def["description"]
+                    del current_prop_schema["$ref"]  # Remove the $ref as it's been inlined
+
+        processed_properties[prop_name] = current_prop_schema
+
+    # The input_schema for Anthropic tools is a JSON Schema object.
+    # Pydantic's model_json_schema() provides most of what's needed.
+    input_schema = {
+        "type": "object",
+        "properties": processed_properties,
+    }
+
+    # Include 'required' fields if specified in the Pydantic model
+    if "required" in model_schema and model_schema["required"]:
+        input_schema["required"] = model_schema["required"]
+
+    return anthropic.types.ToolParam(name=name, description=description, input_schema=input_schema)
+
+
 def is_reasoning_model(model_name: str) -> bool:
     return any(model_name.startswith(model) for model in REASONING_MODELS)

khoj/processor/conversation/google/gemini_chat.py

@@ -28,7 +28,6 @@ def gemini_send_message_to_model(
     api_base_url=None,
     response_type="text",
     response_schema=None,
-    tools=None,
     model_kwargs=None,
     deepthought=False,
     tracer={},
@@ -38,10 +37,8 @@ def gemini_send_message_to_model(
     """
     model_kwargs = {}
 
-    if tools:
-        model_kwargs["tools"] = tools
     # Monitor for flakiness in 1.5+ models. This would cause unwanted behavior and terminate response early in 1.5 models.
-    elif response_type == "json_object" and not model.startswith("gemini-1.5"):
+    if response_type == "json_object" and not model.startswith("gemini-1.5"):
         model_kwargs["response_mime_type"] = "application/json"
         if response_schema:
             model_kwargs["response_schema"] = response_schema