khoj 1.28.4.dev23__py3-none-any.whl → 1.28.4.dev77__py3-none-any.whl

khoj/routers/api_chat.py

@@ -19,7 +19,6 @@ from khoj.database.adapters import (
     AgentAdapters,
     ConversationAdapters,
     EntryAdapters,
-    FileObjectAdapters,
     PublicConversationAdapters,
     aget_user_name,
 )
@@ -45,12 +44,13 @@ from khoj.routers.helpers import (
     ConversationCommandRateLimiter,
     DeleteMessageRequestBody,
     FeedbackData,
+    acreate_title_from_history,
     agenerate_chat_response,
     aget_relevant_information_sources,
     aget_relevant_output_modes,
     construct_automation_created_message,
     create_automation,
-    extract_relevant_info,
+    gather_raw_query_files,
     generate_excalidraw_diagram,
     generate_summary_from_files,
     get_conversation_command,
@@ -76,7 +76,12 @@ from khoj.utils.helpers import (
     get_device,
     is_none_or_empty,
 )
-from khoj.utils.rawconfig import FileFilterRequest, FilesFilterRequest, LocationData
+from khoj.utils.rawconfig import (
+    ChatRequestBody,
+    FileFilterRequest,
+    FilesFilterRequest,
+    LocationData,
+)
 
 # Initialize Router
 logger = logging.getLogger(__name__)
@@ -374,7 +379,7 @@ def fork_public_conversation(
             {
                 "status": "ok",
                 "next_url": redirect_uri,
-                "conversation_id": new_conversation.id,
+                "conversation_id": str(new_conversation.id),
             }
         ),
     )
@@ -530,6 +535,32 @@ async def set_conversation_title(
     )
 
 
+@api_chat.post("/title")
+@requires(["authenticated"])
+async def generate_chat_title(
+    request: Request,
+    common: CommonQueryParams,
+    conversation_id: str,
+):
+    user: KhojUser = request.user.object
+    conversation = await ConversationAdapters.aget_conversation_by_user(user=user, conversation_id=conversation_id)
+
+    # Conversation.title is explicitly set by the user. Do not override.
+    if conversation.title:
+        return {"status": "ok", "title": conversation.title}
+
+    if not conversation:
+        raise HTTPException(status_code=404, detail="Conversation not found")
+
+    new_title = await acreate_title_from_history(request.user.object, conversation=conversation)
+
+    conversation.slug = new_title
+
+    conversation.asave()
+
+    return {"status": "ok", "title": new_title}
+
+
 @api_chat.delete("/conversation/message", response_class=Response)
 @requires(["authenticated"])
 def delete_message(request: Request, delete_request: DeleteMessageRequestBody) -> Response:
@@ -571,6 +602,7 @@ async def chat(
     country_code = body.country_code or get_country_code_from_timezone(body.timezone)
     timezone = body.timezone
     raw_images = body.images
+    raw_query_files = body.files
 
     async def event_generator(q: str, images: list[str]):
         start_time = time.perf_counter()
@@ -582,6 +614,7 @@ async def chat(
         q = unquote(q)
         train_of_thought = []
         nonlocal conversation_id
+        nonlocal raw_query_files
 
         tracer: dict = {
             "mid": turn_id,
@@ -601,6 +634,11 @@ async def chat(
                 if uploaded_image:
                     uploaded_images.append(uploaded_image)
 
+        query_files: Dict[str, str] = {}
+        if raw_query_files:
+            for file in raw_query_files:
+                query_files[file.name] = file.content
+
         async def send_event(event_type: ChatEvent, data: str | dict):
             nonlocal connection_alive, ttft, train_of_thought
             if not connection_alive or await request.is_disconnected():
@@ -711,6 +749,8 @@ async def chat(
         ## Extract Document References
         compiled_references: List[Any] = []
         inferred_queries: List[Any] = []
+        file_filters = conversation.file_filters if conversation and conversation.file_filters else []
+        attached_file_context = gather_raw_query_files(query_files)
 
         if conversation_commands == [ConversationCommand.Default] or is_automated_task:
             conversation_commands = await aget_relevant_information_sources(
@@ -720,6 +760,7 @@ async def chat(
                 user=user,
                 query_images=uploaded_images,
                 agent=agent,
+                query_files=attached_file_context,
                 tracer=tracer,
             )
 
@@ -765,6 +806,7 @@ async def chat(
                 user_name=user_name,
                 location=location,
                 file_filters=conversation.file_filters if conversation else [],
+                query_files=attached_file_context,
                 tracer=tracer,
             ):
                 if isinstance(research_result, InformationCollectionIteration):
@@ -804,10 +846,6 @@ async def chat(
                 response_log = "No files selected for summarization. Please add files using the section on the left."
                 async for result in send_llm_response(response_log):
                     yield result
-            elif len(file_filters) > 1 and not agent_has_entries:
-                response_log = "Only one file can be selected for summarization."
-                async for result in send_llm_response(response_log):
-                    yield result
             else:
                 async for response in generate_summary_from_files(
                     q=q,
@@ -817,6 +855,7 @@ async def chat(
                     query_images=uploaded_images,
                     agent=agent,
                     send_status_func=partial(send_event, ChatEvent.STATUS),
+                    query_files=attached_file_context,
                     tracer=tracer,
                 ):
                     if isinstance(response, dict) and ChatEvent.STATUS in response:
@@ -837,8 +876,9 @@ async def chat(
                 client_application=request.user.client_app,
                 conversation_id=conversation_id,
                 query_images=uploaded_images,
-                tracer=tracer,
                 train_of_thought=train_of_thought,
+                raw_query_files=raw_query_files,
+                tracer=tracer,
             )
             return
 
@@ -882,8 +922,9 @@ async def chat(
                 inferred_queries=[query_to_run],
                 automation_id=automation.id,
                 query_images=uploaded_images,
-                tracer=tracer,
                 train_of_thought=train_of_thought,
+                raw_query_files=raw_query_files,
+                tracer=tracer,
             )
             async for result in send_llm_response(llm_response):
                 yield result
@@ -905,6 +946,7 @@ async def chat(
                     partial(send_event, ChatEvent.STATUS),
                     query_images=uploaded_images,
                     agent=agent,
+                    query_files=attached_file_context,
                     tracer=tracer,
                 ):
                     if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -950,6 +992,7 @@ async def chat(
                     custom_filters,
                     query_images=uploaded_images,
                     agent=agent,
+                    query_files=attached_file_context,
                     tracer=tracer,
                 ):
                     if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -975,6 +1018,7 @@ async def chat(
                     partial(send_event, ChatEvent.STATUS),
                     query_images=uploaded_images,
                     agent=agent,
+                    query_files=attached_file_context,
                     tracer=tracer,
                 ):
                     if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -1015,6 +1059,7 @@ async def chat(
                     partial(send_event, ChatEvent.STATUS),
                     query_images=uploaded_images,
                     agent=agent,
+                    query_files=attached_file_context,
                     tracer=tracer,
                 ):
                     if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -1055,6 +1100,7 @@ async def chat(
                 send_status_func=partial(send_event, ChatEvent.STATUS),
                 query_images=uploaded_images,
                 agent=agent,
+                query_files=attached_file_context,
                 tracer=tracer,
             ):
                 if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -1086,8 +1132,9 @@ async def chat(
                 compiled_references=compiled_references,
                 online_results=online_results,
                 query_images=uploaded_images,
-                tracer=tracer,
                 train_of_thought=train_of_thought,
+                raw_query_files=raw_query_files,
+                tracer=tracer,
             )
             content_obj = {
                 "intentType": intent_type,
@@ -1116,6 +1163,7 @@ async def chat(
                 user=user,
                 agent=agent,
                 send_status_func=partial(send_event, ChatEvent.STATUS),
+                query_files=attached_file_context,
                 tracer=tracer,
             ):
                 if isinstance(result, dict) and ChatEvent.STATUS in result:
@@ -1144,8 +1192,9 @@ async def chat(
                 compiled_references=compiled_references,
                 online_results=online_results,
                 query_images=uploaded_images,
-                tracer=tracer,
                 train_of_thought=train_of_thought,
+                raw_query_files=raw_query_files,
+                tracer=tracer,
             )
 
             async for result in send_llm_response(json.dumps(content_obj)):
@@ -1171,8 +1220,10 @@ async def chat(
             user_name,
             researched_results,
             uploaded_images,
-            tracer,
             train_of_thought,
+            attached_file_context,
+            raw_query_files,
+            tracer,
         )
 
         # Send Response

khoj/routers/api_content.py

@@ -36,16 +36,18 @@ from khoj.database.models import (
     LocalPlaintextConfig,
     NotionConfig,
 )
+from khoj.processor.content.docx.docx_to_entries import DocxToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
 from khoj.routers.helpers import (
     ApiIndexedDataLimiter,
     CommonQueryParams,
     configure_content,
+    get_file_content,
     get_user_config,
     update_telemetry_state,
 )
 from khoj.utils import constants, state
 from khoj.utils.config import SearchModels
-from khoj.utils.helpers import get_file_type
 from khoj.utils.rawconfig import (
     ContentConfig,
     FullConfig,
@@ -237,7 +239,7 @@ async def set_content_notion(
 
     if updated_config.token:
         # Trigger an async job to configure_content. Let it run without blocking the response.
-        background_tasks.add_task(run_in_executor, configure_content, {}, False, SearchType.Notion, user)
+        background_tasks.add_task(run_in_executor, configure_content, user, {}, False, SearchType.Notion)
 
     update_telemetry_state(
         request=request,
@@ -375,6 +377,75 @@ async def delete_content_source(
     return {"status": "ok"}
 
 
+@api_content.post("/convert", status_code=200)
+@requires(["authenticated"])
+async def convert_documents(
+    request: Request,
+    files: List[UploadFile],
+    client: Optional[str] = None,
+):
+    MAX_FILE_SIZE_MB = 10  # 10MB limit
+    MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
+
+    converted_files = []
+    supported_files = ["org", "markdown", "pdf", "plaintext", "docx"]
+
+    for file in files:
+        # Check file size first
+        file_size = 0
+        content = await file.read()
+        file_size = len(content)
+        await file.seek(0)  # Reset file pointer
+
+        if file_size > MAX_FILE_SIZE_BYTES:
+            logger.warning(
+                f"Skipped converting oversized file ({file_size / 1024 / 1024:.1f}MB) sent by {client} client: {file.filename}"
+            )
+            continue
+
+        file_data = get_file_content(file)
+        if file_data.file_type in supported_files:
+            extracted_content = (
+                file_data.content.decode(file_data.encoding) if file_data.encoding else file_data.content
+            )
+
+            if file_data.file_type == "docx":
+                entries_per_page = DocxToEntries.extract_text(file_data.content)
+                annotated_pages = [
+                    f"Page {index} of {file_data.name}:\n\n{entry}" for index, entry in enumerate(entries_per_page)
+                ]
+                extracted_content = "\n".join(annotated_pages)
+
+            elif file_data.file_type == "pdf":
+                entries_per_page = PdfToEntries.extract_text(file_data.content)
+                annotated_pages = [
+                    f"Page {index} of {file_data.name}:\n\n{entry}" for index, entry in enumerate(entries_per_page)
+                ]
+                extracted_content = "\n".join(annotated_pages)
+
+            size_in_bytes = len(extracted_content.encode("utf-8"))
+
+            converted_files.append(
+                {
+                    "name": file_data.name,
+                    "content": extracted_content,
+                    "file_type": file_data.file_type,
+                    "size": size_in_bytes,
+                }
+            )
+        else:
+            logger.warning(f"Skipped converting unsupported file type sent by {client} client: {file.filename}")
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="convert_documents",
+        client=client,
+    )
+
+    return Response(content=json.dumps(converted_files), media_type="application/json", status_code=200)
+
+
 async def indexer(
     request: Request,
     files: list[UploadFile],
@@ -398,12 +469,13 @@ async def indexer(
     try:
         logger.info(f"📬 Updating content index via API call by {client} client")
         for file in files:
-            file_content = file.file.read()
-            file_type, encoding = get_file_type(file.content_type, file_content)
-            if file_type in index_files:
-                index_files[file_type][file.filename] = file_content.decode(encoding) if encoding else file_content
+            file_data = get_file_content(file)
+            if file_data.file_type in index_files:
+                index_files[file_data.file_type][file_data.name] = (
+                    file_data.content.decode(file_data.encoding) if file_data.encoding else file_data.content
+                )
             else:
-                logger.warning(f"Skipped indexing unsupported file type sent by {client} client: {file.filename}")
+                logger.warning(f"Skipped indexing unsupported file type sent by {client} client: {file_data.name}")
 
         indexer_input = IndexerInput(
             org=index_files["org"],
@@ -440,10 +512,10 @@ async def indexer(
         success = await loop.run_in_executor(
             None,
             configure_content,
+            user,
             indexer_input.model_dump(),
             regenerate,
             t,
-            user,
         )
         if not success:
             raise RuntimeError(f"Failed to {method} {t} data sent by {client} client into content index")