khoj 1.28.3__py3-none-any.whl → 1.28.4.dev92__py3-none-any.whl

khoj/routers/api_content.py

@@ -36,16 +36,18 @@ from khoj.database.models import (
     LocalPlaintextConfig,
     NotionConfig,
 )
+from khoj.processor.content.docx.docx_to_entries import DocxToEntries
+from khoj.processor.content.pdf.pdf_to_entries import PdfToEntries
 from khoj.routers.helpers import (
     ApiIndexedDataLimiter,
     CommonQueryParams,
     configure_content,
+    get_file_content,
     get_user_config,
     update_telemetry_state,
 )
 from khoj.utils import constants, state
 from khoj.utils.config import SearchModels
-from khoj.utils.helpers import get_file_type
 from khoj.utils.rawconfig import (
     ContentConfig,
     FullConfig,
@@ -237,7 +239,7 @@ async def set_content_notion(
 
     if updated_config.token:
         # Trigger an async job to configure_content. Let it run without blocking the response.
-        background_tasks.add_task(run_in_executor, configure_content, {}, False, SearchType.Notion, user)
+        background_tasks.add_task(run_in_executor, configure_content, user, {}, False, SearchType.Notion)
 
     update_telemetry_state(
         request=request,
@@ -375,6 +377,75 @@ async def delete_content_source(
     return {"status": "ok"}
 
 
+@api_content.post("/convert", status_code=200)
+@requires(["authenticated"])
+async def convert_documents(
+    request: Request,
+    files: List[UploadFile],
+    client: Optional[str] = None,
+):
+    MAX_FILE_SIZE_MB = 10  # 10MB limit
+    MAX_FILE_SIZE_BYTES = MAX_FILE_SIZE_MB * 1024 * 1024
+
+    converted_files = []
+    supported_files = ["org", "markdown", "pdf", "plaintext", "docx"]
+
+    for file in files:
+        # Check file size first
+        file_size = 0
+        content = await file.read()
+        file_size = len(content)
+        await file.seek(0)  # Reset file pointer
+
+        if file_size > MAX_FILE_SIZE_BYTES:
+            logger.warning(
+                f"Skipped converting oversized file ({file_size / 1024 / 1024:.1f}MB) sent by {client} client: {file.filename}"
+            )
+            continue
+
+        file_data = get_file_content(file)
+        if file_data.file_type in supported_files:
+            extracted_content = (
+                file_data.content.decode(file_data.encoding) if file_data.encoding else file_data.content
+            )
+
+            if file_data.file_type == "docx":
+                entries_per_page = DocxToEntries.extract_text(file_data.content)
+                annotated_pages = [
+                    f"Page {index} of {file_data.name}:\n\n{entry}" for index, entry in enumerate(entries_per_page)
+                ]
+                extracted_content = "\n".join(annotated_pages)
+
+            elif file_data.file_type == "pdf":
+                entries_per_page = PdfToEntries.extract_text(file_data.content)
+                annotated_pages = [
+                    f"Page {index} of {file_data.name}:\n\n{entry}" for index, entry in enumerate(entries_per_page)
+                ]
+                extracted_content = "\n".join(annotated_pages)
+
+            size_in_bytes = len(extracted_content.encode("utf-8"))
+
+            converted_files.append(
+                {
+                    "name": file_data.name,
+                    "content": extracted_content,
+                    "file_type": file_data.file_type,
+                    "size": size_in_bytes,
+                }
+            )
+        else:
+            logger.warning(f"Skipped converting unsupported file type sent by {client} client: {file.filename}")
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="convert_documents",
+        client=client,
+    )
+
+    return Response(content=json.dumps(converted_files), media_type="application/json", status_code=200)
+
+
 async def indexer(
     request: Request,
     files: list[UploadFile],
@@ -398,12 +469,13 @@ async def indexer(
     try:
         logger.info(f"📬 Updating content index via API call by {client} client")
         for file in files:
-            file_content = file.file.read()
-            file_type, encoding = get_file_type(file.content_type, file_content)
-            if file_type in index_files:
-                index_files[file_type][file.filename] = file_content.decode(encoding) if encoding else file_content
+            file_data = get_file_content(file)
+            if file_data.file_type in index_files:
+                index_files[file_data.file_type][file_data.name] = (
+                    file_data.content.decode(file_data.encoding) if file_data.encoding else file_data.content
+                )
             else:
-                logger.warning(f"Skipped indexing unsupported file type sent by {client} client: {file.filename}")
+                logger.warning(f"Skipped indexing unsupported file type sent by {client} client: {file_data.name}")
 
         indexer_input = IndexerInput(
             org=index_files["org"],
@@ -440,10 +512,10 @@ async def indexer(
         success = await loop.run_in_executor(
             None,
             configure_content,
+            user,
             indexer_input.model_dump(),
             regenerate,
             t,
-            user,
         )
         if not success:
             raise RuntimeError(f"Failed to {method} {t} data sent by {client} client into content index")