khoj 1.16.1.dev15__py3-none-any.whl → 1.16.1.dev47__py3-none-any.whl

khoj/routers/helpers.py

@@ -8,6 +8,7 @@ import math
 import re
 from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timedelta, timezone
+from enum import Enum
 from functools import partial
 from random import random
 from typing import (
@@ -307,6 +308,7 @@ async def aget_relevant_output_modes(query: str, conversation_history: dict, is_
             # Check whether the tool exists as a valid ConversationCommand
             return ConversationCommand(response)
 
+        logger.error(f"Invalid output mode selected: {response}. Defaulting to text.")
         return ConversationCommand.Text
     except Exception:
         logger.error(f"Invalid response for determining relevant mode: {response}")
@@ -519,9 +521,6 @@ async def send_message_to_model_wrapper(
         chat_model_option or await ConversationAdapters.aget_default_conversation_config()
     )
 
-    if conversation_config is None:
-        raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.")
-
     chat_model = conversation_config.chat_model
     max_tokens = conversation_config.max_prompt_size
     tokenizer = conversation_config.tokenizer
@@ -755,7 +754,7 @@ async def text_to_image(
     references: List[Dict[str, Any]],
     online_results: Dict[str, Any],
     send_status_func: Optional[Callable] = None,
-) -> Tuple[Optional[str], int, Optional[str], str]:
+):
     status_code = 200
     image = None
     response = None
@@ -767,7 +766,8 @@ async def text_to_image(
         # If the user has not configured a text to image model, return an unsupported on server error
         status_code = 501
         message = "Failed to generate image. Setup image generation on the server."
-        return image_url or image, status_code, message, intent_type.value
+        yield image_url or image, status_code, message, intent_type.value
+        return
 
     text2image_model = text_to_image_config.model_name
     chat_history = ""
@@ -779,20 +779,21 @@ async def text_to_image(
             chat_history += f"Q: Prompt: {chat['intent']['query']}\n"
             chat_history += f"A: Improved Prompt: {chat['intent']['inferred-queries'][0]}\n"
 
-    with timer("Improve the original user query", logger):
-        if send_status_func:
-            await send_status_func("**✍🏽 Enhancing the Painting Prompt**")
-        improved_image_prompt = await generate_better_image_prompt(
-            message,
-            chat_history,
-            location_data=location_data,
-            note_references=references,
-            online_results=online_results,
-            model_type=text_to_image_config.model_type,
-        )
+    if send_status_func:
+        async for event in send_status_func("**✍🏽 Enhancing the Painting Prompt**"):
+            yield {ChatEvent.STATUS: event}
+    improved_image_prompt = await generate_better_image_prompt(
+        message,
+        chat_history,
+        location_data=location_data,
+        note_references=references,
+        online_results=online_results,
+        model_type=text_to_image_config.model_type,
+    )
 
     if send_status_func:
-        await send_status_func(f"**🖼️ Painting using Enhanced Prompt**:\n{improved_image_prompt}")
+        async for event in send_status_func(f"**🖼️ Painting using Enhanced Prompt**:\n{improved_image_prompt}"):
+            yield {ChatEvent.STATUS: event}
 
     if text_to_image_config.model_type == TextToImageModelConfig.ModelType.OPENAI:
         with timer("Generate image with OpenAI", logger):
@@ -817,12 +818,14 @@ async def text_to_image(
                     logger.error(f"Image Generation blocked by OpenAI: {e}")
                     status_code = e.status_code  # type: ignore
                     message = f"Image generation blocked by OpenAI: {e.message}"  # type: ignore
-                    return image_url or image, status_code, message, intent_type.value
+                    yield image_url or image, status_code, message, intent_type.value
+                    return
                 else:
                     logger.error(f"Image Generation failed with {e}", exc_info=True)
                     message = f"Image generation failed with OpenAI error: {e.message}"  # type: ignore
                     status_code = e.status_code  # type: ignore
-                    return image_url or image, status_code, message, intent_type.value
+                    yield image_url or image, status_code, message, intent_type.value
+                    return
 
     elif text_to_image_config.model_type == TextToImageModelConfig.ModelType.STABILITYAI:
         with timer("Generate image with Stability AI", logger):
@@ -844,7 +847,8 @@ async def text_to_image(
                 logger.error(f"Image Generation failed with {e}", exc_info=True)
                 message = f"Image generation failed with Stability AI error: {e}"
                 status_code = e.status_code  # type: ignore
-                return image_url or image, status_code, message, intent_type.value
+                yield image_url or image, status_code, message, intent_type.value
+                return
 
     with timer("Convert image to webp", logger):
         # Convert png to webp for faster loading
@@ -864,7 +868,7 @@ async def text_to_image(
         intent_type = ImageIntentType.TEXT_TO_IMAGE_V3
         image = base64.b64encode(webp_image_bytes).decode("utf-8")
 
-    return image_url or image, status_code, improved_image_prompt, intent_type.value
+    yield image_url or image, status_code, improved_image_prompt, intent_type.value
 
 
 class ApiUserRateLimiter:
@@ -1186,3 +1190,11 @@ def construct_automation_created_message(automation: Job, crontime: str, query_t
 
 Manage your automations [here](/automations).
     """.strip()
+
+
+class ChatEvent(Enum):
+    START_LLM_RESPONSE = "start_llm_response"
+    END_LLM_RESPONSE = "end_llm_response"
+    MESSAGE = "message"
+    REFERENCES = "references"
+    STATUS = "status"

khoj/routers/indexer.py

@@ -60,7 +60,7 @@ async def update(
     indexed_data_limiter: ApiIndexedDataLimiter = Depends(
         ApiIndexedDataLimiter(
             incoming_entries_size_limit=10,
-            subscribed_incoming_entries_size_limit=25,
+            subscribed_incoming_entries_size_limit=75,
             total_entries_size_limit=10,
             subscribed_total_entries_size_limit=100,
         )

khoj/utils/fs_syncer.py

@@ -22,7 +22,7 @@ magika = Magika()
 
 
 def collect_files(search_type: Optional[SearchType] = SearchType.All, user=None) -> dict:
-    files = {}
+    files: dict[str, dict] = {"docx": {}, "image": {}}
 
     if search_type == SearchType.All or search_type == SearchType.Org:
         org_config = LocalOrgConfig.objects.filter(user=user).first()

{khoj-1.16.1.dev15.dist-info → khoj-1.16.1.dev47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.16.1.dev15
+Version: 1.16.1.dev47
 Summary: An AI copilot for your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
@@ -43,7 +43,7 @@ Requires-Dist: jinja2==3.1.4
 Requires-Dist: langchain-community==0.2.5
 Requires-Dist: langchain-openai==0.1.7
 Requires-Dist: langchain==0.2.5
-Requires-Dist: llama-cpp-python==0.2.76
+Requires-Dist: llama-cpp-python==0.2.82
 Requires-Dist: lxml==4.9.3
 Requires-Dist: magika~=0.5.1
 Requires-Dist: markdown-it-py~=3.0.0

{khoj-1.16.1.dev15.dist-info → khoj-1.16.1.dev47.dist-info}/RECORD

@@ -11,7 +11,7 @@ khoj/database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/database/admin.py,sha256=P1zguxSlqnpCvF7CcF8wS9tGCsO3SNDnl6-C_9eyYFA,8814
 khoj/database/apps.py,sha256=pM4tkX5Odw4YW_hLLKK8Nd5kqGddf1en0oMCea44RZw,153
 khoj/database/tests.py,sha256=mrbGGRNg5jwbTJtWWa7zSKdDyeB4vmgZCRc2nk6VY-g,60
-khoj/database/adapters/__init__.py,sha256=RwYIWf8r7VFjF0HDzgevinO_msnBN6TT-mIanLK_r3Y,47740
+khoj/database/adapters/__init__.py,sha256=KpvAuICVwOz_zUhkqy-2n3xMfvLN-Pm9kqzm-WlI8kY,47525
 khoj/database/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/database/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/database/management/commands/change_generated_images_url.py,sha256=w52FwRlyOL4YRpp9O6jJUjSIuGLxVhaS2w1D7gtQgOE,2644
@@ -87,7 +87,7 @@ khoj/interface/web/404.html,sha256=_WKN17MwiW1b1-8-vx240ZxTO2LmPUh5YnUCaGQpoZk,1
 khoj/interface/web/agent.html,sha256=vH3p6gwGlKkNmGO_TSwJMoOvZMvv8t4K8GwaN0QuOAU,9901
 khoj/interface/web/agents.html,sha256=_6WZIYGav87WGJR9QA_UTLESD_itSINuxVOl3ybnRRM,9320
 khoj/interface/web/base_config.html,sha256=vORIYJQ5UzaA3TAMcywRisk-YaaJEwkVAiEJGsbYxvA,12173
-khoj/interface/web/chat.html,sha256=ODNMRyFjd_SY3R4NZopB2phCSCq9GaxvMw0O7I4d1t0,155666
+khoj/interface/web/chat.html,sha256=B8Hpb9dwcHEXMC5tSyZc3945IkWCOwjWC2zcatflh5Y,149819
 khoj/interface/web/config.html,sha256=8Ge79FbYfWLovc_u8XVkCDGFjK7ZqlRFI1hHI_Z2KWs,46026
 khoj/interface/web/config_automation.html,sha256=B_-uUBi-uVlhPqFNsxxNdp-m0BGST0maqJ1qTzfkkzA,46095
 khoj/interface/web/content_source_computer_input.html,sha256=FzJbF1XpOF2HV6C1l0iKiU4LdRtMa5T0ETSuLyC7PPM,5983
@@ -184,33 +184,33 @@ khoj/processor/content/pdf/pdf_to_entries.py,sha256=wGXSmxifrjRE9eJWoFokI0G66kyM
 khoj/processor/content/plaintext/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/processor/content/plaintext/plaintext_to_entries.py,sha256=4JpWWulSjzrYsGNLPt7Z0qZykjU6isF58vx-0RJjQRw,5127
 khoj/processor/conversation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/processor/conversation/prompts.py,sha256=uP8rCZPSg70yf-qfT_919B4_OrpJ_BepOUy1mxyAtus,30923
-khoj/processor/conversation/utils.py,sha256=80QyNvijSgL3dDcMvLkNtTsHkKD16MozprzoeuE03NQ,10678
+khoj/processor/conversation/prompts.py,sha256=8X0FzJ8iZR0lTdNtCv0WmGPTm4EyP-YZiKJsluZkC9g,32086
+khoj/processor/conversation/utils.py,sha256=_uWu1nxcY-Cv2ip-TBdyqepUkMYhijvzjnproumvzXk,10586
 khoj/processor/conversation/anthropic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/processor/conversation/anthropic/anthropic_chat.py,sha256=ovhrqcRHJV7XoMpio2Wzwr1hugfzwA6CezUNzuiUtPc,7704
+khoj/processor/conversation/anthropic/anthropic_chat.py,sha256=7yBlY26PNI4wzwHph4TTQ_tJlLmoNjiXyhzHPJeUVmI,7887
 khoj/processor/conversation/anthropic/utils.py,sha256=uc9d_gIk4Ux2NRlkw3FP9L9KeLRoUI7nC_qb2Qp6d_4,3253
 khoj/processor/conversation/offline/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/processor/conversation/offline/chat_model.py,sha256=dbwVp2irUpS47U9OX-5kLyFARdqd9DWvugQ9QqhlH_s,8884
-khoj/processor/conversation/offline/utils.py,sha256=-e-YYVQIWozAjvh1SvkrNNh9WArMRgP-DmOt5b3ae5s,2954
+khoj/processor/conversation/offline/chat_model.py,sha256=7OIexLDXfdyF2jk-JFMqojNW-grlc2jtCt1zxCO-VAY,9349
+khoj/processor/conversation/offline/utils.py,sha256=n2T3vwAIZnSe9-UN1VORLPrLEUcamXXE9isL2ie-9R8,3033
 khoj/processor/conversation/offline/whisper.py,sha256=DJI-8y8DULO2cQ49m2VOvRyIZ2TxBypc15gM8O3HuMI,470
 khoj/processor/conversation/openai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/processor/conversation/openai/gpt.py,sha256=CIB9LRbgLnw6q5p_MxqVYXDRFP8ZC_4IBWFm8a29Av0,6936
+khoj/processor/conversation/openai/gpt.py,sha256=bbEyiOeRgko65bLsFHNlkMw1cvbiawQsjmDipCpy0E4,7111
 khoj/processor/conversation/openai/utils.py,sha256=UCfu-dHnkgcKxMajUaWnlxW5Zdidqecv5jIdmimie6o,4067
 khoj/processor/conversation/openai/whisper.py,sha256=RuwDtxSJrVWYdZz4aVnk0XiMQy9w8W9lFcVfE0hMiFY,432
 khoj/processor/speech/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/processor/speech/text_to_speech.py,sha256=Q7sapi5Hv6woXOumtrGqR0t6izZrFBkWXFOGrHM6dJ4,1929
 khoj/processor/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/processor/tools/online_search.py,sha256=FGAu-FmD158ILXoVo3j2fbqM5SnaWusKV39o68GhvQY,9373
+khoj/processor/tools/online_search.py,sha256=3qVHg3oU7nU3O2MFG9M6PsdXNusp_B4oTSTIpxN_oLc,9640
 khoj/routers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-khoj/routers/api.py,sha256=CzDpN6zX3XfQkRHKEgVYffFkG4v3XoHvqsKvHfFbhz8,23690
+khoj/routers/api.py,sha256=simxuuN0ro2JtWnNg7qZX1ZoNzZdL5Zvz3MKhL3sM5I,23823
 khoj/routers/api_agents.py,sha256=A8gS-LIgg5cjV9Ec7P-TgCmmylaNtT43TqXcxIia3PQ,1443
-khoj/routers/api_chat.py,sha256=SdBkCveilzM75uyBc0RAJEk-pOolQdl_Xa7L2B3ceQA,45409
+khoj/routers/api_chat.py,sha256=yyhs22HLeqDkd7gEE0LgwZ2pGvuxt-KwL5o7umwDRLA,36232
 khoj/routers/api_config.py,sha256=qK4he5cH-Rdx5WagOOfoNRpHuJUoD8ja4zbl0ksBh9M,13446
 khoj/routers/api_phone.py,sha256=p9yfc4WeMHDC0hg3aQk60a2VBy8rZPdEnz9wdJ7DzkU,2208
 khoj/routers/auth.py,sha256=pCOLSRihJWcn097DRPxLjPdlejsjHJFRs9jHIzLujZU,6247
 khoj/routers/email.py,sha256=hhxyLdmgSUfeHTXTco7g4aWDAEaVzF82mV-hshxzHsg,3695
-khoj/routers/helpers.py,sha256=dmy8_HTt5fcAhLcYH_lE6NCHU44Xgneb431QhAjoX-U,47418
-khoj/routers/indexer.py,sha256=1fDmOraiCHsggZgzwyH1H7VPAJccjuMa6VHAnJkSmDo,12560
+khoj/routers/helpers.py,sha256=xVE3ZQl5A0HC64OYekl1GSATpf0LrBraxGAGBqP7XR8,47617
+khoj/routers/indexer.py,sha256=CnXbxJ5ZCBBuNvJalGyMNFa4FBa47loEjSiPz_KjMhQ,12560
 khoj/routers/notion.py,sha256=DaEvbBLqg0F7UArRalKZqzvl0fxW7snShByo-BOyRO8,3137
 khoj/routers/storage.py,sha256=9ZfBsr_omxdFV-Lcj6p30xTQcF_7wwCZ9XFJukzjITE,1429
 khoj/routers/subscription.py,sha256=qEyV7m7mrY6MGtaij8W3v61tpzX2a7ydm2B-E8h_R-M,4285
@@ -227,7 +227,7 @@ khoj/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 khoj/utils/cli.py,sha256=oTDt6ClKvYNJ3_j9Omwx49H03IxC7sq4oL1qFpormWc,3456
 khoj/utils/config.py,sha256=ZNVZjyJhd7KmeKCA1a1Dtns2ghhAtZtsCF9Lsm-gilw,1805
 khoj/utils/constants.py,sha256=vtQx9UTXwFrZdgdEWudTYMEDxecXP5g2rRYWYIMAsdU,856
-khoj/utils/fs_syncer.py,sha256=Se7414BWUqww0BuRmXyiPvP5lNH6aanhLp3607LasY4,9890
+khoj/utils/fs_syncer.py,sha256=A_ME2w5O7RRzku06CrM3NBY7QakkgfGV62CUuIeRjMs,9930
 khoj/utils/helpers.py,sha256=b7q_aQw52C_CO9y18KWRD2Z1Oyo9E6Ef7F9_jfi5S00,14041
 khoj/utils/initialization.py,sha256=Q8csjz5YqQ7WZ6mYJWvmp_Dh36dFRpZ3WRWxEgT2Ql0,7117
 khoj/utils/jsonl.py,sha256=0Ac_COqr8sLCXntzZtquxuCEVRM2c3yKeDRGhgOBRpQ,1192
@@ -235,8 +235,8 @@ khoj/utils/models.py,sha256=Q5tcC9-z25sCiub048fLnvZ6_IIO1bcPNxt5payekk0,2009
 khoj/utils/rawconfig.py,sha256=iqyidXyM4AM6QAPLLMFrMuYR6dYplj374UBNFDOBwkY,3892
 khoj/utils/state.py,sha256=x4GTewP1YhOA6c_32N4wOjnV-3AA3xG_qbY1-wC2Uxc,1559
 khoj/utils/yaml.py,sha256=H0mfw0ZvBFUvFmCQn8pWkfxdmIebsrSykza7D8Wv6wQ,1430
-khoj-1.16.1.dev15.dist-info/METADATA,sha256=tOakxp8QiBGK_TbsIWg-YOS1ql0RS28NoNwZjWqArvA,6894
-khoj-1.16.1.dev15.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
-khoj-1.16.1.dev15.dist-info/entry_points.txt,sha256=KBIcez5N_jCgq_ER4Uxf-e1lxTBMTE_BBjMwwfeZyAg,39
-khoj-1.16.1.dev15.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
-khoj-1.16.1.dev15.dist-info/RECORD,,
+khoj-1.16.1.dev47.dist-info/METADATA,sha256=-bQLRCGiG5DM8ByuoW0zZ5dqoyN3T9vOFhDpHai88uw,6894
+khoj-1.16.1.dev47.dist-info/WHEEL,sha256=1yFddiXMmvYK7QYTqtRNtX66WJ0Mz8PYEiEUoOUUxRY,87
+khoj-1.16.1.dev47.dist-info/entry_points.txt,sha256=KBIcez5N_jCgq_ER4Uxf-e1lxTBMTE_BBjMwwfeZyAg,39
+khoj-1.16.1.dev47.dist-info/licenses/LICENSE,sha256=hIahDEOTzuHCU5J2nd07LWwkLW7Hko4UFO__ffsvB-8,34523
+khoj-1.16.1.dev47.dist-info/RECORD,,