khoj 1.33.3.dev32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (393) hide show
  1. khoj/__init__.py +0 -0
  2. khoj/app/README.md +94 -0
  3. khoj/app/__init__.py +0 -0
  4. khoj/app/asgi.py +16 -0
  5. khoj/app/settings.py +218 -0
  6. khoj/app/urls.py +25 -0
  7. khoj/configure.py +452 -0
  8. khoj/database/__init__.py +0 -0
  9. khoj/database/adapters/__init__.py +1821 -0
  10. khoj/database/admin.py +417 -0
  11. khoj/database/apps.py +6 -0
  12. khoj/database/management/__init__.py +0 -0
  13. khoj/database/management/commands/__init__.py +0 -0
  14. khoj/database/management/commands/change_default_model.py +116 -0
  15. khoj/database/management/commands/change_generated_images_url.py +61 -0
  16. khoj/database/management/commands/convert_images_png_to_webp.py +99 -0
  17. khoj/database/migrations/0001_khojuser.py +98 -0
  18. khoj/database/migrations/0002_googleuser.py +32 -0
  19. khoj/database/migrations/0003_vector_extension.py +10 -0
  20. khoj/database/migrations/0004_content_types_and_more.py +181 -0
  21. khoj/database/migrations/0005_embeddings_corpus_id.py +19 -0
  22. khoj/database/migrations/0006_embeddingsdates.py +33 -0
  23. khoj/database/migrations/0007_add_conversation.py +27 -0
  24. khoj/database/migrations/0008_alter_conversation_conversation_log.py +17 -0
  25. khoj/database/migrations/0009_khojapiuser.py +24 -0
  26. khoj/database/migrations/0010_chatmodeloptions_and_more.py +83 -0
  27. khoj/database/migrations/0010_rename_embeddings_entry_and_more.py +30 -0
  28. khoj/database/migrations/0011_merge_20231102_0138.py +14 -0
  29. khoj/database/migrations/0012_entry_file_source.py +21 -0
  30. khoj/database/migrations/0013_subscription.py +37 -0
  31. khoj/database/migrations/0014_alter_googleuser_picture.py +17 -0
  32. khoj/database/migrations/0015_alter_subscription_user.py +21 -0
  33. khoj/database/migrations/0016_alter_subscription_renewal_date.py +17 -0
  34. khoj/database/migrations/0017_searchmodel.py +32 -0
  35. khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py +30 -0
  36. khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py +27 -0
  37. khoj/database/migrations/0020_reflectivequestion.py +36 -0
  38. khoj/database/migrations/0021_speechtotextmodeloptions_and_more.py +42 -0
  39. khoj/database/migrations/0022_texttoimagemodelconfig.py +25 -0
  40. khoj/database/migrations/0023_usersearchmodelconfig.py +33 -0
  41. khoj/database/migrations/0024_alter_entry_embeddings.py +18 -0
  42. khoj/database/migrations/0025_clientapplication_khojuser_phone_number_and_more.py +46 -0
  43. khoj/database/migrations/0025_searchmodelconfig_embeddings_inference_endpoint_and_more.py +22 -0
  44. khoj/database/migrations/0026_searchmodelconfig_cross_encoder_inference_endpoint_and_more.py +22 -0
  45. khoj/database/migrations/0027_merge_20240118_1324.py +13 -0
  46. khoj/database/migrations/0028_khojuser_verified_phone_number.py +17 -0
  47. khoj/database/migrations/0029_userrequests.py +27 -0
  48. khoj/database/migrations/0030_conversation_slug_and_title.py +38 -0
  49. khoj/database/migrations/0031_agent_conversation_agent.py +53 -0
  50. khoj/database/migrations/0031_alter_googleuser_locale.py +30 -0
  51. khoj/database/migrations/0032_merge_20240322_0427.py +14 -0
  52. khoj/database/migrations/0033_rename_tuning_agent_personality.py +17 -0
  53. khoj/database/migrations/0034_alter_chatmodeloptions_chat_model.py +32 -0
  54. khoj/database/migrations/0035_processlock.py +26 -0
  55. khoj/database/migrations/0036_alter_processlock_name.py +19 -0
  56. khoj/database/migrations/0036_delete_offlinechatprocessorconversationconfig.py +15 -0
  57. khoj/database/migrations/0036_publicconversation.py +42 -0
  58. khoj/database/migrations/0037_chatmodeloptions_openai_config_and_more.py +51 -0
  59. khoj/database/migrations/0037_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +32 -0
  60. khoj/database/migrations/0038_merge_20240425_0857.py +14 -0
  61. khoj/database/migrations/0038_merge_20240426_1640.py +12 -0
  62. khoj/database/migrations/0039_merge_20240501_0301.py +12 -0
  63. khoj/database/migrations/0040_alter_processlock_name.py +26 -0
  64. khoj/database/migrations/0040_merge_20240504_1010.py +14 -0
  65. khoj/database/migrations/0041_merge_20240505_1234.py +14 -0
  66. khoj/database/migrations/0042_serverchatsettings.py +46 -0
  67. khoj/database/migrations/0043_alter_chatmodeloptions_model_type.py +21 -0
  68. khoj/database/migrations/0044_conversation_file_filters.py +17 -0
  69. khoj/database/migrations/0045_fileobject.py +37 -0
  70. khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py +22 -0
  71. khoj/database/migrations/0047_alter_entry_file_type.py +31 -0
  72. khoj/database/migrations/0048_voicemodeloption_uservoicemodelconfig.py +52 -0
  73. khoj/database/migrations/0049_datastore.py +38 -0
  74. khoj/database/migrations/0049_texttoimagemodelconfig_api_key_and_more.py +58 -0
  75. khoj/database/migrations/0050_alter_processlock_name.py +25 -0
  76. khoj/database/migrations/0051_merge_20240702_1220.py +14 -0
  77. khoj/database/migrations/0052_alter_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +27 -0
  78. khoj/database/migrations/0053_agent_style_color_agent_style_icon.py +61 -0
  79. khoj/database/migrations/0054_alter_agent_style_color.py +38 -0
  80. khoj/database/migrations/0055_alter_agent_style_icon.py +37 -0
  81. khoj/database/migrations/0056_chatmodeloptions_vision_enabled.py +17 -0
  82. khoj/database/migrations/0056_searchmodelconfig_cross_encoder_model_config.py +17 -0
  83. khoj/database/migrations/0057_merge_20240816_1409.py +13 -0
  84. khoj/database/migrations/0057_remove_serverchatsettings_default_model_and_more.py +51 -0
  85. khoj/database/migrations/0058_alter_chatmodeloptions_chat_model.py +17 -0
  86. khoj/database/migrations/0059_searchmodelconfig_bi_encoder_confidence_threshold.py +17 -0
  87. khoj/database/migrations/0060_merge_20240905_1828.py +14 -0
  88. khoj/database/migrations/0061_alter_chatmodeloptions_model_type.py +26 -0
  89. khoj/database/migrations/0061_alter_texttoimagemodelconfig_model_type.py +21 -0
  90. khoj/database/migrations/0062_merge_20240913_0222.py +14 -0
  91. khoj/database/migrations/0063_conversation_temp_id.py +36 -0
  92. khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +86 -0
  93. khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
  94. khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
  95. khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
  96. khoj/database/migrations/0068_alter_agent_output_modes.py +24 -0
  97. khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py +89 -0
  98. khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +46 -0
  99. khoj/database/migrations/0071_subscription_enabled_trial_at_and_more.py +32 -0
  100. khoj/database/migrations/0072_entry_search_model.py +24 -0
  101. khoj/database/migrations/0073_delete_usersearchmodelconfig.py +15 -0
  102. khoj/database/migrations/0074_alter_conversation_title.py +17 -0
  103. khoj/database/migrations/0075_migrate_generated_assets_and_validate.py +85 -0
  104. khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py +26 -0
  105. khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py +62 -0
  106. khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py +17 -0
  107. khoj/database/migrations/__init__.py +0 -0
  108. khoj/database/models/__init__.py +725 -0
  109. khoj/database/tests.py +3 -0
  110. khoj/interface/compiled/404/index.html +1 -0
  111. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_buildManifest.js +1 -0
  112. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_ssgManifest.js +1 -0
  113. khoj/interface/compiled/_next/static/chunks/1010-8f39bb4648b5ba10.js +1 -0
  114. khoj/interface/compiled/_next/static/chunks/182-f1c48a203dc91e0e.js +20 -0
  115. khoj/interface/compiled/_next/static/chunks/1915-d3c36ad6ce697ce7.js +1 -0
  116. khoj/interface/compiled/_next/static/chunks/2117-165ef4747a5b836b.js +2 -0
  117. khoj/interface/compiled/_next/static/chunks/2581-455000f8aeb08fc3.js +1 -0
  118. khoj/interface/compiled/_next/static/chunks/3727.dcea8f2193111552.js +1 -0
  119. khoj/interface/compiled/_next/static/chunks/3789-a09e37a819171a9d.js +1 -0
  120. khoj/interface/compiled/_next/static/chunks/4124-6c28322ce218d2d5.js +1 -0
  121. khoj/interface/compiled/_next/static/chunks/5427-b52d95253e692bfa.js +1 -0
  122. khoj/interface/compiled/_next/static/chunks/5473-b1cf56dedac6577a.js +1 -0
  123. khoj/interface/compiled/_next/static/chunks/5477-0bbddb79c25a54a7.js +1 -0
  124. khoj/interface/compiled/_next/static/chunks/6065-64db9ad305ba0bcd.js +1 -0
  125. khoj/interface/compiled/_next/static/chunks/6293-469dd16402ea8a6f.js +3 -0
  126. khoj/interface/compiled/_next/static/chunks/688-b5b4391bbc0376f1.js +1 -0
  127. khoj/interface/compiled/_next/static/chunks/8667-b6bf63c72b2d76eb.js +1 -0
  128. khoj/interface/compiled/_next/static/chunks/9259-1172dbaca0515237.js +1 -0
  129. khoj/interface/compiled/_next/static/chunks/94ca1967.1d9b42d929a1ee8c.js +1 -0
  130. khoj/interface/compiled/_next/static/chunks/9597.83583248dfbf6e73.js +1 -0
  131. khoj/interface/compiled/_next/static/chunks/964ecbae.51d6faf8801d15e6.js +1 -0
  132. khoj/interface/compiled/_next/static/chunks/9665-391df1e5c51c960a.js +1 -0
  133. khoj/interface/compiled/_next/static/chunks/app/_not-found/page-a834eddae3e235df.js +1 -0
  134. khoj/interface/compiled/_next/static/chunks/app/agents/layout-e00fb81dca656a10.js +1 -0
  135. khoj/interface/compiled/_next/static/chunks/app/agents/page-28ce086a1129bca2.js +1 -0
  136. khoj/interface/compiled/_next/static/chunks/app/automations/layout-1fe1537449f43496.js +1 -0
  137. khoj/interface/compiled/_next/static/chunks/app/automations/page-bf365a60829d347f.js +1 -0
  138. khoj/interface/compiled/_next/static/chunks/app/chat/layout-33934fc2d6ae6838.js +1 -0
  139. khoj/interface/compiled/_next/static/chunks/app/chat/page-0e476e57eb2015e3.js +1 -0
  140. khoj/interface/compiled/_next/static/chunks/app/layout-30e7fda7262713ce.js +1 -0
  141. khoj/interface/compiled/_next/static/chunks/app/page-a5515ea71aec5ef0.js +1 -0
  142. khoj/interface/compiled/_next/static/chunks/app/search/layout-c02531d586972d7d.js +1 -0
  143. khoj/interface/compiled/_next/static/chunks/app/search/page-9140541e67ea307d.js +1 -0
  144. khoj/interface/compiled/_next/static/chunks/app/settings/layout-d09d6510a45cd4bd.js +1 -0
  145. khoj/interface/compiled/_next/static/chunks/app/settings/page-951ba40b5b94b23a.js +1 -0
  146. khoj/interface/compiled/_next/static/chunks/app/share/chat/layout-e8e5db7830bf3f47.js +1 -0
  147. khoj/interface/compiled/_next/static/chunks/app/share/chat/page-1beb80d8d741c932.js +1 -0
  148. khoj/interface/compiled/_next/static/chunks/d3ac728e-44ebd2a0c99b12a0.js +1 -0
  149. khoj/interface/compiled/_next/static/chunks/fd9d1056-4482b99a36fd1673.js +1 -0
  150. khoj/interface/compiled/_next/static/chunks/framework-8e0e0f4a6b83a956.js +1 -0
  151. khoj/interface/compiled/_next/static/chunks/main-app-de1f09df97a3cfc7.js +1 -0
  152. khoj/interface/compiled/_next/static/chunks/main-db4bfac6b0a8d00b.js +1 -0
  153. khoj/interface/compiled/_next/static/chunks/pages/_app-3c9ca398d360b709.js +1 -0
  154. khoj/interface/compiled/_next/static/chunks/pages/_error-cf5ca766ac8f493f.js +1 -0
  155. khoj/interface/compiled/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
  156. khoj/interface/compiled/_next/static/chunks/webpack-a03962458328b163.js +1 -0
  157. khoj/interface/compiled/_next/static/css/089de1d8526b96e9.css +1 -0
  158. khoj/interface/compiled/_next/static/css/37a73b87f02df402.css +1 -0
  159. khoj/interface/compiled/_next/static/css/4e4e6a4a1c920d06.css +1 -0
  160. khoj/interface/compiled/_next/static/css/8d02837c730f8d13.css +25 -0
  161. khoj/interface/compiled/_next/static/css/8e6a3ca11a60b189.css +1 -0
  162. khoj/interface/compiled/_next/static/css/9c164d9727dd8092.css +1 -0
  163. khoj/interface/compiled/_next/static/css/dac88c17aaee5fcf.css +1 -0
  164. khoj/interface/compiled/_next/static/css/df4b47a2d0d85eae.css +1 -0
  165. khoj/interface/compiled/_next/static/css/e4eb883b5265d372.css +1 -0
  166. khoj/interface/compiled/_next/static/media/1d8a05b60287ae6c-s.p.woff2 +0 -0
  167. khoj/interface/compiled/_next/static/media/6f22fce21a7c433c-s.woff2 +0 -0
  168. khoj/interface/compiled/_next/static/media/77c207b095007c34-s.p.woff2 +0 -0
  169. khoj/interface/compiled/_next/static/media/82ef96de0e8f4d8c-s.p.woff2 +0 -0
  170. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.1608a09b.woff +0 -0
  171. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.4aafdb68.ttf +0 -0
  172. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.a79f1c31.woff2 +0 -0
  173. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.b6770918.woff +0 -0
  174. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.cce5b8ec.ttf +0 -0
  175. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.ec17d132.woff2 +0 -0
  176. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.07ef19e7.ttf +0 -0
  177. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.55fac258.woff2 +0 -0
  178. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.dad44a7f.woff +0 -0
  179. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.9f256b85.woff +0 -0
  180. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.b18f59e1.ttf +0 -0
  181. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.d42a5579.woff2 +0 -0
  182. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.7c187121.woff +0 -0
  183. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.d3c882a6.woff2 +0 -0
  184. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.ed38e79f.ttf +0 -0
  185. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.b74a1a8b.ttf +0 -0
  186. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.c3fb5ac2.woff2 +0 -0
  187. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.d181c465.woff +0 -0
  188. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.6f2bb1df.woff2 +0 -0
  189. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.70d8b0a5.ttf +0 -0
  190. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.e3f82f9d.woff +0 -0
  191. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.47373d1e.ttf +0 -0
  192. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.8916142b.woff2 +0 -0
  193. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.9024d815.woff +0 -0
  194. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.0462f03b.woff2 +0 -0
  195. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.7f51fe03.woff +0 -0
  196. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.b7f8fe9b.ttf +0 -0
  197. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.572d331f.woff2 +0 -0
  198. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.a879cf83.ttf +0 -0
  199. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.f1035d8d.woff +0 -0
  200. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.5295ba48.woff +0 -0
  201. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.939bc644.ttf +0 -0
  202. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.f28c23ac.woff2 +0 -0
  203. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.8c5b5494.woff2 +0 -0
  204. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.94e1e8dc.ttf +0 -0
  205. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.bf59d231.woff +0 -0
  206. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.3b1e59b3.woff2 +0 -0
  207. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.7c9bc82b.woff +0 -0
  208. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.b4c20c84.ttf +0 -0
  209. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.74048478.woff +0 -0
  210. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.ba21ed5f.woff2 +0 -0
  211. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.d4d7ba48.ttf +0 -0
  212. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.03e9641d.woff2 +0 -0
  213. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.07505710.woff +0 -0
  214. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.fe9cbbe1.ttf +0 -0
  215. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.e1e279cb.woff +0 -0
  216. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.eae34984.woff2 +0 -0
  217. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.fabc004a.ttf +0 -0
  218. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.57727022.woff +0 -0
  219. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.5916a24f.woff2 +0 -0
  220. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.d6b476ec.ttf +0 -0
  221. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.9acaf01c.woff +0 -0
  222. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.a144ef58.ttf +0 -0
  223. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.b4230e7e.woff2 +0 -0
  224. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.10d95fd3.woff2 +0 -0
  225. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.7a996c9d.woff +0 -0
  226. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.fbccdabe.ttf +0 -0
  227. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.6258592b.woff +0 -0
  228. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.a8709e36.woff2 +0 -0
  229. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.d97aaf4a.ttf +0 -0
  230. khoj/interface/compiled/_next/static/media/a6ecd16fa044d500-s.p.woff2 +0 -0
  231. khoj/interface/compiled/_next/static/media/bd82c78e5b7b3fe9-s.p.woff2 +0 -0
  232. khoj/interface/compiled/_next/static/media/c32c8052c071fc42-s.woff2 +0 -0
  233. khoj/interface/compiled/_next/static/media/c4250770ab8708b6-s.p.woff2 +0 -0
  234. khoj/interface/compiled/_next/static/media/e098aaaecc9cfbb2-s.p.woff2 +0 -0
  235. khoj/interface/compiled/_next/static/media/flags.3afdda2f.webp +0 -0
  236. khoj/interface/compiled/_next/static/media/flags@2x.5fbe9fc1.webp +0 -0
  237. khoj/interface/compiled/_next/static/media/globe.98e105ca.webp +0 -0
  238. khoj/interface/compiled/_next/static/media/globe@2x.974df6f8.webp +0 -0
  239. khoj/interface/compiled/agents/index.html +1 -0
  240. khoj/interface/compiled/agents/index.txt +7 -0
  241. khoj/interface/compiled/agents.svg +6 -0
  242. khoj/interface/compiled/assets/icons/khoj_lantern.ico +0 -0
  243. khoj/interface/compiled/assets/icons/khoj_lantern.svg +100 -0
  244. khoj/interface/compiled/assets/icons/khoj_lantern_1200x1200.png +0 -0
  245. khoj/interface/compiled/assets/icons/khoj_lantern_128x128.png +0 -0
  246. khoj/interface/compiled/assets/icons/khoj_lantern_128x128_dark.png +0 -0
  247. khoj/interface/compiled/assets/icons/khoj_lantern_256x256.png +0 -0
  248. khoj/interface/compiled/assets/icons/khoj_lantern_512x512.png +0 -0
  249. khoj/interface/compiled/assets/icons/khoj_lantern_logomarktype_1200x630.png +0 -0
  250. khoj/interface/compiled/assets/samples/desktop-browse-draw-sample.png +0 -0
  251. khoj/interface/compiled/assets/samples/desktop-plain-chat-sample.png +0 -0
  252. khoj/interface/compiled/assets/samples/desktop-remember-plan-sample.png +0 -0
  253. khoj/interface/compiled/assets/samples/phone-browse-draw-sample.png +0 -0
  254. khoj/interface/compiled/assets/samples/phone-plain-chat-sample.png +0 -0
  255. khoj/interface/compiled/assets/samples/phone-remember-plan-sample.png +0 -0
  256. khoj/interface/compiled/automation.svg +37 -0
  257. khoj/interface/compiled/automations/index.html +1 -0
  258. khoj/interface/compiled/automations/index.txt +8 -0
  259. khoj/interface/compiled/chat/index.html +1 -0
  260. khoj/interface/compiled/chat/index.txt +7 -0
  261. khoj/interface/compiled/chat.svg +24 -0
  262. khoj/interface/compiled/close.svg +5 -0
  263. khoj/interface/compiled/copy-button-success.svg +6 -0
  264. khoj/interface/compiled/copy-button.svg +5 -0
  265. khoj/interface/compiled/index.html +1 -0
  266. khoj/interface/compiled/index.txt +7 -0
  267. khoj/interface/compiled/khoj.webmanifest +76 -0
  268. khoj/interface/compiled/logo.svg +24 -0
  269. khoj/interface/compiled/search/index.html +1 -0
  270. khoj/interface/compiled/search/index.txt +7 -0
  271. khoj/interface/compiled/send.svg +1 -0
  272. khoj/interface/compiled/settings/index.html +1 -0
  273. khoj/interface/compiled/settings/index.txt +9 -0
  274. khoj/interface/compiled/share/chat/index.html +1 -0
  275. khoj/interface/compiled/share/chat/index.txt +7 -0
  276. khoj/interface/compiled/share.svg +8 -0
  277. khoj/interface/compiled/thumbs-down.svg +6 -0
  278. khoj/interface/compiled/thumbs-up.svg +6 -0
  279. khoj/interface/email/feedback.html +34 -0
  280. khoj/interface/email/magic_link.html +40 -0
  281. khoj/interface/email/task.html +37 -0
  282. khoj/interface/email/welcome.html +90 -0
  283. khoj/interface/web/.well-known/assetlinks.json +11 -0
  284. khoj/interface/web/assets/icons/agents.svg +19 -0
  285. khoj/interface/web/assets/icons/automation.svg +43 -0
  286. khoj/interface/web/assets/icons/chat.svg +24 -0
  287. khoj/interface/web/assets/icons/github.svg +1 -0
  288. khoj/interface/web/assets/icons/khoj-logo-sideways-200.png +0 -0
  289. khoj/interface/web/assets/icons/khoj-logo-sideways-500.png +0 -0
  290. khoj/interface/web/assets/icons/khoj-logo-sideways.svg +32 -0
  291. khoj/interface/web/assets/icons/khoj.svg +26 -0
  292. khoj/interface/web/assets/icons/logotype.svg +1 -0
  293. khoj/interface/web/assets/icons/search.svg +57 -0
  294. khoj/interface/web/assets/icons/sync.svg +4 -0
  295. khoj/interface/web/assets/khoj.css +237 -0
  296. khoj/interface/web/assets/utils.js +33 -0
  297. khoj/interface/web/base_config.html +445 -0
  298. khoj/interface/web/content_source_github_input.html +208 -0
  299. khoj/interface/web/login.html +310 -0
  300. khoj/interface/web/utils.html +48 -0
  301. khoj/main.py +249 -0
  302. khoj/manage.py +22 -0
  303. khoj/migrations/__init__.py +0 -0
  304. khoj/migrations/migrate_offline_chat_default_model.py +69 -0
  305. khoj/migrations/migrate_offline_chat_default_model_2.py +71 -0
  306. khoj/migrations/migrate_offline_chat_schema.py +83 -0
  307. khoj/migrations/migrate_offline_model.py +29 -0
  308. khoj/migrations/migrate_processor_config_openai.py +67 -0
  309. khoj/migrations/migrate_server_pg.py +132 -0
  310. khoj/migrations/migrate_version.py +17 -0
  311. khoj/processor/__init__.py +0 -0
  312. khoj/processor/content/__init__.py +0 -0
  313. khoj/processor/content/docx/__init__.py +0 -0
  314. khoj/processor/content/docx/docx_to_entries.py +111 -0
  315. khoj/processor/content/github/__init__.py +0 -0
  316. khoj/processor/content/github/github_to_entries.py +226 -0
  317. khoj/processor/content/images/__init__.py +0 -0
  318. khoj/processor/content/images/image_to_entries.py +117 -0
  319. khoj/processor/content/markdown/__init__.py +0 -0
  320. khoj/processor/content/markdown/markdown_to_entries.py +160 -0
  321. khoj/processor/content/notion/notion_to_entries.py +259 -0
  322. khoj/processor/content/org_mode/__init__.py +0 -0
  323. khoj/processor/content/org_mode/org_to_entries.py +226 -0
  324. khoj/processor/content/org_mode/orgnode.py +532 -0
  325. khoj/processor/content/pdf/__init__.py +0 -0
  326. khoj/processor/content/pdf/pdf_to_entries.py +119 -0
  327. khoj/processor/content/plaintext/__init__.py +0 -0
  328. khoj/processor/content/plaintext/plaintext_to_entries.py +117 -0
  329. khoj/processor/content/text_to_entries.py +296 -0
  330. khoj/processor/conversation/__init__.py +0 -0
  331. khoj/processor/conversation/anthropic/__init__.py +0 -0
  332. khoj/processor/conversation/anthropic/anthropic_chat.py +243 -0
  333. khoj/processor/conversation/anthropic/utils.py +217 -0
  334. khoj/processor/conversation/google/__init__.py +0 -0
  335. khoj/processor/conversation/google/gemini_chat.py +253 -0
  336. khoj/processor/conversation/google/utils.py +260 -0
  337. khoj/processor/conversation/offline/__init__.py +0 -0
  338. khoj/processor/conversation/offline/chat_model.py +308 -0
  339. khoj/processor/conversation/offline/utils.py +80 -0
  340. khoj/processor/conversation/offline/whisper.py +15 -0
  341. khoj/processor/conversation/openai/__init__.py +0 -0
  342. khoj/processor/conversation/openai/gpt.py +243 -0
  343. khoj/processor/conversation/openai/utils.py +232 -0
  344. khoj/processor/conversation/openai/whisper.py +13 -0
  345. khoj/processor/conversation/prompts.py +1188 -0
  346. khoj/processor/conversation/utils.py +867 -0
  347. khoj/processor/embeddings.py +122 -0
  348. khoj/processor/image/generate.py +215 -0
  349. khoj/processor/speech/__init__.py +0 -0
  350. khoj/processor/speech/text_to_speech.py +51 -0
  351. khoj/processor/tools/__init__.py +0 -0
  352. khoj/processor/tools/online_search.py +472 -0
  353. khoj/processor/tools/run_code.py +179 -0
  354. khoj/routers/__init__.py +0 -0
  355. khoj/routers/api.py +760 -0
  356. khoj/routers/api_agents.py +295 -0
  357. khoj/routers/api_chat.py +1273 -0
  358. khoj/routers/api_content.py +634 -0
  359. khoj/routers/api_model.py +123 -0
  360. khoj/routers/api_phone.py +86 -0
  361. khoj/routers/api_subscription.py +144 -0
  362. khoj/routers/auth.py +307 -0
  363. khoj/routers/email.py +135 -0
  364. khoj/routers/helpers.py +2333 -0
  365. khoj/routers/notion.py +85 -0
  366. khoj/routers/research.py +364 -0
  367. khoj/routers/storage.py +63 -0
  368. khoj/routers/twilio.py +36 -0
  369. khoj/routers/web_client.py +141 -0
  370. khoj/search_filter/__init__.py +0 -0
  371. khoj/search_filter/base_filter.py +15 -0
  372. khoj/search_filter/date_filter.py +215 -0
  373. khoj/search_filter/file_filter.py +32 -0
  374. khoj/search_filter/word_filter.py +29 -0
  375. khoj/search_type/__init__.py +0 -0
  376. khoj/search_type/text_search.py +255 -0
  377. khoj/utils/__init__.py +0 -0
  378. khoj/utils/cli.py +101 -0
  379. khoj/utils/config.py +81 -0
  380. khoj/utils/constants.py +51 -0
  381. khoj/utils/fs_syncer.py +252 -0
  382. khoj/utils/helpers.py +627 -0
  383. khoj/utils/initialization.py +301 -0
  384. khoj/utils/jsonl.py +43 -0
  385. khoj/utils/models.py +47 -0
  386. khoj/utils/rawconfig.py +208 -0
  387. khoj/utils/state.py +48 -0
  388. khoj/utils/yaml.py +47 -0
  389. khoj-1.33.3.dev32.dist-info/METADATA +190 -0
  390. khoj-1.33.3.dev32.dist-info/RECORD +393 -0
  391. khoj-1.33.3.dev32.dist-info/WHEEL +4 -0
  392. khoj-1.33.3.dev32.dist-info/entry_points.txt +2 -0
  393. khoj-1.33.3.dev32.dist-info/licenses/LICENSE +661 -0
khoj/routers/api.py ADDED
@@ -0,0 +1,760 @@
1
+ import concurrent.futures
2
+ import json
3
+ import logging
4
+ import math
5
+ import os
6
+ import threading
7
+ import time
8
+ import uuid
9
+ from typing import Any, Callable, List, Optional, Set, Union
10
+
11
+ import cron_descriptor
12
+ import pytz
13
+ from apscheduler.job import Job
14
+ from apscheduler.triggers.cron import CronTrigger
15
+ from asgiref.sync import sync_to_async
16
+ from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
17
+ from fastapi.requests import Request
18
+ from fastapi.responses import Response
19
+ from starlette.authentication import has_required_scope, requires
20
+
21
+ from khoj.configure import initialize_content
22
+ from khoj.database import adapters
23
+ from khoj.database.adapters import (
24
+ AgentAdapters,
25
+ AutomationAdapters,
26
+ ConversationAdapters,
27
+ EntryAdapters,
28
+ get_default_search_model,
29
+ get_user_photo,
30
+ )
31
+ from khoj.database.models import Agent, ChatModel, KhojUser, SpeechToTextModelOptions
32
+ from khoj.processor.conversation import prompts
33
+ from khoj.processor.conversation.anthropic.anthropic_chat import (
34
+ extract_questions_anthropic,
35
+ )
36
+ from khoj.processor.conversation.google.gemini_chat import extract_questions_gemini
37
+ from khoj.processor.conversation.offline.chat_model import extract_questions_offline
38
+ from khoj.processor.conversation.offline.whisper import transcribe_audio_offline
39
+ from khoj.processor.conversation.openai.gpt import extract_questions
40
+ from khoj.processor.conversation.openai.whisper import transcribe_audio
41
+ from khoj.processor.conversation.utils import clean_json, defilter_query
42
+ from khoj.routers.helpers import (
43
+ ApiUserRateLimiter,
44
+ ChatEvent,
45
+ CommonQueryParams,
46
+ ConversationCommandRateLimiter,
47
+ get_user_config,
48
+ schedule_automation,
49
+ schedule_query,
50
+ update_telemetry_state,
51
+ )
52
+ from khoj.search_filter.date_filter import DateFilter
53
+ from khoj.search_filter.file_filter import FileFilter
54
+ from khoj.search_filter.word_filter import WordFilter
55
+ from khoj.search_type import text_search
56
+ from khoj.utils import state
57
+ from khoj.utils.config import OfflineChatProcessorModel
58
+ from khoj.utils.helpers import ConversationCommand, is_none_or_empty, timer
59
+ from khoj.utils.rawconfig import LocationData, SearchResponse
60
+ from khoj.utils.state import SearchType
61
+
62
+ # Initialize Router
63
+ api = APIRouter()
64
+ logger = logging.getLogger(__name__)
65
+ conversation_command_rate_limiter = ConversationCommandRateLimiter(
66
+ trial_rate_limit=2, subscribed_rate_limit=100, slug="command"
67
+ )
68
+
69
+
70
+ @api.get("/search", response_model=List[SearchResponse])
71
+ @requires(["authenticated"])
72
+ async def search(
73
+ q: str,
74
+ request: Request,
75
+ common: CommonQueryParams,
76
+ n: Optional[int] = 5,
77
+ t: Optional[SearchType] = SearchType.All,
78
+ r: Optional[bool] = False,
79
+ max_distance: Optional[Union[float, None]] = None,
80
+ dedupe: Optional[bool] = True,
81
+ ):
82
+ user = request.user.object
83
+
84
+ results = await execute_search(
85
+ user=user,
86
+ q=q,
87
+ n=n,
88
+ t=t,
89
+ r=r,
90
+ max_distance=max_distance or math.inf,
91
+ dedupe=dedupe,
92
+ )
93
+
94
+ update_telemetry_state(
95
+ request=request,
96
+ telemetry_type="api",
97
+ api="search",
98
+ **common.__dict__,
99
+ )
100
+
101
+ return results
102
+
103
+
104
+ async def execute_search(
105
+ user: KhojUser,
106
+ q: str,
107
+ n: Optional[int] = 5,
108
+ t: Optional[SearchType] = SearchType.All,
109
+ r: Optional[bool] = False,
110
+ max_distance: Optional[Union[float, None]] = None,
111
+ dedupe: Optional[bool] = True,
112
+ agent: Optional[Agent] = None,
113
+ ):
114
+ # Run validation checks
115
+ results: List[SearchResponse] = []
116
+
117
+ start_time = time.time()
118
+
119
+ # Ensure the agent, if present, is accessible by the user
120
+ if user and agent and not await AgentAdapters.ais_agent_accessible(agent, user):
121
+ logger.error(f"Agent {agent.slug} is not accessible by user {user}")
122
+ return results
123
+
124
+ if q is None or q == "":
125
+ logger.warning(f"No query param (q) passed in API call to initiate search")
126
+ return results
127
+
128
+ # initialize variables
129
+ user_query = q.strip()
130
+ results_count = n or 5
131
+ search_futures: List[concurrent.futures.Future] = []
132
+
133
+ # return cached results, if available
134
+ if user:
135
+ query_cache_key = f"{user_query}-{n}-{t}-{r}-{max_distance}-{dedupe}"
136
+ if query_cache_key in state.query_cache[user.uuid]:
137
+ logger.debug(f"Return response from query cache")
138
+ return state.query_cache[user.uuid][query_cache_key]
139
+
140
+ # Encode query with filter terms removed
141
+ defiltered_query = user_query
142
+ for filter in [DateFilter(), WordFilter(), FileFilter()]:
143
+ defiltered_query = filter.defilter(defiltered_query)
144
+
145
+ encoded_asymmetric_query = None
146
+ if t != SearchType.Image:
147
+ with timer("Encoding query took", logger=logger):
148
+ search_model = await sync_to_async(get_default_search_model)()
149
+ encoded_asymmetric_query = state.embeddings_model[search_model.name].embed_query(defiltered_query)
150
+
151
+ with concurrent.futures.ThreadPoolExecutor() as executor:
152
+ if t in [
153
+ SearchType.All,
154
+ SearchType.Org,
155
+ SearchType.Markdown,
156
+ SearchType.Github,
157
+ SearchType.Notion,
158
+ SearchType.Plaintext,
159
+ SearchType.Pdf,
160
+ ]:
161
+ # query markdown notes
162
+ search_futures += [
163
+ executor.submit(
164
+ text_search.query,
165
+ user_query,
166
+ user,
167
+ t,
168
+ question_embedding=encoded_asymmetric_query,
169
+ max_distance=max_distance,
170
+ agent=agent,
171
+ )
172
+ ]
173
+
174
+ # Query across each requested content types in parallel
175
+ with timer("Query took", logger):
176
+ for search_future in concurrent.futures.as_completed(search_futures):
177
+ hits = await search_future.result()
178
+ # Collate results
179
+ results += text_search.collate_results(hits, dedupe=dedupe)
180
+
181
+ # Sort results across all content types and take top results
182
+ results = text_search.rerank_and_sort_results(
183
+ results, query=defiltered_query, rank_results=r, search_model_name=search_model.name
184
+ )[:results_count]
185
+
186
+ # Cache results
187
+ if user:
188
+ state.query_cache[user.uuid][query_cache_key] = results
189
+
190
+ end_time = time.time()
191
+ logger.debug(f"🔍 Search took: {end_time - start_time:.3f} seconds")
192
+
193
+ return results
194
+
195
+
196
+ @api.get("/update")
197
+ @requires(["authenticated"])
198
+ def update(
199
+ request: Request,
200
+ common: CommonQueryParams,
201
+ t: Optional[SearchType] = None,
202
+ force: Optional[bool] = False,
203
+ ):
204
+ user = request.user.object
205
+ if not state.config:
206
+ error_msg = f"🚨 Khoj is not configured.\nConfigure it via http://localhost:42110/settings, plugins or by editing {state.config_file}."
207
+ logger.warning(error_msg)
208
+ raise HTTPException(status_code=500, detail=error_msg)
209
+ try:
210
+ initialize_content(user=user, regenerate=force, search_type=t)
211
+ except Exception as e:
212
+ error_msg = f"🚨 Failed to update server via API: {e}"
213
+ logger.error(error_msg, exc_info=True)
214
+ raise HTTPException(status_code=500, detail=error_msg)
215
+ else:
216
+ components = []
217
+ if state.search_models:
218
+ components.append("Search models")
219
+ components_msg = ", ".join(components)
220
+ logger.info(f"📪 {components_msg} updated via API")
221
+
222
+ update_telemetry_state(
223
+ request=request,
224
+ telemetry_type="api",
225
+ api="update",
226
+ **common.__dict__,
227
+ )
228
+
229
+ return {"status": "ok", "message": "khoj reloaded"}
230
+
231
+
232
+ @api.post("/transcribe")
233
+ @requires(["authenticated"])
234
+ async def transcribe(
235
+ request: Request,
236
+ common: CommonQueryParams,
237
+ file: UploadFile = File(...),
238
+ rate_limiter_per_minute=Depends(
239
+ ApiUserRateLimiter(requests=20, subscribed_requests=20, window=60, slug="transcribe_minute")
240
+ ),
241
+ rate_limiter_per_day=Depends(
242
+ ApiUserRateLimiter(requests=60, subscribed_requests=600, window=60 * 60 * 24, slug="transcribe_day")
243
+ ),
244
+ ):
245
+ user: KhojUser = request.user.object
246
+ audio_filename = f"{user.uuid}-{str(uuid.uuid4())}.webm"
247
+ user_message: str = None
248
+
249
+ # If the file is too large, return an unprocessable entity error
250
+ if file.size > 10 * 1024 * 1024:
251
+ logger.warning(f"Audio file too large to transcribe. Audio file size: {file.size}. Exceeds 10Mb limit.")
252
+ return Response(content="Audio size larger than 10Mb limit", status_code=422)
253
+
254
+ # Transcribe the audio from the request
255
+ try:
256
+ # Store the audio from the request in a temporary file
257
+ audio_data = await file.read()
258
+ with open(audio_filename, "wb") as audio_file_writer:
259
+ audio_file_writer.write(audio_data)
260
+ audio_file = open(audio_filename, "rb")
261
+
262
+ # Send the audio data to the Whisper API
263
+ speech_to_text_config = await ConversationAdapters.get_speech_to_text_config()
264
+ if not speech_to_text_config:
265
+ # If the user has not configured a speech to text model, return an unsupported on server error
266
+ status_code = 501
267
+ elif state.openai_client and speech_to_text_config.model_type == SpeechToTextModelOptions.ModelType.OPENAI:
268
+ speech2text_model = speech_to_text_config.model_name
269
+ user_message = await transcribe_audio(audio_file, speech2text_model, client=state.openai_client)
270
+ elif speech_to_text_config.model_type == SpeechToTextModelOptions.ModelType.OFFLINE:
271
+ speech2text_model = speech_to_text_config.model_name
272
+ user_message = await transcribe_audio_offline(audio_filename, speech2text_model)
273
+ finally:
274
+ # Close and Delete the temporary audio file
275
+ audio_file.close()
276
+ os.remove(audio_filename)
277
+
278
+ if user_message is None:
279
+ return Response(status_code=status_code or 500)
280
+
281
+ update_telemetry_state(
282
+ request=request,
283
+ telemetry_type="api",
284
+ api="transcribe",
285
+ **common.__dict__,
286
+ )
287
+
288
+ # Return the spoken text
289
+ content = json.dumps({"text": user_message})
290
+ return Response(content=content, media_type="application/json", status_code=200)
291
+
292
+
293
+ @api.get("/settings", response_class=Response)
294
+ @requires(["authenticated"])
295
+ def get_settings(request: Request, detailed: Optional[bool] = False) -> Response:
296
+ user = request.user.object
297
+ user_config = get_user_config(user, request, is_detailed=detailed)
298
+ del user_config["request"]
299
+
300
+ # Return config data as a JSON response
301
+ return Response(content=json.dumps(user_config), media_type="application/json", status_code=200)
302
+
303
+
304
+ @api.patch("/user/name", status_code=200)
305
+ @requires(["authenticated"])
306
+ def set_user_name(
307
+ request: Request,
308
+ name: str,
309
+ client: Optional[str] = None,
310
+ ):
311
+ user = request.user.object
312
+
313
+ split_name = name.split(" ")
314
+
315
+ if len(split_name) > 2:
316
+ raise HTTPException(status_code=400, detail="Name must be in the format: Firstname Lastname")
317
+
318
+ if len(split_name) == 1:
319
+ first_name = split_name[0]
320
+ last_name = ""
321
+ else:
322
+ first_name, last_name = split_name[0], split_name[-1]
323
+
324
+ adapters.set_user_name(user, first_name, last_name)
325
+
326
+ update_telemetry_state(
327
+ request=request,
328
+ telemetry_type="api",
329
+ api="set_user_name",
330
+ client=client,
331
+ )
332
+
333
+ return {"status": "ok"}
334
+
335
+
336
+ async def extract_references_and_questions(
337
+ request: Request,
338
+ meta_log: dict,
339
+ q: str,
340
+ n: int,
341
+ d: float,
342
+ conversation_id: str,
343
+ conversation_commands: List[ConversationCommand] = [ConversationCommand.Default],
344
+ location_data: LocationData = None,
345
+ send_status_func: Optional[Callable] = None,
346
+ query_images: Optional[List[str]] = None,
347
+ previous_inferred_queries: Set = set(),
348
+ agent: Agent = None,
349
+ query_files: str = None,
350
+ tracer: dict = {},
351
+ ):
352
+ user = request.user.object if request.user.is_authenticated else None
353
+
354
+ # Initialize Variables
355
+ compiled_references: List[dict[str, str]] = []
356
+ inferred_queries: List[str] = []
357
+
358
+ agent_has_entries = False
359
+
360
+ if agent:
361
+ agent_has_entries = await sync_to_async(EntryAdapters.agent_has_entries)(agent=agent)
362
+
363
+ if (
364
+ not ConversationCommand.Notes in conversation_commands
365
+ and not ConversationCommand.Default in conversation_commands
366
+ and not agent_has_entries
367
+ ):
368
+ yield compiled_references, inferred_queries, q
369
+ return
370
+
371
+ # If Notes or Default is not in the conversation command, then the search should be restricted to the agent's knowledge base
372
+ should_limit_to_agent_knowledge = (
373
+ ConversationCommand.Notes not in conversation_commands
374
+ and ConversationCommand.Default not in conversation_commands
375
+ )
376
+
377
+ if not await sync_to_async(EntryAdapters.user_has_entries)(user=user):
378
+ if not agent_has_entries:
379
+ logger.debug("No documents in knowledge base. Use a Khoj client to sync and chat with your docs.")
380
+ yield compiled_references, inferred_queries, q
381
+ return
382
+
383
+ # Extract filter terms from user message
384
+ defiltered_query = defilter_query(q)
385
+ filters_in_query = q.replace(defiltered_query, "").strip()
386
+ conversation = await sync_to_async(ConversationAdapters.get_conversation_by_id)(conversation_id)
387
+
388
+ if not conversation:
389
+ logger.error(f"Conversation with id {conversation_id} not found when extracting references.")
390
+ yield compiled_references, inferred_queries, defiltered_query
391
+ return
392
+
393
+ filters_in_query += " ".join([f'file:"{filter}"' for filter in conversation.file_filters])
394
+ using_offline_chat = False
395
+ if is_none_or_empty(filters_in_query):
396
+ logger.debug(f"Filters in query: {filters_in_query}")
397
+
398
+ personality_context = prompts.personality_context.format(personality=agent.personality) if agent else ""
399
+
400
+ # Infer search queries from user message
401
+ with timer("Extracting search queries took", logger):
402
+ # If we've reached here, either the user has enabled offline chat or the openai model is enabled.
403
+ chat_model = await ConversationAdapters.aget_default_chat_model(user)
404
+ vision_enabled = chat_model.vision_enabled
405
+
406
+ if chat_model.model_type == ChatModel.ModelType.OFFLINE:
407
+ using_offline_chat = True
408
+ chat_model_name = chat_model.name
409
+ max_tokens = chat_model.max_prompt_size
410
+ if state.offline_chat_processor_config is None:
411
+ state.offline_chat_processor_config = OfflineChatProcessorModel(chat_model_name, max_tokens)
412
+
413
+ loaded_model = state.offline_chat_processor_config.loaded_model
414
+
415
+ inferred_queries = extract_questions_offline(
416
+ defiltered_query,
417
+ model=chat_model,
418
+ loaded_model=loaded_model,
419
+ conversation_log=meta_log,
420
+ should_extract_questions=True,
421
+ location_data=location_data,
422
+ user=user,
423
+ max_prompt_size=chat_model.max_prompt_size,
424
+ personality_context=personality_context,
425
+ query_files=query_files,
426
+ tracer=tracer,
427
+ )
428
+ elif chat_model.model_type == ChatModel.ModelType.OPENAI:
429
+ api_key = chat_model.ai_model_api.api_key
430
+ base_url = chat_model.ai_model_api.api_base_url
431
+ chat_model_name = chat_model.name
432
+ inferred_queries = extract_questions(
433
+ defiltered_query,
434
+ model=chat_model_name,
435
+ api_key=api_key,
436
+ api_base_url=base_url,
437
+ conversation_log=meta_log,
438
+ location_data=location_data,
439
+ user=user,
440
+ query_images=query_images,
441
+ vision_enabled=vision_enabled,
442
+ personality_context=personality_context,
443
+ query_files=query_files,
444
+ tracer=tracer,
445
+ )
446
+ elif chat_model.model_type == ChatModel.ModelType.ANTHROPIC:
447
+ api_key = chat_model.ai_model_api.api_key
448
+ chat_model_name = chat_model.name
449
+ inferred_queries = extract_questions_anthropic(
450
+ defiltered_query,
451
+ query_images=query_images,
452
+ model=chat_model_name,
453
+ api_key=api_key,
454
+ conversation_log=meta_log,
455
+ location_data=location_data,
456
+ user=user,
457
+ vision_enabled=vision_enabled,
458
+ personality_context=personality_context,
459
+ query_files=query_files,
460
+ tracer=tracer,
461
+ )
462
+ elif chat_model.model_type == ChatModel.ModelType.GOOGLE:
463
+ api_key = chat_model.ai_model_api.api_key
464
+ chat_model_name = chat_model.name
465
+ inferred_queries = extract_questions_gemini(
466
+ defiltered_query,
467
+ query_images=query_images,
468
+ model=chat_model_name,
469
+ api_key=api_key,
470
+ conversation_log=meta_log,
471
+ location_data=location_data,
472
+ max_tokens=chat_model.max_prompt_size,
473
+ user=user,
474
+ vision_enabled=vision_enabled,
475
+ personality_context=personality_context,
476
+ query_files=query_files,
477
+ tracer=tracer,
478
+ )
479
+
480
+ # Collate search results as context for GPT
481
+ inferred_queries = list(set(inferred_queries) - previous_inferred_queries)
482
+ with timer("Searching knowledge base took", logger):
483
+ search_results = []
484
+ logger.info(f"🔍 Searching knowledge base with queries: {inferred_queries}")
485
+ if send_status_func:
486
+ inferred_queries_str = "\n- " + "\n- ".join(inferred_queries)
487
+ async for event in send_status_func(f"**Searching Documents for:** {inferred_queries_str}"):
488
+ yield {ChatEvent.STATUS: event}
489
+ for query in inferred_queries:
490
+ n_items = min(n, 3) if using_offline_chat else n
491
+ search_results.extend(
492
+ await execute_search(
493
+ user if not should_limit_to_agent_knowledge else None,
494
+ f"{query} {filters_in_query}",
495
+ n=n_items,
496
+ t=SearchType.All,
497
+ r=True,
498
+ max_distance=d,
499
+ dedupe=False,
500
+ agent=agent,
501
+ )
502
+ )
503
+ search_results = text_search.deduplicated_search_responses(search_results)
504
+ compiled_references = [
505
+ {"query": q, "compiled": item.additional["compiled"], "file": item.additional["file"]}
506
+ for q, item in zip(inferred_queries, search_results)
507
+ ]
508
+
509
+ yield compiled_references, inferred_queries, defiltered_query
510
+
511
+
512
+ @api.get("/health", response_class=Response)
513
+ @requires(["authenticated"], status_code=200)
514
+ def health_check(request: Request) -> Response:
515
+ response_obj = {"email": request.user.object.email}
516
+ return Response(content=json.dumps(response_obj), media_type="application/json", status_code=200)
517
+
518
+
519
+ @api.get("/v1/user", response_class=Response)
520
+ @requires(["authenticated"])
521
+ def user_info(request: Request) -> Response:
522
+ # Get user information
523
+ user: KhojUser = request.user.object
524
+ user_picture = get_user_photo(user=user)
525
+ is_active = has_required_scope(request, ["premium"])
526
+ has_documents = EntryAdapters.user_has_entries(user=user)
527
+
528
+ # Collect user information in a dictionary
529
+ user_info = {
530
+ "email": user.email,
531
+ "username": user.username,
532
+ "photo": user_picture,
533
+ "is_active": is_active,
534
+ "has_documents": has_documents,
535
+ "khoj_version": state.khoj_version,
536
+ }
537
+
538
+ # Return user information as a JSON response
539
+ return Response(content=json.dumps(user_info), media_type="application/json", status_code=200)
540
+
541
+
542
+ @api.get("/automations", response_class=Response)
543
+ @requires(["authenticated"])
544
+ def get_automations(request: Request) -> Response:
545
+ user: KhojUser = request.user.object
546
+
547
+ # Collate all automations created by user that are still active
548
+ automations_info = [automation_info for automation_info in AutomationAdapters.get_automations_metadata(user)]
549
+
550
+ # Return tasks information as a JSON response
551
+ return Response(content=json.dumps(automations_info), media_type="application/json", status_code=200)
552
+
553
+
554
+ @api.delete("/automation", response_class=Response)
555
+ @requires(["authenticated"])
556
+ def delete_automation(request: Request, automation_id: str) -> Response:
557
+ user: KhojUser = request.user.object
558
+
559
+ try:
560
+ automation_info = AutomationAdapters.delete_automation(user, automation_id)
561
+ except ValueError:
562
+ return Response(status_code=204)
563
+
564
+ # Return deleted automation information as a JSON response
565
+ return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200)
566
+
567
+
568
+ @api.post("/automation", response_class=Response)
569
+ @requires(["authenticated"])
570
+ def post_automation(
571
+ request: Request,
572
+ q: str,
573
+ crontime: str,
574
+ subject: Optional[str] = None,
575
+ city: Optional[str] = None,
576
+ region: Optional[str] = None,
577
+ country: Optional[str] = None,
578
+ timezone: Optional[str] = None,
579
+ ) -> Response:
580
+ user: KhojUser = request.user.object
581
+
582
+ # Perform validation checks
583
+ if is_none_or_empty(q) or is_none_or_empty(crontime):
584
+ return Response(content="A query and crontime is required", status_code=400)
585
+ if not cron_descriptor.get_description(crontime):
586
+ return Response(content="Invalid crontime", status_code=400)
587
+
588
+ # Infer subject, query to run
589
+ _, query_to_run, generated_subject = schedule_query(q, conversation_history={}, user=user)
590
+ subject = subject or generated_subject
591
+
592
+ # Normalize query parameters
593
+ # Add /automated_task prefix to query if not present
594
+ query_to_run = query_to_run.strip()
595
+ if not query_to_run.startswith("/automated_task"):
596
+ query_to_run = f"/automated_task {query_to_run}"
597
+
598
+ # Normalize crontime for AP Scheduler CronTrigger
599
+ crontime = crontime.strip()
600
+ if len(crontime.split(" ")) > 5:
601
+ # Truncate crontime to 5 fields
602
+ crontime = " ".join(crontime.split(" ")[:5])
603
+
604
+ # Convert crontime to standard unix crontime
605
+ crontime = crontime.replace("?", "*")
606
+
607
+ # Disallow minute level automation recurrence
608
+ minute_value = crontime.split(" ")[0]
609
+ if not minute_value.isdigit():
610
+ return Response(
611
+ content="Minute level recurrence is unsupported. Please create a less frequent schedule.",
612
+ status_code=400,
613
+ )
614
+
615
+ # Create new Conversation Session associated with this new task
616
+ title = f"Automation: {subject}"
617
+ conversation = ConversationAdapters.create_conversation_session(user, request.user.client_app, title=title)
618
+
619
+ # Schedule automation with query_to_run, timezone, subject directly provided by user
620
+ try:
621
+ # Use the query to run as the scheduling request if the scheduling request is unset
622
+ calling_url = request.url.replace(query=f"{request.url.query}")
623
+ automation = schedule_automation(
624
+ query_to_run, subject, crontime, timezone, q, user, calling_url, str(conversation.id)
625
+ )
626
+ except Exception as e:
627
+ logger.error(f"Error creating automation {q} for {user.email}: {e}", exc_info=True)
628
+ return Response(
629
+ content=f"Unable to create automation. Ensure the automation doesn't already exist.",
630
+ media_type="text/plain",
631
+ status_code=500,
632
+ )
633
+
634
+ # Collate info about the created user automation
635
+ automation_info = AutomationAdapters.get_automation_metadata(user, automation)
636
+
637
+ # Return information about the created automation as a JSON response
638
+ return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200)
639
+
640
+
641
+ @api.post("/trigger/automation", response_class=Response)
642
+ @requires(["authenticated"])
643
+ def trigger_manual_job(
644
+ request: Request,
645
+ automation_id: str,
646
+ ):
647
+ user: KhojUser = request.user.object
648
+
649
+ # Check, get automation to edit
650
+ try:
651
+ automation: Job = AutomationAdapters.get_automation(user, automation_id)
652
+ except ValueError as e:
653
+ logger.error(f"Error triggering automation {automation_id} for {user.email}: {e}", exc_info=True)
654
+ return Response(content="Invalid automation", status_code=403)
655
+
656
+ # Trigger the job without waiting for the result.
657
+ scheduled_chat_func = automation.func
658
+
659
+ # Run the function in a separate thread
660
+ thread = threading.Thread(target=scheduled_chat_func, args=automation.args, kwargs=automation.kwargs)
661
+ thread.start()
662
+
663
+ return Response(content="Automation triggered", status_code=200)
664
+
665
+
666
+ @api.put("/automation", response_class=Response)
667
+ @requires(["authenticated"])
668
+ def edit_job(
669
+ request: Request,
670
+ automation_id: str,
671
+ q: Optional[str],
672
+ subject: Optional[str],
673
+ crontime: Optional[str],
674
+ city: Optional[str] = None,
675
+ region: Optional[str] = None,
676
+ country: Optional[str] = None,
677
+ timezone: Optional[str] = None,
678
+ ) -> Response:
679
+ user: KhojUser = request.user.object
680
+
681
+ # Perform validation checks
682
+ if is_none_or_empty(q) or is_none_or_empty(subject) or is_none_or_empty(crontime):
683
+ return Response(content="A query, subject and crontime is required", status_code=400)
684
+ if not cron_descriptor.get_description(crontime):
685
+ return Response(content="Invalid crontime", status_code=400)
686
+
687
+ # Check, get automation to edit
688
+ try:
689
+ automation: Job = AutomationAdapters.get_automation(user, automation_id)
690
+ except ValueError as e:
691
+ logger.error(f"Error editing automation {automation_id} for {user.email}: {e}", exc_info=True)
692
+ return Response(content="Invalid automation", status_code=403)
693
+
694
+ # Infer subject, query to run
695
+ _, query_to_run, _ = schedule_query(q, conversation_history={}, user=user)
696
+ subject = subject
697
+
698
+ # Normalize query parameters
699
+ # Add /automated_task prefix to query if not present
700
+ query_to_run = query_to_run.strip()
701
+ if not query_to_run.startswith("/automated_task"):
702
+ query_to_run = f"/automated_task {query_to_run}"
703
+ # Normalize crontime for AP Scheduler CronTrigger
704
+ crontime = crontime.strip()
705
+ if len(crontime.split(" ")) > 5:
706
+ # Truncate crontime to 5 fields
707
+ crontime = " ".join(crontime.split(" ")[:5])
708
+ # Convert crontime to standard unix crontime
709
+ crontime = crontime.replace("?", "*")
710
+
711
+ # Disallow minute level automation recurrence
712
+ minute_value = crontime.split(" ")[0]
713
+ if not minute_value.isdigit():
714
+ return Response(
715
+ content="Recurrence of every X minutes is unsupported. Please create a less frequent schedule.",
716
+ status_code=400,
717
+ )
718
+
719
+ # Construct updated automation metadata
720
+ automation_metadata: dict[str, str] = json.loads(clean_json(automation.name))
721
+ automation_metadata["scheduling_request"] = q
722
+ automation_metadata["query_to_run"] = query_to_run
723
+ automation_metadata["subject"] = subject.strip()
724
+ automation_metadata["crontime"] = crontime
725
+ conversation_id = automation_metadata.get("conversation_id")
726
+
727
+ if not conversation_id:
728
+ title = f"Automation: {subject}"
729
+
730
+ # Create new Conversation Session associated with this new task
731
+ conversation = ConversationAdapters.create_conversation_session(user, request.user.client_app, title=title)
732
+
733
+ conversation_id = str(conversation.id)
734
+ automation_metadata["conversation_id"] = conversation_id
735
+
736
+ # Modify automation with updated query, subject
737
+ automation.modify(
738
+ name=json.dumps(automation_metadata),
739
+ kwargs={
740
+ "query_to_run": query_to_run,
741
+ "subject": subject,
742
+ "scheduling_request": q,
743
+ "user": user,
744
+ "calling_url": request.url,
745
+ "conversation_id": conversation_id,
746
+ },
747
+ )
748
+
749
+ # Reschedule automation if crontime updated
750
+ user_timezone = pytz.timezone(timezone)
751
+ trigger = CronTrigger.from_crontab(crontime, user_timezone)
752
+ if automation.trigger != trigger:
753
+ automation.reschedule(trigger=trigger)
754
+
755
+ # Collate info about the updated user automation
756
+ automation = AutomationAdapters.get_automation(user, automation.id)
757
+ automation_info = AutomationAdapters.get_automation_metadata(user, automation)
758
+
759
+ # Return modified automation information as a JSON response
760
+ return Response(content=json.dumps(automation_info), media_type="application/json", status_code=200)