khoj 1.33.3.dev32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (393) hide show
  1. khoj/__init__.py +0 -0
  2. khoj/app/README.md +94 -0
  3. khoj/app/__init__.py +0 -0
  4. khoj/app/asgi.py +16 -0
  5. khoj/app/settings.py +218 -0
  6. khoj/app/urls.py +25 -0
  7. khoj/configure.py +452 -0
  8. khoj/database/__init__.py +0 -0
  9. khoj/database/adapters/__init__.py +1821 -0
  10. khoj/database/admin.py +417 -0
  11. khoj/database/apps.py +6 -0
  12. khoj/database/management/__init__.py +0 -0
  13. khoj/database/management/commands/__init__.py +0 -0
  14. khoj/database/management/commands/change_default_model.py +116 -0
  15. khoj/database/management/commands/change_generated_images_url.py +61 -0
  16. khoj/database/management/commands/convert_images_png_to_webp.py +99 -0
  17. khoj/database/migrations/0001_khojuser.py +98 -0
  18. khoj/database/migrations/0002_googleuser.py +32 -0
  19. khoj/database/migrations/0003_vector_extension.py +10 -0
  20. khoj/database/migrations/0004_content_types_and_more.py +181 -0
  21. khoj/database/migrations/0005_embeddings_corpus_id.py +19 -0
  22. khoj/database/migrations/0006_embeddingsdates.py +33 -0
  23. khoj/database/migrations/0007_add_conversation.py +27 -0
  24. khoj/database/migrations/0008_alter_conversation_conversation_log.py +17 -0
  25. khoj/database/migrations/0009_khojapiuser.py +24 -0
  26. khoj/database/migrations/0010_chatmodeloptions_and_more.py +83 -0
  27. khoj/database/migrations/0010_rename_embeddings_entry_and_more.py +30 -0
  28. khoj/database/migrations/0011_merge_20231102_0138.py +14 -0
  29. khoj/database/migrations/0012_entry_file_source.py +21 -0
  30. khoj/database/migrations/0013_subscription.py +37 -0
  31. khoj/database/migrations/0014_alter_googleuser_picture.py +17 -0
  32. khoj/database/migrations/0015_alter_subscription_user.py +21 -0
  33. khoj/database/migrations/0016_alter_subscription_renewal_date.py +17 -0
  34. khoj/database/migrations/0017_searchmodel.py +32 -0
  35. khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py +30 -0
  36. khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py +27 -0
  37. khoj/database/migrations/0020_reflectivequestion.py +36 -0
  38. khoj/database/migrations/0021_speechtotextmodeloptions_and_more.py +42 -0
  39. khoj/database/migrations/0022_texttoimagemodelconfig.py +25 -0
  40. khoj/database/migrations/0023_usersearchmodelconfig.py +33 -0
  41. khoj/database/migrations/0024_alter_entry_embeddings.py +18 -0
  42. khoj/database/migrations/0025_clientapplication_khojuser_phone_number_and_more.py +46 -0
  43. khoj/database/migrations/0025_searchmodelconfig_embeddings_inference_endpoint_and_more.py +22 -0
  44. khoj/database/migrations/0026_searchmodelconfig_cross_encoder_inference_endpoint_and_more.py +22 -0
  45. khoj/database/migrations/0027_merge_20240118_1324.py +13 -0
  46. khoj/database/migrations/0028_khojuser_verified_phone_number.py +17 -0
  47. khoj/database/migrations/0029_userrequests.py +27 -0
  48. khoj/database/migrations/0030_conversation_slug_and_title.py +38 -0
  49. khoj/database/migrations/0031_agent_conversation_agent.py +53 -0
  50. khoj/database/migrations/0031_alter_googleuser_locale.py +30 -0
  51. khoj/database/migrations/0032_merge_20240322_0427.py +14 -0
  52. khoj/database/migrations/0033_rename_tuning_agent_personality.py +17 -0
  53. khoj/database/migrations/0034_alter_chatmodeloptions_chat_model.py +32 -0
  54. khoj/database/migrations/0035_processlock.py +26 -0
  55. khoj/database/migrations/0036_alter_processlock_name.py +19 -0
  56. khoj/database/migrations/0036_delete_offlinechatprocessorconversationconfig.py +15 -0
  57. khoj/database/migrations/0036_publicconversation.py +42 -0
  58. khoj/database/migrations/0037_chatmodeloptions_openai_config_and_more.py +51 -0
  59. khoj/database/migrations/0037_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +32 -0
  60. khoj/database/migrations/0038_merge_20240425_0857.py +14 -0
  61. khoj/database/migrations/0038_merge_20240426_1640.py +12 -0
  62. khoj/database/migrations/0039_merge_20240501_0301.py +12 -0
  63. khoj/database/migrations/0040_alter_processlock_name.py +26 -0
  64. khoj/database/migrations/0040_merge_20240504_1010.py +14 -0
  65. khoj/database/migrations/0041_merge_20240505_1234.py +14 -0
  66. khoj/database/migrations/0042_serverchatsettings.py +46 -0
  67. khoj/database/migrations/0043_alter_chatmodeloptions_model_type.py +21 -0
  68. khoj/database/migrations/0044_conversation_file_filters.py +17 -0
  69. khoj/database/migrations/0045_fileobject.py +37 -0
  70. khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py +22 -0
  71. khoj/database/migrations/0047_alter_entry_file_type.py +31 -0
  72. khoj/database/migrations/0048_voicemodeloption_uservoicemodelconfig.py +52 -0
  73. khoj/database/migrations/0049_datastore.py +38 -0
  74. khoj/database/migrations/0049_texttoimagemodelconfig_api_key_and_more.py +58 -0
  75. khoj/database/migrations/0050_alter_processlock_name.py +25 -0
  76. khoj/database/migrations/0051_merge_20240702_1220.py +14 -0
  77. khoj/database/migrations/0052_alter_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +27 -0
  78. khoj/database/migrations/0053_agent_style_color_agent_style_icon.py +61 -0
  79. khoj/database/migrations/0054_alter_agent_style_color.py +38 -0
  80. khoj/database/migrations/0055_alter_agent_style_icon.py +37 -0
  81. khoj/database/migrations/0056_chatmodeloptions_vision_enabled.py +17 -0
  82. khoj/database/migrations/0056_searchmodelconfig_cross_encoder_model_config.py +17 -0
  83. khoj/database/migrations/0057_merge_20240816_1409.py +13 -0
  84. khoj/database/migrations/0057_remove_serverchatsettings_default_model_and_more.py +51 -0
  85. khoj/database/migrations/0058_alter_chatmodeloptions_chat_model.py +17 -0
  86. khoj/database/migrations/0059_searchmodelconfig_bi_encoder_confidence_threshold.py +17 -0
  87. khoj/database/migrations/0060_merge_20240905_1828.py +14 -0
  88. khoj/database/migrations/0061_alter_chatmodeloptions_model_type.py +26 -0
  89. khoj/database/migrations/0061_alter_texttoimagemodelconfig_model_type.py +21 -0
  90. khoj/database/migrations/0062_merge_20240913_0222.py +14 -0
  91. khoj/database/migrations/0063_conversation_temp_id.py +36 -0
  92. khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +86 -0
  93. khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
  94. khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
  95. khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
  96. khoj/database/migrations/0068_alter_agent_output_modes.py +24 -0
  97. khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py +89 -0
  98. khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +46 -0
  99. khoj/database/migrations/0071_subscription_enabled_trial_at_and_more.py +32 -0
  100. khoj/database/migrations/0072_entry_search_model.py +24 -0
  101. khoj/database/migrations/0073_delete_usersearchmodelconfig.py +15 -0
  102. khoj/database/migrations/0074_alter_conversation_title.py +17 -0
  103. khoj/database/migrations/0075_migrate_generated_assets_and_validate.py +85 -0
  104. khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py +26 -0
  105. khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py +62 -0
  106. khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py +17 -0
  107. khoj/database/migrations/__init__.py +0 -0
  108. khoj/database/models/__init__.py +725 -0
  109. khoj/database/tests.py +3 -0
  110. khoj/interface/compiled/404/index.html +1 -0
  111. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_buildManifest.js +1 -0
  112. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_ssgManifest.js +1 -0
  113. khoj/interface/compiled/_next/static/chunks/1010-8f39bb4648b5ba10.js +1 -0
  114. khoj/interface/compiled/_next/static/chunks/182-f1c48a203dc91e0e.js +20 -0
  115. khoj/interface/compiled/_next/static/chunks/1915-d3c36ad6ce697ce7.js +1 -0
  116. khoj/interface/compiled/_next/static/chunks/2117-165ef4747a5b836b.js +2 -0
  117. khoj/interface/compiled/_next/static/chunks/2581-455000f8aeb08fc3.js +1 -0
  118. khoj/interface/compiled/_next/static/chunks/3727.dcea8f2193111552.js +1 -0
  119. khoj/interface/compiled/_next/static/chunks/3789-a09e37a819171a9d.js +1 -0
  120. khoj/interface/compiled/_next/static/chunks/4124-6c28322ce218d2d5.js +1 -0
  121. khoj/interface/compiled/_next/static/chunks/5427-b52d95253e692bfa.js +1 -0
  122. khoj/interface/compiled/_next/static/chunks/5473-b1cf56dedac6577a.js +1 -0
  123. khoj/interface/compiled/_next/static/chunks/5477-0bbddb79c25a54a7.js +1 -0
  124. khoj/interface/compiled/_next/static/chunks/6065-64db9ad305ba0bcd.js +1 -0
  125. khoj/interface/compiled/_next/static/chunks/6293-469dd16402ea8a6f.js +3 -0
  126. khoj/interface/compiled/_next/static/chunks/688-b5b4391bbc0376f1.js +1 -0
  127. khoj/interface/compiled/_next/static/chunks/8667-b6bf63c72b2d76eb.js +1 -0
  128. khoj/interface/compiled/_next/static/chunks/9259-1172dbaca0515237.js +1 -0
  129. khoj/interface/compiled/_next/static/chunks/94ca1967.1d9b42d929a1ee8c.js +1 -0
  130. khoj/interface/compiled/_next/static/chunks/9597.83583248dfbf6e73.js +1 -0
  131. khoj/interface/compiled/_next/static/chunks/964ecbae.51d6faf8801d15e6.js +1 -0
  132. khoj/interface/compiled/_next/static/chunks/9665-391df1e5c51c960a.js +1 -0
  133. khoj/interface/compiled/_next/static/chunks/app/_not-found/page-a834eddae3e235df.js +1 -0
  134. khoj/interface/compiled/_next/static/chunks/app/agents/layout-e00fb81dca656a10.js +1 -0
  135. khoj/interface/compiled/_next/static/chunks/app/agents/page-28ce086a1129bca2.js +1 -0
  136. khoj/interface/compiled/_next/static/chunks/app/automations/layout-1fe1537449f43496.js +1 -0
  137. khoj/interface/compiled/_next/static/chunks/app/automations/page-bf365a60829d347f.js +1 -0
  138. khoj/interface/compiled/_next/static/chunks/app/chat/layout-33934fc2d6ae6838.js +1 -0
  139. khoj/interface/compiled/_next/static/chunks/app/chat/page-0e476e57eb2015e3.js +1 -0
  140. khoj/interface/compiled/_next/static/chunks/app/layout-30e7fda7262713ce.js +1 -0
  141. khoj/interface/compiled/_next/static/chunks/app/page-a5515ea71aec5ef0.js +1 -0
  142. khoj/interface/compiled/_next/static/chunks/app/search/layout-c02531d586972d7d.js +1 -0
  143. khoj/interface/compiled/_next/static/chunks/app/search/page-9140541e67ea307d.js +1 -0
  144. khoj/interface/compiled/_next/static/chunks/app/settings/layout-d09d6510a45cd4bd.js +1 -0
  145. khoj/interface/compiled/_next/static/chunks/app/settings/page-951ba40b5b94b23a.js +1 -0
  146. khoj/interface/compiled/_next/static/chunks/app/share/chat/layout-e8e5db7830bf3f47.js +1 -0
  147. khoj/interface/compiled/_next/static/chunks/app/share/chat/page-1beb80d8d741c932.js +1 -0
  148. khoj/interface/compiled/_next/static/chunks/d3ac728e-44ebd2a0c99b12a0.js +1 -0
  149. khoj/interface/compiled/_next/static/chunks/fd9d1056-4482b99a36fd1673.js +1 -0
  150. khoj/interface/compiled/_next/static/chunks/framework-8e0e0f4a6b83a956.js +1 -0
  151. khoj/interface/compiled/_next/static/chunks/main-app-de1f09df97a3cfc7.js +1 -0
  152. khoj/interface/compiled/_next/static/chunks/main-db4bfac6b0a8d00b.js +1 -0
  153. khoj/interface/compiled/_next/static/chunks/pages/_app-3c9ca398d360b709.js +1 -0
  154. khoj/interface/compiled/_next/static/chunks/pages/_error-cf5ca766ac8f493f.js +1 -0
  155. khoj/interface/compiled/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
  156. khoj/interface/compiled/_next/static/chunks/webpack-a03962458328b163.js +1 -0
  157. khoj/interface/compiled/_next/static/css/089de1d8526b96e9.css +1 -0
  158. khoj/interface/compiled/_next/static/css/37a73b87f02df402.css +1 -0
  159. khoj/interface/compiled/_next/static/css/4e4e6a4a1c920d06.css +1 -0
  160. khoj/interface/compiled/_next/static/css/8d02837c730f8d13.css +25 -0
  161. khoj/interface/compiled/_next/static/css/8e6a3ca11a60b189.css +1 -0
  162. khoj/interface/compiled/_next/static/css/9c164d9727dd8092.css +1 -0
  163. khoj/interface/compiled/_next/static/css/dac88c17aaee5fcf.css +1 -0
  164. khoj/interface/compiled/_next/static/css/df4b47a2d0d85eae.css +1 -0
  165. khoj/interface/compiled/_next/static/css/e4eb883b5265d372.css +1 -0
  166. khoj/interface/compiled/_next/static/media/1d8a05b60287ae6c-s.p.woff2 +0 -0
  167. khoj/interface/compiled/_next/static/media/6f22fce21a7c433c-s.woff2 +0 -0
  168. khoj/interface/compiled/_next/static/media/77c207b095007c34-s.p.woff2 +0 -0
  169. khoj/interface/compiled/_next/static/media/82ef96de0e8f4d8c-s.p.woff2 +0 -0
  170. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.1608a09b.woff +0 -0
  171. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.4aafdb68.ttf +0 -0
  172. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.a79f1c31.woff2 +0 -0
  173. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.b6770918.woff +0 -0
  174. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.cce5b8ec.ttf +0 -0
  175. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.ec17d132.woff2 +0 -0
  176. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.07ef19e7.ttf +0 -0
  177. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.55fac258.woff2 +0 -0
  178. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.dad44a7f.woff +0 -0
  179. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.9f256b85.woff +0 -0
  180. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.b18f59e1.ttf +0 -0
  181. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.d42a5579.woff2 +0 -0
  182. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.7c187121.woff +0 -0
  183. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.d3c882a6.woff2 +0 -0
  184. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.ed38e79f.ttf +0 -0
  185. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.b74a1a8b.ttf +0 -0
  186. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.c3fb5ac2.woff2 +0 -0
  187. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.d181c465.woff +0 -0
  188. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.6f2bb1df.woff2 +0 -0
  189. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.70d8b0a5.ttf +0 -0
  190. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.e3f82f9d.woff +0 -0
  191. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.47373d1e.ttf +0 -0
  192. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.8916142b.woff2 +0 -0
  193. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.9024d815.woff +0 -0
  194. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.0462f03b.woff2 +0 -0
  195. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.7f51fe03.woff +0 -0
  196. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.b7f8fe9b.ttf +0 -0
  197. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.572d331f.woff2 +0 -0
  198. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.a879cf83.ttf +0 -0
  199. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.f1035d8d.woff +0 -0
  200. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.5295ba48.woff +0 -0
  201. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.939bc644.ttf +0 -0
  202. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.f28c23ac.woff2 +0 -0
  203. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.8c5b5494.woff2 +0 -0
  204. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.94e1e8dc.ttf +0 -0
  205. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.bf59d231.woff +0 -0
  206. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.3b1e59b3.woff2 +0 -0
  207. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.7c9bc82b.woff +0 -0
  208. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.b4c20c84.ttf +0 -0
  209. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.74048478.woff +0 -0
  210. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.ba21ed5f.woff2 +0 -0
  211. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.d4d7ba48.ttf +0 -0
  212. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.03e9641d.woff2 +0 -0
  213. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.07505710.woff +0 -0
  214. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.fe9cbbe1.ttf +0 -0
  215. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.e1e279cb.woff +0 -0
  216. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.eae34984.woff2 +0 -0
  217. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.fabc004a.ttf +0 -0
  218. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.57727022.woff +0 -0
  219. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.5916a24f.woff2 +0 -0
  220. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.d6b476ec.ttf +0 -0
  221. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.9acaf01c.woff +0 -0
  222. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.a144ef58.ttf +0 -0
  223. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.b4230e7e.woff2 +0 -0
  224. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.10d95fd3.woff2 +0 -0
  225. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.7a996c9d.woff +0 -0
  226. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.fbccdabe.ttf +0 -0
  227. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.6258592b.woff +0 -0
  228. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.a8709e36.woff2 +0 -0
  229. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.d97aaf4a.ttf +0 -0
  230. khoj/interface/compiled/_next/static/media/a6ecd16fa044d500-s.p.woff2 +0 -0
  231. khoj/interface/compiled/_next/static/media/bd82c78e5b7b3fe9-s.p.woff2 +0 -0
  232. khoj/interface/compiled/_next/static/media/c32c8052c071fc42-s.woff2 +0 -0
  233. khoj/interface/compiled/_next/static/media/c4250770ab8708b6-s.p.woff2 +0 -0
  234. khoj/interface/compiled/_next/static/media/e098aaaecc9cfbb2-s.p.woff2 +0 -0
  235. khoj/interface/compiled/_next/static/media/flags.3afdda2f.webp +0 -0
  236. khoj/interface/compiled/_next/static/media/flags@2x.5fbe9fc1.webp +0 -0
  237. khoj/interface/compiled/_next/static/media/globe.98e105ca.webp +0 -0
  238. khoj/interface/compiled/_next/static/media/globe@2x.974df6f8.webp +0 -0
  239. khoj/interface/compiled/agents/index.html +1 -0
  240. khoj/interface/compiled/agents/index.txt +7 -0
  241. khoj/interface/compiled/agents.svg +6 -0
  242. khoj/interface/compiled/assets/icons/khoj_lantern.ico +0 -0
  243. khoj/interface/compiled/assets/icons/khoj_lantern.svg +100 -0
  244. khoj/interface/compiled/assets/icons/khoj_lantern_1200x1200.png +0 -0
  245. khoj/interface/compiled/assets/icons/khoj_lantern_128x128.png +0 -0
  246. khoj/interface/compiled/assets/icons/khoj_lantern_128x128_dark.png +0 -0
  247. khoj/interface/compiled/assets/icons/khoj_lantern_256x256.png +0 -0
  248. khoj/interface/compiled/assets/icons/khoj_lantern_512x512.png +0 -0
  249. khoj/interface/compiled/assets/icons/khoj_lantern_logomarktype_1200x630.png +0 -0
  250. khoj/interface/compiled/assets/samples/desktop-browse-draw-sample.png +0 -0
  251. khoj/interface/compiled/assets/samples/desktop-plain-chat-sample.png +0 -0
  252. khoj/interface/compiled/assets/samples/desktop-remember-plan-sample.png +0 -0
  253. khoj/interface/compiled/assets/samples/phone-browse-draw-sample.png +0 -0
  254. khoj/interface/compiled/assets/samples/phone-plain-chat-sample.png +0 -0
  255. khoj/interface/compiled/assets/samples/phone-remember-plan-sample.png +0 -0
  256. khoj/interface/compiled/automation.svg +37 -0
  257. khoj/interface/compiled/automations/index.html +1 -0
  258. khoj/interface/compiled/automations/index.txt +8 -0
  259. khoj/interface/compiled/chat/index.html +1 -0
  260. khoj/interface/compiled/chat/index.txt +7 -0
  261. khoj/interface/compiled/chat.svg +24 -0
  262. khoj/interface/compiled/close.svg +5 -0
  263. khoj/interface/compiled/copy-button-success.svg +6 -0
  264. khoj/interface/compiled/copy-button.svg +5 -0
  265. khoj/interface/compiled/index.html +1 -0
  266. khoj/interface/compiled/index.txt +7 -0
  267. khoj/interface/compiled/khoj.webmanifest +76 -0
  268. khoj/interface/compiled/logo.svg +24 -0
  269. khoj/interface/compiled/search/index.html +1 -0
  270. khoj/interface/compiled/search/index.txt +7 -0
  271. khoj/interface/compiled/send.svg +1 -0
  272. khoj/interface/compiled/settings/index.html +1 -0
  273. khoj/interface/compiled/settings/index.txt +9 -0
  274. khoj/interface/compiled/share/chat/index.html +1 -0
  275. khoj/interface/compiled/share/chat/index.txt +7 -0
  276. khoj/interface/compiled/share.svg +8 -0
  277. khoj/interface/compiled/thumbs-down.svg +6 -0
  278. khoj/interface/compiled/thumbs-up.svg +6 -0
  279. khoj/interface/email/feedback.html +34 -0
  280. khoj/interface/email/magic_link.html +40 -0
  281. khoj/interface/email/task.html +37 -0
  282. khoj/interface/email/welcome.html +90 -0
  283. khoj/interface/web/.well-known/assetlinks.json +11 -0
  284. khoj/interface/web/assets/icons/agents.svg +19 -0
  285. khoj/interface/web/assets/icons/automation.svg +43 -0
  286. khoj/interface/web/assets/icons/chat.svg +24 -0
  287. khoj/interface/web/assets/icons/github.svg +1 -0
  288. khoj/interface/web/assets/icons/khoj-logo-sideways-200.png +0 -0
  289. khoj/interface/web/assets/icons/khoj-logo-sideways-500.png +0 -0
  290. khoj/interface/web/assets/icons/khoj-logo-sideways.svg +32 -0
  291. khoj/interface/web/assets/icons/khoj.svg +26 -0
  292. khoj/interface/web/assets/icons/logotype.svg +1 -0
  293. khoj/interface/web/assets/icons/search.svg +57 -0
  294. khoj/interface/web/assets/icons/sync.svg +4 -0
  295. khoj/interface/web/assets/khoj.css +237 -0
  296. khoj/interface/web/assets/utils.js +33 -0
  297. khoj/interface/web/base_config.html +445 -0
  298. khoj/interface/web/content_source_github_input.html +208 -0
  299. khoj/interface/web/login.html +310 -0
  300. khoj/interface/web/utils.html +48 -0
  301. khoj/main.py +249 -0
  302. khoj/manage.py +22 -0
  303. khoj/migrations/__init__.py +0 -0
  304. khoj/migrations/migrate_offline_chat_default_model.py +69 -0
  305. khoj/migrations/migrate_offline_chat_default_model_2.py +71 -0
  306. khoj/migrations/migrate_offline_chat_schema.py +83 -0
  307. khoj/migrations/migrate_offline_model.py +29 -0
  308. khoj/migrations/migrate_processor_config_openai.py +67 -0
  309. khoj/migrations/migrate_server_pg.py +132 -0
  310. khoj/migrations/migrate_version.py +17 -0
  311. khoj/processor/__init__.py +0 -0
  312. khoj/processor/content/__init__.py +0 -0
  313. khoj/processor/content/docx/__init__.py +0 -0
  314. khoj/processor/content/docx/docx_to_entries.py +111 -0
  315. khoj/processor/content/github/__init__.py +0 -0
  316. khoj/processor/content/github/github_to_entries.py +226 -0
  317. khoj/processor/content/images/__init__.py +0 -0
  318. khoj/processor/content/images/image_to_entries.py +117 -0
  319. khoj/processor/content/markdown/__init__.py +0 -0
  320. khoj/processor/content/markdown/markdown_to_entries.py +160 -0
  321. khoj/processor/content/notion/notion_to_entries.py +259 -0
  322. khoj/processor/content/org_mode/__init__.py +0 -0
  323. khoj/processor/content/org_mode/org_to_entries.py +226 -0
  324. khoj/processor/content/org_mode/orgnode.py +532 -0
  325. khoj/processor/content/pdf/__init__.py +0 -0
  326. khoj/processor/content/pdf/pdf_to_entries.py +119 -0
  327. khoj/processor/content/plaintext/__init__.py +0 -0
  328. khoj/processor/content/plaintext/plaintext_to_entries.py +117 -0
  329. khoj/processor/content/text_to_entries.py +296 -0
  330. khoj/processor/conversation/__init__.py +0 -0
  331. khoj/processor/conversation/anthropic/__init__.py +0 -0
  332. khoj/processor/conversation/anthropic/anthropic_chat.py +243 -0
  333. khoj/processor/conversation/anthropic/utils.py +217 -0
  334. khoj/processor/conversation/google/__init__.py +0 -0
  335. khoj/processor/conversation/google/gemini_chat.py +253 -0
  336. khoj/processor/conversation/google/utils.py +260 -0
  337. khoj/processor/conversation/offline/__init__.py +0 -0
  338. khoj/processor/conversation/offline/chat_model.py +308 -0
  339. khoj/processor/conversation/offline/utils.py +80 -0
  340. khoj/processor/conversation/offline/whisper.py +15 -0
  341. khoj/processor/conversation/openai/__init__.py +0 -0
  342. khoj/processor/conversation/openai/gpt.py +243 -0
  343. khoj/processor/conversation/openai/utils.py +232 -0
  344. khoj/processor/conversation/openai/whisper.py +13 -0
  345. khoj/processor/conversation/prompts.py +1188 -0
  346. khoj/processor/conversation/utils.py +867 -0
  347. khoj/processor/embeddings.py +122 -0
  348. khoj/processor/image/generate.py +215 -0
  349. khoj/processor/speech/__init__.py +0 -0
  350. khoj/processor/speech/text_to_speech.py +51 -0
  351. khoj/processor/tools/__init__.py +0 -0
  352. khoj/processor/tools/online_search.py +472 -0
  353. khoj/processor/tools/run_code.py +179 -0
  354. khoj/routers/__init__.py +0 -0
  355. khoj/routers/api.py +760 -0
  356. khoj/routers/api_agents.py +295 -0
  357. khoj/routers/api_chat.py +1273 -0
  358. khoj/routers/api_content.py +634 -0
  359. khoj/routers/api_model.py +123 -0
  360. khoj/routers/api_phone.py +86 -0
  361. khoj/routers/api_subscription.py +144 -0
  362. khoj/routers/auth.py +307 -0
  363. khoj/routers/email.py +135 -0
  364. khoj/routers/helpers.py +2333 -0
  365. khoj/routers/notion.py +85 -0
  366. khoj/routers/research.py +364 -0
  367. khoj/routers/storage.py +63 -0
  368. khoj/routers/twilio.py +36 -0
  369. khoj/routers/web_client.py +141 -0
  370. khoj/search_filter/__init__.py +0 -0
  371. khoj/search_filter/base_filter.py +15 -0
  372. khoj/search_filter/date_filter.py +215 -0
  373. khoj/search_filter/file_filter.py +32 -0
  374. khoj/search_filter/word_filter.py +29 -0
  375. khoj/search_type/__init__.py +0 -0
  376. khoj/search_type/text_search.py +255 -0
  377. khoj/utils/__init__.py +0 -0
  378. khoj/utils/cli.py +101 -0
  379. khoj/utils/config.py +81 -0
  380. khoj/utils/constants.py +51 -0
  381. khoj/utils/fs_syncer.py +252 -0
  382. khoj/utils/helpers.py +627 -0
  383. khoj/utils/initialization.py +301 -0
  384. khoj/utils/jsonl.py +43 -0
  385. khoj/utils/models.py +47 -0
  386. khoj/utils/rawconfig.py +208 -0
  387. khoj/utils/state.py +48 -0
  388. khoj/utils/yaml.py +47 -0
  389. khoj-1.33.3.dev32.dist-info/METADATA +190 -0
  390. khoj-1.33.3.dev32.dist-info/RECORD +393 -0
  391. khoj-1.33.3.dev32.dist-info/WHEEL +4 -0
  392. khoj-1.33.3.dev32.dist-info/entry_points.txt +2 -0
  393. khoj-1.33.3.dev32.dist-info/licenses/LICENSE +661 -0
khoj/configure.py ADDED
@@ -0,0 +1,452 @@
1
+ import json
2
+ import logging
3
+ import os
4
+ from datetime import datetime
5
+ from enum import Enum
6
+ from functools import wraps
7
+ from typing import Optional
8
+
9
+ import openai
10
+ import requests
11
+ import schedule
12
+ from asgiref.sync import sync_to_async
13
+ from django.conf import settings
14
+ from django.db import close_old_connections, connections
15
+ from django.utils.timezone import make_aware
16
+ from fastapi import Response
17
+ from starlette.authentication import (
18
+ AuthCredentials,
19
+ AuthenticationBackend,
20
+ SimpleUser,
21
+ UnauthenticatedUser,
22
+ )
23
+ from starlette.concurrency import run_in_threadpool
24
+ from starlette.middleware import Middleware
25
+ from starlette.middleware.authentication import AuthenticationMiddleware
26
+ from starlette.middleware.base import BaseHTTPMiddleware
27
+ from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware
28
+ from starlette.middleware.sessions import SessionMiddleware
29
+ from starlette.requests import HTTPConnection
30
+ from starlette.types import ASGIApp, Receive, Scope, Send
31
+
32
+ from khoj.database.adapters import (
33
+ AgentAdapters,
34
+ ClientApplicationAdapters,
35
+ ConversationAdapters,
36
+ ProcessLockAdapters,
37
+ aget_or_create_user_by_phone_number,
38
+ aget_user_by_phone_number,
39
+ ais_user_subscribed,
40
+ delete_user_requests,
41
+ get_all_users,
42
+ get_or_create_search_models,
43
+ )
44
+ from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
45
+ from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
46
+ from khoj.routers.api_content import configure_content, configure_search
47
+ from khoj.routers.twilio import is_twilio_enabled
48
+ from khoj.utils import constants, state
49
+ from khoj.utils.config import SearchType
50
+ from khoj.utils.fs_syncer import collect_files
51
+ from khoj.utils.helpers import is_none_or_empty, telemetry_disabled
52
+ from khoj.utils.rawconfig import FullConfig
53
+
54
+ logger = logging.getLogger(__name__)
55
+
56
+
57
+ class AuthenticatedKhojUser(SimpleUser):
58
+ def __init__(self, user, client_app: Optional[ClientApplication] = None):
59
+ self.object = user
60
+ self.client_app = client_app
61
+ super().__init__(user.username)
62
+
63
+
64
+ class AsyncCloseConnectionsMiddleware(BaseHTTPMiddleware):
65
+ """
66
+ Using this middleware to call close_old_connections() twice is a pretty yucky hack,
67
+ as it appears that run_in_threadpool (used by Starlette/FastAPI) and sync_to_async
68
+ (used by Django) have divergent behavior, ultimately acquiring the incorrect thread
69
+ in mixed sync/async which has the effect of duplicating connections.
70
+ We could fix the duplicate connections too if we normalized the thread behavior,
71
+ but at minimum we need to clean up connections in each case to prevent persistent
72
+ "InterfaceError: connection already closed" errors when the database connection is
73
+ reset via a database restart or something -- so here we are!
74
+ If we always use smart_sync_to_async(), this double calling isn't necessary, but
75
+ depending on what levels of abstraction we introduce, we might silently break the
76
+ assumptions. Better to be safe than sorry!
77
+ Attribution: https://gist.github.com/bryanhelmig/6fb091f23c1a4b7462dddce51cfaa1ca
78
+ """
79
+
80
+ async def dispatch(self, request, call_next):
81
+ await run_in_threadpool(close_old_connections)
82
+ await sync_to_async(close_old_connections)()
83
+ try:
84
+ response = await call_next(request)
85
+ finally:
86
+ # in tests, use @override_settings(CLOSE_CONNECTIONS_AFTER_REQUEST=True)
87
+ if getattr(settings, "CLOSE_CONNECTIONS_AFTER_REQUEST", False):
88
+ await run_in_threadpool(connections.close_all)
89
+ await sync_to_async(connections.close_all)()
90
+ return response
91
+
92
+
93
+ class UserAuthenticationBackend(AuthenticationBackend):
94
+ def __init__(
95
+ self,
96
+ ):
97
+ from khoj.database.models import KhojApiUser, KhojUser
98
+
99
+ self.khojuser_manager = KhojUser.objects
100
+ self.khojapiuser_manager = KhojApiUser.objects
101
+ self._initialize_default_user()
102
+ super().__init__()
103
+
104
+ def _initialize_default_user(self):
105
+ if not self.khojuser_manager.filter(username="default").exists():
106
+ default_user = self.khojuser_manager.create_user(
107
+ username="default",
108
+ email="default@example.com",
109
+ password="default",
110
+ )
111
+ renewal_date = make_aware(datetime.strptime("2100-04-01", "%Y-%m-%d"))
112
+ Subscription.objects.create(user=default_user, type=Subscription.Type.STANDARD, renewal_date=renewal_date)
113
+
114
+ async def authenticate(self, request: HTTPConnection):
115
+ current_user = request.session.get("user")
116
+ if current_user and current_user.get("email"):
117
+ user = (
118
+ await self.khojuser_manager.filter(email=current_user.get("email"))
119
+ .prefetch_related("subscription")
120
+ .afirst()
121
+ )
122
+ if user:
123
+ subscribed = await ais_user_subscribed(user)
124
+ if subscribed:
125
+ return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user)
126
+ return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user)
127
+
128
+ # Request from Desktop, Emacs, Obsidian clients
129
+ if len(request.headers.get("Authorization", "").split("Bearer ")) == 2:
130
+ # Get bearer token from header
131
+ bearer_token = request.headers["Authorization"].split("Bearer ")[1]
132
+ # Get user owning token
133
+ user_with_token = (
134
+ await self.khojapiuser_manager.filter(token=bearer_token)
135
+ .select_related("user")
136
+ .prefetch_related("user__subscription")
137
+ .afirst()
138
+ )
139
+ if user_with_token:
140
+ subscribed = await ais_user_subscribed(user_with_token.user)
141
+ if subscribed:
142
+ return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user_with_token.user)
143
+ return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user_with_token.user)
144
+
145
+ # Request from Whatsapp client
146
+ client_id = request.query_params.get("client_id")
147
+ if client_id:
148
+ # Get the client secret, which is passed in the Authorization header
149
+ client_secret = request.headers["Authorization"].split("Bearer ")[1]
150
+ if not client_secret:
151
+ return Response(
152
+ status_code=401,
153
+ content="Please provide a client secret in the Authorization header with a client_id query param.",
154
+ )
155
+
156
+ # Get the client application
157
+ client_application = await ClientApplicationAdapters.aget_client_application_by_id(client_id, client_secret)
158
+ if client_application is None:
159
+ return AuthCredentials(), UnauthenticatedUser()
160
+ # Get the identifier used for the user
161
+ phone_number = request.query_params.get("phone_number")
162
+ if is_none_or_empty(phone_number):
163
+ return AuthCredentials(), UnauthenticatedUser()
164
+
165
+ if not phone_number.startswith("+"):
166
+ phone_number = f"+{phone_number}"
167
+
168
+ create_if_not_exists = request.query_params.get("create_if_not_exists")
169
+ if create_if_not_exists:
170
+ user, is_new = await aget_or_create_user_by_phone_number(phone_number)
171
+ if user and is_new:
172
+ logger.log(logging.INFO, f"🥳 New User Created: {user.uuid}")
173
+ else:
174
+ user = await aget_user_by_phone_number(phone_number)
175
+
176
+ if user is None:
177
+ return AuthCredentials(), UnauthenticatedUser()
178
+
179
+ subscribed = await ais_user_subscribed(user)
180
+
181
+ if subscribed:
182
+ return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user, client_application)
183
+ return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user, client_application)
184
+
185
+ # No auth required if server in anonymous mode
186
+ if state.anonymous_mode:
187
+ user = await self.khojuser_manager.filter(username="default").prefetch_related("subscription").afirst()
188
+ if user:
189
+ return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user)
190
+
191
+ return AuthCredentials(), UnauthenticatedUser()
192
+
193
+
194
+ def initialize_server(config: Optional[FullConfig]):
195
+ try:
196
+ configure_server(config, init=True)
197
+ except Exception as e:
198
+ logger.error(f"🚨 Failed to configure server on app load: {e}", exc_info=True)
199
+ raise e
200
+
201
+
202
+ def clean_connections(func):
203
+ """
204
+ A decorator that ensures that Django database connections that have become unusable, or are obsolete, are closed
205
+ before and after a method is executed (see: https://docs.djangoproject.com/en/dev/ref/databases/#general-notes
206
+ for background).
207
+ """
208
+
209
+ @wraps(func)
210
+ def func_wrapper(*args, **kwargs):
211
+ close_old_connections()
212
+ try:
213
+ result = func(*args, **kwargs)
214
+ finally:
215
+ close_old_connections()
216
+
217
+ return result
218
+
219
+ return func_wrapper
220
+
221
+
222
+ def configure_server(
223
+ config: FullConfig,
224
+ regenerate: bool = False,
225
+ search_type: Optional[SearchType] = None,
226
+ init=False,
227
+ user: KhojUser = None,
228
+ ):
229
+ # Update Config
230
+ if config == None:
231
+ logger.info(f"Initializing with default config.")
232
+ config = FullConfig()
233
+ state.config = config
234
+
235
+ if ConversationAdapters.has_valid_ai_model_api():
236
+ ai_model_api = ConversationAdapters.get_ai_model_api()
237
+ state.openai_client = openai.OpenAI(api_key=ai_model_api.api_key)
238
+
239
+ # Initialize Search Models from Config and initialize content
240
+ try:
241
+ search_models = get_or_create_search_models()
242
+ state.embeddings_model = dict()
243
+ state.cross_encoder_model = dict()
244
+
245
+ for model in search_models:
246
+ state.embeddings_model.update(
247
+ {
248
+ model.name: EmbeddingsModel(
249
+ model.bi_encoder,
250
+ model.embeddings_inference_endpoint,
251
+ model.embeddings_inference_endpoint_api_key,
252
+ query_encode_kwargs=model.bi_encoder_query_encode_config,
253
+ docs_encode_kwargs=model.bi_encoder_docs_encode_config,
254
+ model_kwargs=model.bi_encoder_model_config,
255
+ )
256
+ }
257
+ )
258
+ state.cross_encoder_model.update(
259
+ {
260
+ model.name: CrossEncoderModel(
261
+ model.cross_encoder,
262
+ model.cross_encoder_inference_endpoint,
263
+ model.cross_encoder_inference_endpoint_api_key,
264
+ model_kwargs=model.cross_encoder_model_config,
265
+ )
266
+ }
267
+ )
268
+
269
+ state.SearchType = configure_search_types()
270
+ state.search_models = configure_search(state.search_models, state.config.search_type)
271
+ setup_default_agent(user)
272
+
273
+ message = (
274
+ "📡 Telemetry disabled"
275
+ if telemetry_disabled(state.config.app, state.telemetry_disabled)
276
+ else "📡 Telemetry enabled"
277
+ )
278
+ logger.info(message)
279
+
280
+ if not init:
281
+ initialize_content(user, regenerate, search_type)
282
+
283
+ except Exception as e:
284
+ logger.error(f"Failed to load some search models: {e}", exc_info=True)
285
+
286
+
287
+ def setup_default_agent(user: KhojUser):
288
+ AgentAdapters.create_default_agent(user)
289
+
290
+
291
+ def initialize_content(user: KhojUser, regenerate: bool, search_type: Optional[SearchType] = None):
292
+ # Initialize Content from Config
293
+ if state.search_models:
294
+ try:
295
+ logger.info("📬 Updating content index...")
296
+ all_files = collect_files(user=user)
297
+ status = configure_content(
298
+ user,
299
+ all_files,
300
+ regenerate,
301
+ search_type,
302
+ )
303
+ if not status:
304
+ raise RuntimeError("Failed to update content index")
305
+ except Exception as e:
306
+ raise e
307
+
308
+
309
+ def configure_routes(app):
310
+ # Import APIs here to setup search types before while configuring server
311
+ from khoj.routers.api import api
312
+ from khoj.routers.api_agents import api_agents
313
+ from khoj.routers.api_chat import api_chat
314
+ from khoj.routers.api_content import api_content
315
+ from khoj.routers.api_model import api_model
316
+ from khoj.routers.notion import notion_router
317
+ from khoj.routers.web_client import web_client
318
+
319
+ app.include_router(api, prefix="/api")
320
+ app.include_router(api_chat, prefix="/api/chat")
321
+ app.include_router(api_agents, prefix="/api/agents")
322
+ app.include_router(api_model, prefix="/api/model")
323
+ app.include_router(api_content, prefix="/api/content")
324
+ app.include_router(notion_router, prefix="/api/notion")
325
+ app.include_router(web_client)
326
+
327
+ if not state.anonymous_mode:
328
+ from khoj.routers.auth import auth_router
329
+
330
+ app.include_router(auth_router, prefix="/auth")
331
+ logger.info("🔑 Enabled Authentication")
332
+
333
+ if state.billing_enabled:
334
+ from khoj.routers.api_subscription import subscription_router
335
+
336
+ app.include_router(subscription_router, prefix="/api/subscription")
337
+ logger.info("💳 Enabled Billing")
338
+
339
+ if is_twilio_enabled():
340
+ from khoj.routers.api_phone import api_phone
341
+
342
+ app.include_router(api_phone, prefix="/api/phone")
343
+ logger.info("📞 Enabled Twilio")
344
+
345
+
346
+ def configure_middleware(app, ssl_enabled: bool = False):
347
+ class NextJsMiddleware(Middleware):
348
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
349
+ if scope["type"] == "http" and scope["path"].startswith("/_next"):
350
+ scope["path"] = "/static" + scope["path"]
351
+ await self.app(scope, receive, send)
352
+
353
+ def __init__(self, app: ASGIApp) -> None:
354
+ super().__init__(app)
355
+ self.app = app
356
+
357
+ if ssl_enabled:
358
+ app.add_middleware(HTTPSRedirectMiddleware)
359
+ app.add_middleware(AsyncCloseConnectionsMiddleware)
360
+ app.add_middleware(AuthenticationMiddleware, backend=UserAuthenticationBackend())
361
+ app.add_middleware(NextJsMiddleware)
362
+ app.add_middleware(SessionMiddleware, secret_key=os.environ.get("KHOJ_DJANGO_SECRET_KEY", "!secret"))
363
+
364
+
365
+ def update_content_index():
366
+ for user in get_all_users():
367
+ all_files = collect_files(user=user)
368
+ success = configure_content(user, all_files)
369
+ if not success:
370
+ raise RuntimeError("Failed to update content index")
371
+ logger.info("📪 Content index updated via Scheduler")
372
+
373
+
374
+ @schedule.repeat(schedule.every(22).to(25).hours)
375
+ @clean_connections
376
+ def update_content_index_regularly():
377
+ ProcessLockAdapters.run_with_lock(
378
+ update_content_index, ProcessLock.Operation.INDEX_CONTENT, max_duration_in_seconds=60 * 60 * 2
379
+ )
380
+
381
+
382
+ def configure_search_types():
383
+ # Extract core search types
384
+ core_search_types = {e.name: e.value for e in SearchType}
385
+
386
+ # Dynamically generate search type enum by merging core search types with configured plugin search types
387
+ return Enum("SearchType", core_search_types)
388
+
389
+
390
+ @schedule.repeat(schedule.every(2).minutes)
391
+ @clean_connections
392
+ def upload_telemetry():
393
+ if telemetry_disabled(state.config.app, state.telemetry_disabled) or not state.telemetry:
394
+ return
395
+
396
+ try:
397
+ logger.info(f"📡 Uploading telemetry to {constants.telemetry_server}...")
398
+ logger.debug(f"Telemetry state:\n{state.telemetry}")
399
+ for log in state.telemetry:
400
+ for field in log:
401
+ # Check if the value for the field is JSON serializable
402
+ if log[field] is None:
403
+ log[field] = ""
404
+ try:
405
+ json.dumps(log[field])
406
+ except TypeError:
407
+ log[field] = str(log[field])
408
+ response = requests.post(constants.telemetry_server, json=state.telemetry)
409
+ response.raise_for_status()
410
+ except Exception as e:
411
+ logger.error(f"📡 Error uploading telemetry: {e}", exc_info=True)
412
+ else:
413
+ state.telemetry = []
414
+
415
+
416
+ @schedule.repeat(schedule.every(31).minutes)
417
+ @clean_connections
418
+ def delete_old_user_requests():
419
+ num_deleted = delete_user_requests()
420
+ logger.debug(f"🗑️ Deleted {num_deleted[0]} day-old user requests")
421
+
422
+
423
+ @schedule.repeat(schedule.every(17).minutes)
424
+ @clean_connections
425
+ def wakeup_scheduler():
426
+ # Wake up the scheduler to ensure it runs the scheduled tasks. This is because the elected leader may not always be aware of tasks scheduled on other workers.
427
+ TWELVE_HOURS = 43200
428
+
429
+ # If the worker currently possesses a process lock, check if it is valid.
430
+
431
+ if state.schedule_leader_process_lock:
432
+ if not ProcessLockAdapters.is_process_locked(state.schedule_leader_process_lock):
433
+ state.schedule_leader_process_lock = None
434
+ state.scheduler.pause()
435
+
436
+ # Get the current process lock
437
+ schedule_leader_process_lock = ProcessLockAdapters.get_process_lock(ProcessLock.Operation.SCHEDULE_LEADER)
438
+
439
+ # Check if the process lock is still active. If not, create a new process lock. This worker will become the scheduler leader.
440
+ if not schedule_leader_process_lock or not ProcessLockAdapters.is_process_locked(schedule_leader_process_lock):
441
+ schedule_leader_process_lock = ProcessLockAdapters.set_process_lock(
442
+ ProcessLock.Operation.SCHEDULE_LEADER, max_duration_in_seconds=TWELVE_HOURS
443
+ )
444
+ state.schedule_leader_process_lock = schedule_leader_process_lock
445
+ state.scheduler.resume()
446
+ logger.info("🔔 Scheduler leader process lock acquired")
447
+
448
+ if state.schedule_leader_process_lock:
449
+ state.scheduler.wakeup()
450
+ else:
451
+ # Make sure the other workers don't run the scheduled tasks
452
+ state.scheduler.pause()
File without changes