khoj 1.33.3.dev32__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (393) hide show
  1. khoj/__init__.py +0 -0
  2. khoj/app/README.md +94 -0
  3. khoj/app/__init__.py +0 -0
  4. khoj/app/asgi.py +16 -0
  5. khoj/app/settings.py +218 -0
  6. khoj/app/urls.py +25 -0
  7. khoj/configure.py +452 -0
  8. khoj/database/__init__.py +0 -0
  9. khoj/database/adapters/__init__.py +1821 -0
  10. khoj/database/admin.py +417 -0
  11. khoj/database/apps.py +6 -0
  12. khoj/database/management/__init__.py +0 -0
  13. khoj/database/management/commands/__init__.py +0 -0
  14. khoj/database/management/commands/change_default_model.py +116 -0
  15. khoj/database/management/commands/change_generated_images_url.py +61 -0
  16. khoj/database/management/commands/convert_images_png_to_webp.py +99 -0
  17. khoj/database/migrations/0001_khojuser.py +98 -0
  18. khoj/database/migrations/0002_googleuser.py +32 -0
  19. khoj/database/migrations/0003_vector_extension.py +10 -0
  20. khoj/database/migrations/0004_content_types_and_more.py +181 -0
  21. khoj/database/migrations/0005_embeddings_corpus_id.py +19 -0
  22. khoj/database/migrations/0006_embeddingsdates.py +33 -0
  23. khoj/database/migrations/0007_add_conversation.py +27 -0
  24. khoj/database/migrations/0008_alter_conversation_conversation_log.py +17 -0
  25. khoj/database/migrations/0009_khojapiuser.py +24 -0
  26. khoj/database/migrations/0010_chatmodeloptions_and_more.py +83 -0
  27. khoj/database/migrations/0010_rename_embeddings_entry_and_more.py +30 -0
  28. khoj/database/migrations/0011_merge_20231102_0138.py +14 -0
  29. khoj/database/migrations/0012_entry_file_source.py +21 -0
  30. khoj/database/migrations/0013_subscription.py +37 -0
  31. khoj/database/migrations/0014_alter_googleuser_picture.py +17 -0
  32. khoj/database/migrations/0015_alter_subscription_user.py +21 -0
  33. khoj/database/migrations/0016_alter_subscription_renewal_date.py +17 -0
  34. khoj/database/migrations/0017_searchmodel.py +32 -0
  35. khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py +30 -0
  36. khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py +27 -0
  37. khoj/database/migrations/0020_reflectivequestion.py +36 -0
  38. khoj/database/migrations/0021_speechtotextmodeloptions_and_more.py +42 -0
  39. khoj/database/migrations/0022_texttoimagemodelconfig.py +25 -0
  40. khoj/database/migrations/0023_usersearchmodelconfig.py +33 -0
  41. khoj/database/migrations/0024_alter_entry_embeddings.py +18 -0
  42. khoj/database/migrations/0025_clientapplication_khojuser_phone_number_and_more.py +46 -0
  43. khoj/database/migrations/0025_searchmodelconfig_embeddings_inference_endpoint_and_more.py +22 -0
  44. khoj/database/migrations/0026_searchmodelconfig_cross_encoder_inference_endpoint_and_more.py +22 -0
  45. khoj/database/migrations/0027_merge_20240118_1324.py +13 -0
  46. khoj/database/migrations/0028_khojuser_verified_phone_number.py +17 -0
  47. khoj/database/migrations/0029_userrequests.py +27 -0
  48. khoj/database/migrations/0030_conversation_slug_and_title.py +38 -0
  49. khoj/database/migrations/0031_agent_conversation_agent.py +53 -0
  50. khoj/database/migrations/0031_alter_googleuser_locale.py +30 -0
  51. khoj/database/migrations/0032_merge_20240322_0427.py +14 -0
  52. khoj/database/migrations/0033_rename_tuning_agent_personality.py +17 -0
  53. khoj/database/migrations/0034_alter_chatmodeloptions_chat_model.py +32 -0
  54. khoj/database/migrations/0035_processlock.py +26 -0
  55. khoj/database/migrations/0036_alter_processlock_name.py +19 -0
  56. khoj/database/migrations/0036_delete_offlinechatprocessorconversationconfig.py +15 -0
  57. khoj/database/migrations/0036_publicconversation.py +42 -0
  58. khoj/database/migrations/0037_chatmodeloptions_openai_config_and_more.py +51 -0
  59. khoj/database/migrations/0037_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +32 -0
  60. khoj/database/migrations/0038_merge_20240425_0857.py +14 -0
  61. khoj/database/migrations/0038_merge_20240426_1640.py +12 -0
  62. khoj/database/migrations/0039_merge_20240501_0301.py +12 -0
  63. khoj/database/migrations/0040_alter_processlock_name.py +26 -0
  64. khoj/database/migrations/0040_merge_20240504_1010.py +14 -0
  65. khoj/database/migrations/0041_merge_20240505_1234.py +14 -0
  66. khoj/database/migrations/0042_serverchatsettings.py +46 -0
  67. khoj/database/migrations/0043_alter_chatmodeloptions_model_type.py +21 -0
  68. khoj/database/migrations/0044_conversation_file_filters.py +17 -0
  69. khoj/database/migrations/0045_fileobject.py +37 -0
  70. khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py +22 -0
  71. khoj/database/migrations/0047_alter_entry_file_type.py +31 -0
  72. khoj/database/migrations/0048_voicemodeloption_uservoicemodelconfig.py +52 -0
  73. khoj/database/migrations/0049_datastore.py +38 -0
  74. khoj/database/migrations/0049_texttoimagemodelconfig_api_key_and_more.py +58 -0
  75. khoj/database/migrations/0050_alter_processlock_name.py +25 -0
  76. khoj/database/migrations/0051_merge_20240702_1220.py +14 -0
  77. khoj/database/migrations/0052_alter_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +27 -0
  78. khoj/database/migrations/0053_agent_style_color_agent_style_icon.py +61 -0
  79. khoj/database/migrations/0054_alter_agent_style_color.py +38 -0
  80. khoj/database/migrations/0055_alter_agent_style_icon.py +37 -0
  81. khoj/database/migrations/0056_chatmodeloptions_vision_enabled.py +17 -0
  82. khoj/database/migrations/0056_searchmodelconfig_cross_encoder_model_config.py +17 -0
  83. khoj/database/migrations/0057_merge_20240816_1409.py +13 -0
  84. khoj/database/migrations/0057_remove_serverchatsettings_default_model_and_more.py +51 -0
  85. khoj/database/migrations/0058_alter_chatmodeloptions_chat_model.py +17 -0
  86. khoj/database/migrations/0059_searchmodelconfig_bi_encoder_confidence_threshold.py +17 -0
  87. khoj/database/migrations/0060_merge_20240905_1828.py +14 -0
  88. khoj/database/migrations/0061_alter_chatmodeloptions_model_type.py +26 -0
  89. khoj/database/migrations/0061_alter_texttoimagemodelconfig_model_type.py +21 -0
  90. khoj/database/migrations/0062_merge_20240913_0222.py +14 -0
  91. khoj/database/migrations/0063_conversation_temp_id.py +36 -0
  92. khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +86 -0
  93. khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
  94. khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
  95. khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
  96. khoj/database/migrations/0068_alter_agent_output_modes.py +24 -0
  97. khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py +89 -0
  98. khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +46 -0
  99. khoj/database/migrations/0071_subscription_enabled_trial_at_and_more.py +32 -0
  100. khoj/database/migrations/0072_entry_search_model.py +24 -0
  101. khoj/database/migrations/0073_delete_usersearchmodelconfig.py +15 -0
  102. khoj/database/migrations/0074_alter_conversation_title.py +17 -0
  103. khoj/database/migrations/0075_migrate_generated_assets_and_validate.py +85 -0
  104. khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py +26 -0
  105. khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py +62 -0
  106. khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py +17 -0
  107. khoj/database/migrations/__init__.py +0 -0
  108. khoj/database/models/__init__.py +725 -0
  109. khoj/database/tests.py +3 -0
  110. khoj/interface/compiled/404/index.html +1 -0
  111. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_buildManifest.js +1 -0
  112. khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_ssgManifest.js +1 -0
  113. khoj/interface/compiled/_next/static/chunks/1010-8f39bb4648b5ba10.js +1 -0
  114. khoj/interface/compiled/_next/static/chunks/182-f1c48a203dc91e0e.js +20 -0
  115. khoj/interface/compiled/_next/static/chunks/1915-d3c36ad6ce697ce7.js +1 -0
  116. khoj/interface/compiled/_next/static/chunks/2117-165ef4747a5b836b.js +2 -0
  117. khoj/interface/compiled/_next/static/chunks/2581-455000f8aeb08fc3.js +1 -0
  118. khoj/interface/compiled/_next/static/chunks/3727.dcea8f2193111552.js +1 -0
  119. khoj/interface/compiled/_next/static/chunks/3789-a09e37a819171a9d.js +1 -0
  120. khoj/interface/compiled/_next/static/chunks/4124-6c28322ce218d2d5.js +1 -0
  121. khoj/interface/compiled/_next/static/chunks/5427-b52d95253e692bfa.js +1 -0
  122. khoj/interface/compiled/_next/static/chunks/5473-b1cf56dedac6577a.js +1 -0
  123. khoj/interface/compiled/_next/static/chunks/5477-0bbddb79c25a54a7.js +1 -0
  124. khoj/interface/compiled/_next/static/chunks/6065-64db9ad305ba0bcd.js +1 -0
  125. khoj/interface/compiled/_next/static/chunks/6293-469dd16402ea8a6f.js +3 -0
  126. khoj/interface/compiled/_next/static/chunks/688-b5b4391bbc0376f1.js +1 -0
  127. khoj/interface/compiled/_next/static/chunks/8667-b6bf63c72b2d76eb.js +1 -0
  128. khoj/interface/compiled/_next/static/chunks/9259-1172dbaca0515237.js +1 -0
  129. khoj/interface/compiled/_next/static/chunks/94ca1967.1d9b42d929a1ee8c.js +1 -0
  130. khoj/interface/compiled/_next/static/chunks/9597.83583248dfbf6e73.js +1 -0
  131. khoj/interface/compiled/_next/static/chunks/964ecbae.51d6faf8801d15e6.js +1 -0
  132. khoj/interface/compiled/_next/static/chunks/9665-391df1e5c51c960a.js +1 -0
  133. khoj/interface/compiled/_next/static/chunks/app/_not-found/page-a834eddae3e235df.js +1 -0
  134. khoj/interface/compiled/_next/static/chunks/app/agents/layout-e00fb81dca656a10.js +1 -0
  135. khoj/interface/compiled/_next/static/chunks/app/agents/page-28ce086a1129bca2.js +1 -0
  136. khoj/interface/compiled/_next/static/chunks/app/automations/layout-1fe1537449f43496.js +1 -0
  137. khoj/interface/compiled/_next/static/chunks/app/automations/page-bf365a60829d347f.js +1 -0
  138. khoj/interface/compiled/_next/static/chunks/app/chat/layout-33934fc2d6ae6838.js +1 -0
  139. khoj/interface/compiled/_next/static/chunks/app/chat/page-0e476e57eb2015e3.js +1 -0
  140. khoj/interface/compiled/_next/static/chunks/app/layout-30e7fda7262713ce.js +1 -0
  141. khoj/interface/compiled/_next/static/chunks/app/page-a5515ea71aec5ef0.js +1 -0
  142. khoj/interface/compiled/_next/static/chunks/app/search/layout-c02531d586972d7d.js +1 -0
  143. khoj/interface/compiled/_next/static/chunks/app/search/page-9140541e67ea307d.js +1 -0
  144. khoj/interface/compiled/_next/static/chunks/app/settings/layout-d09d6510a45cd4bd.js +1 -0
  145. khoj/interface/compiled/_next/static/chunks/app/settings/page-951ba40b5b94b23a.js +1 -0
  146. khoj/interface/compiled/_next/static/chunks/app/share/chat/layout-e8e5db7830bf3f47.js +1 -0
  147. khoj/interface/compiled/_next/static/chunks/app/share/chat/page-1beb80d8d741c932.js +1 -0
  148. khoj/interface/compiled/_next/static/chunks/d3ac728e-44ebd2a0c99b12a0.js +1 -0
  149. khoj/interface/compiled/_next/static/chunks/fd9d1056-4482b99a36fd1673.js +1 -0
  150. khoj/interface/compiled/_next/static/chunks/framework-8e0e0f4a6b83a956.js +1 -0
  151. khoj/interface/compiled/_next/static/chunks/main-app-de1f09df97a3cfc7.js +1 -0
  152. khoj/interface/compiled/_next/static/chunks/main-db4bfac6b0a8d00b.js +1 -0
  153. khoj/interface/compiled/_next/static/chunks/pages/_app-3c9ca398d360b709.js +1 -0
  154. khoj/interface/compiled/_next/static/chunks/pages/_error-cf5ca766ac8f493f.js +1 -0
  155. khoj/interface/compiled/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
  156. khoj/interface/compiled/_next/static/chunks/webpack-a03962458328b163.js +1 -0
  157. khoj/interface/compiled/_next/static/css/089de1d8526b96e9.css +1 -0
  158. khoj/interface/compiled/_next/static/css/37a73b87f02df402.css +1 -0
  159. khoj/interface/compiled/_next/static/css/4e4e6a4a1c920d06.css +1 -0
  160. khoj/interface/compiled/_next/static/css/8d02837c730f8d13.css +25 -0
  161. khoj/interface/compiled/_next/static/css/8e6a3ca11a60b189.css +1 -0
  162. khoj/interface/compiled/_next/static/css/9c164d9727dd8092.css +1 -0
  163. khoj/interface/compiled/_next/static/css/dac88c17aaee5fcf.css +1 -0
  164. khoj/interface/compiled/_next/static/css/df4b47a2d0d85eae.css +1 -0
  165. khoj/interface/compiled/_next/static/css/e4eb883b5265d372.css +1 -0
  166. khoj/interface/compiled/_next/static/media/1d8a05b60287ae6c-s.p.woff2 +0 -0
  167. khoj/interface/compiled/_next/static/media/6f22fce21a7c433c-s.woff2 +0 -0
  168. khoj/interface/compiled/_next/static/media/77c207b095007c34-s.p.woff2 +0 -0
  169. khoj/interface/compiled/_next/static/media/82ef96de0e8f4d8c-s.p.woff2 +0 -0
  170. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.1608a09b.woff +0 -0
  171. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.4aafdb68.ttf +0 -0
  172. khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.a79f1c31.woff2 +0 -0
  173. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.b6770918.woff +0 -0
  174. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.cce5b8ec.ttf +0 -0
  175. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.ec17d132.woff2 +0 -0
  176. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.07ef19e7.ttf +0 -0
  177. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.55fac258.woff2 +0 -0
  178. khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.dad44a7f.woff +0 -0
  179. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.9f256b85.woff +0 -0
  180. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.b18f59e1.ttf +0 -0
  181. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.d42a5579.woff2 +0 -0
  182. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.7c187121.woff +0 -0
  183. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.d3c882a6.woff2 +0 -0
  184. khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.ed38e79f.ttf +0 -0
  185. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.b74a1a8b.ttf +0 -0
  186. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.c3fb5ac2.woff2 +0 -0
  187. khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.d181c465.woff +0 -0
  188. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.6f2bb1df.woff2 +0 -0
  189. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.70d8b0a5.ttf +0 -0
  190. khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.e3f82f9d.woff +0 -0
  191. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.47373d1e.ttf +0 -0
  192. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.8916142b.woff2 +0 -0
  193. khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.9024d815.woff +0 -0
  194. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.0462f03b.woff2 +0 -0
  195. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.7f51fe03.woff +0 -0
  196. khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.b7f8fe9b.ttf +0 -0
  197. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.572d331f.woff2 +0 -0
  198. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.a879cf83.ttf +0 -0
  199. khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.f1035d8d.woff +0 -0
  200. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.5295ba48.woff +0 -0
  201. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.939bc644.ttf +0 -0
  202. khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.f28c23ac.woff2 +0 -0
  203. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.8c5b5494.woff2 +0 -0
  204. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.94e1e8dc.ttf +0 -0
  205. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.bf59d231.woff +0 -0
  206. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.3b1e59b3.woff2 +0 -0
  207. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.7c9bc82b.woff +0 -0
  208. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.b4c20c84.ttf +0 -0
  209. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.74048478.woff +0 -0
  210. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.ba21ed5f.woff2 +0 -0
  211. khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.d4d7ba48.ttf +0 -0
  212. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.03e9641d.woff2 +0 -0
  213. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.07505710.woff +0 -0
  214. khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.fe9cbbe1.ttf +0 -0
  215. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.e1e279cb.woff +0 -0
  216. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.eae34984.woff2 +0 -0
  217. khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.fabc004a.ttf +0 -0
  218. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.57727022.woff +0 -0
  219. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.5916a24f.woff2 +0 -0
  220. khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.d6b476ec.ttf +0 -0
  221. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.9acaf01c.woff +0 -0
  222. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.a144ef58.ttf +0 -0
  223. khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.b4230e7e.woff2 +0 -0
  224. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.10d95fd3.woff2 +0 -0
  225. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.7a996c9d.woff +0 -0
  226. khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.fbccdabe.ttf +0 -0
  227. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.6258592b.woff +0 -0
  228. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.a8709e36.woff2 +0 -0
  229. khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.d97aaf4a.ttf +0 -0
  230. khoj/interface/compiled/_next/static/media/a6ecd16fa044d500-s.p.woff2 +0 -0
  231. khoj/interface/compiled/_next/static/media/bd82c78e5b7b3fe9-s.p.woff2 +0 -0
  232. khoj/interface/compiled/_next/static/media/c32c8052c071fc42-s.woff2 +0 -0
  233. khoj/interface/compiled/_next/static/media/c4250770ab8708b6-s.p.woff2 +0 -0
  234. khoj/interface/compiled/_next/static/media/e098aaaecc9cfbb2-s.p.woff2 +0 -0
  235. khoj/interface/compiled/_next/static/media/flags.3afdda2f.webp +0 -0
  236. khoj/interface/compiled/_next/static/media/flags@2x.5fbe9fc1.webp +0 -0
  237. khoj/interface/compiled/_next/static/media/globe.98e105ca.webp +0 -0
  238. khoj/interface/compiled/_next/static/media/globe@2x.974df6f8.webp +0 -0
  239. khoj/interface/compiled/agents/index.html +1 -0
  240. khoj/interface/compiled/agents/index.txt +7 -0
  241. khoj/interface/compiled/agents.svg +6 -0
  242. khoj/interface/compiled/assets/icons/khoj_lantern.ico +0 -0
  243. khoj/interface/compiled/assets/icons/khoj_lantern.svg +100 -0
  244. khoj/interface/compiled/assets/icons/khoj_lantern_1200x1200.png +0 -0
  245. khoj/interface/compiled/assets/icons/khoj_lantern_128x128.png +0 -0
  246. khoj/interface/compiled/assets/icons/khoj_lantern_128x128_dark.png +0 -0
  247. khoj/interface/compiled/assets/icons/khoj_lantern_256x256.png +0 -0
  248. khoj/interface/compiled/assets/icons/khoj_lantern_512x512.png +0 -0
  249. khoj/interface/compiled/assets/icons/khoj_lantern_logomarktype_1200x630.png +0 -0
  250. khoj/interface/compiled/assets/samples/desktop-browse-draw-sample.png +0 -0
  251. khoj/interface/compiled/assets/samples/desktop-plain-chat-sample.png +0 -0
  252. khoj/interface/compiled/assets/samples/desktop-remember-plan-sample.png +0 -0
  253. khoj/interface/compiled/assets/samples/phone-browse-draw-sample.png +0 -0
  254. khoj/interface/compiled/assets/samples/phone-plain-chat-sample.png +0 -0
  255. khoj/interface/compiled/assets/samples/phone-remember-plan-sample.png +0 -0
  256. khoj/interface/compiled/automation.svg +37 -0
  257. khoj/interface/compiled/automations/index.html +1 -0
  258. khoj/interface/compiled/automations/index.txt +8 -0
  259. khoj/interface/compiled/chat/index.html +1 -0
  260. khoj/interface/compiled/chat/index.txt +7 -0
  261. khoj/interface/compiled/chat.svg +24 -0
  262. khoj/interface/compiled/close.svg +5 -0
  263. khoj/interface/compiled/copy-button-success.svg +6 -0
  264. khoj/interface/compiled/copy-button.svg +5 -0
  265. khoj/interface/compiled/index.html +1 -0
  266. khoj/interface/compiled/index.txt +7 -0
  267. khoj/interface/compiled/khoj.webmanifest +76 -0
  268. khoj/interface/compiled/logo.svg +24 -0
  269. khoj/interface/compiled/search/index.html +1 -0
  270. khoj/interface/compiled/search/index.txt +7 -0
  271. khoj/interface/compiled/send.svg +1 -0
  272. khoj/interface/compiled/settings/index.html +1 -0
  273. khoj/interface/compiled/settings/index.txt +9 -0
  274. khoj/interface/compiled/share/chat/index.html +1 -0
  275. khoj/interface/compiled/share/chat/index.txt +7 -0
  276. khoj/interface/compiled/share.svg +8 -0
  277. khoj/interface/compiled/thumbs-down.svg +6 -0
  278. khoj/interface/compiled/thumbs-up.svg +6 -0
  279. khoj/interface/email/feedback.html +34 -0
  280. khoj/interface/email/magic_link.html +40 -0
  281. khoj/interface/email/task.html +37 -0
  282. khoj/interface/email/welcome.html +90 -0
  283. khoj/interface/web/.well-known/assetlinks.json +11 -0
  284. khoj/interface/web/assets/icons/agents.svg +19 -0
  285. khoj/interface/web/assets/icons/automation.svg +43 -0
  286. khoj/interface/web/assets/icons/chat.svg +24 -0
  287. khoj/interface/web/assets/icons/github.svg +1 -0
  288. khoj/interface/web/assets/icons/khoj-logo-sideways-200.png +0 -0
  289. khoj/interface/web/assets/icons/khoj-logo-sideways-500.png +0 -0
  290. khoj/interface/web/assets/icons/khoj-logo-sideways.svg +32 -0
  291. khoj/interface/web/assets/icons/khoj.svg +26 -0
  292. khoj/interface/web/assets/icons/logotype.svg +1 -0
  293. khoj/interface/web/assets/icons/search.svg +57 -0
  294. khoj/interface/web/assets/icons/sync.svg +4 -0
  295. khoj/interface/web/assets/khoj.css +237 -0
  296. khoj/interface/web/assets/utils.js +33 -0
  297. khoj/interface/web/base_config.html +445 -0
  298. khoj/interface/web/content_source_github_input.html +208 -0
  299. khoj/interface/web/login.html +310 -0
  300. khoj/interface/web/utils.html +48 -0
  301. khoj/main.py +249 -0
  302. khoj/manage.py +22 -0
  303. khoj/migrations/__init__.py +0 -0
  304. khoj/migrations/migrate_offline_chat_default_model.py +69 -0
  305. khoj/migrations/migrate_offline_chat_default_model_2.py +71 -0
  306. khoj/migrations/migrate_offline_chat_schema.py +83 -0
  307. khoj/migrations/migrate_offline_model.py +29 -0
  308. khoj/migrations/migrate_processor_config_openai.py +67 -0
  309. khoj/migrations/migrate_server_pg.py +132 -0
  310. khoj/migrations/migrate_version.py +17 -0
  311. khoj/processor/__init__.py +0 -0
  312. khoj/processor/content/__init__.py +0 -0
  313. khoj/processor/content/docx/__init__.py +0 -0
  314. khoj/processor/content/docx/docx_to_entries.py +111 -0
  315. khoj/processor/content/github/__init__.py +0 -0
  316. khoj/processor/content/github/github_to_entries.py +226 -0
  317. khoj/processor/content/images/__init__.py +0 -0
  318. khoj/processor/content/images/image_to_entries.py +117 -0
  319. khoj/processor/content/markdown/__init__.py +0 -0
  320. khoj/processor/content/markdown/markdown_to_entries.py +160 -0
  321. khoj/processor/content/notion/notion_to_entries.py +259 -0
  322. khoj/processor/content/org_mode/__init__.py +0 -0
  323. khoj/processor/content/org_mode/org_to_entries.py +226 -0
  324. khoj/processor/content/org_mode/orgnode.py +532 -0
  325. khoj/processor/content/pdf/__init__.py +0 -0
  326. khoj/processor/content/pdf/pdf_to_entries.py +119 -0
  327. khoj/processor/content/plaintext/__init__.py +0 -0
  328. khoj/processor/content/plaintext/plaintext_to_entries.py +117 -0
  329. khoj/processor/content/text_to_entries.py +296 -0
  330. khoj/processor/conversation/__init__.py +0 -0
  331. khoj/processor/conversation/anthropic/__init__.py +0 -0
  332. khoj/processor/conversation/anthropic/anthropic_chat.py +243 -0
  333. khoj/processor/conversation/anthropic/utils.py +217 -0
  334. khoj/processor/conversation/google/__init__.py +0 -0
  335. khoj/processor/conversation/google/gemini_chat.py +253 -0
  336. khoj/processor/conversation/google/utils.py +260 -0
  337. khoj/processor/conversation/offline/__init__.py +0 -0
  338. khoj/processor/conversation/offline/chat_model.py +308 -0
  339. khoj/processor/conversation/offline/utils.py +80 -0
  340. khoj/processor/conversation/offline/whisper.py +15 -0
  341. khoj/processor/conversation/openai/__init__.py +0 -0
  342. khoj/processor/conversation/openai/gpt.py +243 -0
  343. khoj/processor/conversation/openai/utils.py +232 -0
  344. khoj/processor/conversation/openai/whisper.py +13 -0
  345. khoj/processor/conversation/prompts.py +1188 -0
  346. khoj/processor/conversation/utils.py +867 -0
  347. khoj/processor/embeddings.py +122 -0
  348. khoj/processor/image/generate.py +215 -0
  349. khoj/processor/speech/__init__.py +0 -0
  350. khoj/processor/speech/text_to_speech.py +51 -0
  351. khoj/processor/tools/__init__.py +0 -0
  352. khoj/processor/tools/online_search.py +472 -0
  353. khoj/processor/tools/run_code.py +179 -0
  354. khoj/routers/__init__.py +0 -0
  355. khoj/routers/api.py +760 -0
  356. khoj/routers/api_agents.py +295 -0
  357. khoj/routers/api_chat.py +1273 -0
  358. khoj/routers/api_content.py +634 -0
  359. khoj/routers/api_model.py +123 -0
  360. khoj/routers/api_phone.py +86 -0
  361. khoj/routers/api_subscription.py +144 -0
  362. khoj/routers/auth.py +307 -0
  363. khoj/routers/email.py +135 -0
  364. khoj/routers/helpers.py +2333 -0
  365. khoj/routers/notion.py +85 -0
  366. khoj/routers/research.py +364 -0
  367. khoj/routers/storage.py +63 -0
  368. khoj/routers/twilio.py +36 -0
  369. khoj/routers/web_client.py +141 -0
  370. khoj/search_filter/__init__.py +0 -0
  371. khoj/search_filter/base_filter.py +15 -0
  372. khoj/search_filter/date_filter.py +215 -0
  373. khoj/search_filter/file_filter.py +32 -0
  374. khoj/search_filter/word_filter.py +29 -0
  375. khoj/search_type/__init__.py +0 -0
  376. khoj/search_type/text_search.py +255 -0
  377. khoj/utils/__init__.py +0 -0
  378. khoj/utils/cli.py +101 -0
  379. khoj/utils/config.py +81 -0
  380. khoj/utils/constants.py +51 -0
  381. khoj/utils/fs_syncer.py +252 -0
  382. khoj/utils/helpers.py +627 -0
  383. khoj/utils/initialization.py +301 -0
  384. khoj/utils/jsonl.py +43 -0
  385. khoj/utils/models.py +47 -0
  386. khoj/utils/rawconfig.py +208 -0
  387. khoj/utils/state.py +48 -0
  388. khoj/utils/yaml.py +47 -0
  389. khoj-1.33.3.dev32.dist-info/METADATA +190 -0
  390. khoj-1.33.3.dev32.dist-info/RECORD +393 -0
  391. khoj-1.33.3.dev32.dist-info/WHEEL +4 -0
  392. khoj-1.33.3.dev32.dist-info/entry_points.txt +2 -0
  393. khoj-1.33.3.dev32.dist-info/licenses/LICENSE +661 -0
khoj/routers/notion.py ADDED
@@ -0,0 +1,85 @@
1
+ import asyncio
2
+ import base64
3
+ import json
4
+ import logging
5
+ import os
6
+ from concurrent.futures import ThreadPoolExecutor
7
+
8
+ import requests
9
+ from fastapi import APIRouter, BackgroundTasks, Request, Response
10
+ from starlette.responses import RedirectResponse
11
+
12
+ from khoj.database.adapters import aget_user_by_uuid
13
+ from khoj.database.models import KhojUser, NotionConfig
14
+ from khoj.routers.helpers import configure_content
15
+ from khoj.utils.state import SearchType
16
+
17
+ NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID")
18
+ NOTION_OAUTH_CLIENT_SECRET = os.getenv("NOTION_OAUTH_CLIENT_SECRET")
19
+ NOTION_REDIRECT_URI = os.getenv("NOTION_REDIRECT_URI")
20
+
21
+ notion_router = APIRouter()
22
+
23
+ executor = ThreadPoolExecutor()
24
+
25
+ logger = logging.getLogger(__name__)
26
+
27
+
28
+ async def run_in_executor(func, *args):
29
+ loop = asyncio.get_event_loop()
30
+ return await loop.run_in_executor(executor, func, *args)
31
+
32
+
33
+ @notion_router.get("/auth/callback")
34
+ async def notion_auth_callback(request: Request, background_tasks: BackgroundTasks):
35
+ code = request.query_params.get("code")
36
+ state = request.query_params.get("state")
37
+ if not code or not state:
38
+ return Response("Missing code or state", status_code=400)
39
+
40
+ user: KhojUser = await aget_user_by_uuid(state)
41
+
42
+ await NotionConfig.objects.filter(user=user).adelete()
43
+
44
+ if not user:
45
+ raise Exception("User not found")
46
+
47
+ bearer_token = f"{NOTION_OAUTH_CLIENT_ID}:{NOTION_OAUTH_CLIENT_SECRET}"
48
+ base64_encoded_token = base64.b64encode(bearer_token.encode()).decode()
49
+
50
+ headers = {
51
+ "Accept": "application/json",
52
+ "Content-Type": "application/json",
53
+ "Authorization": f"Basic {base64_encoded_token}",
54
+ }
55
+
56
+ data = {
57
+ "grant_type": "authorization_code",
58
+ "code": code,
59
+ "redirect_uri": NOTION_REDIRECT_URI,
60
+ }
61
+
62
+ response = requests.post("https://api.notion.com/v1/oauth/token", data=json.dumps(data), headers=headers)
63
+
64
+ final_response = response.json()
65
+
66
+ logger.info(f"Notion auth callback response: {final_response}")
67
+
68
+ access_token = final_response.get("access_token")
69
+ await NotionConfig.objects.acreate(token=access_token, user=user)
70
+
71
+ owner = final_response.get("owner")
72
+ workspace_id = final_response.get("workspace_id")
73
+ workspace_name = final_response.get("workspace_name")
74
+ bot_id = final_response.get("bot_id")
75
+
76
+ logger.info(
77
+ f"Notion integration. Owner: {owner}, Workspace ID: {workspace_id}, Workspace Name: {workspace_name}, Bot ID: {bot_id}"
78
+ )
79
+
80
+ notion_redirect = str(request.app.url_path_for("config_page"))
81
+
82
+ # Trigger an async job to configure_content. Let it run without blocking the response.
83
+ background_tasks.add_task(run_in_executor, configure_content, user, {}, False, SearchType.Notion)
84
+
85
+ return RedirectResponse(notion_redirect)
@@ -0,0 +1,364 @@
1
+ import logging
2
+ from datetime import datetime
3
+ from typing import Callable, Dict, List, Optional
4
+
5
+ import yaml
6
+ from fastapi import Request
7
+
8
+ from khoj.database.models import Agent, KhojUser
9
+ from khoj.processor.conversation import prompts
10
+ from khoj.processor.conversation.utils import (
11
+ InformationCollectionIteration,
12
+ construct_chat_history,
13
+ construct_iteration_history,
14
+ construct_tool_chat_history,
15
+ load_complex_json,
16
+ )
17
+ from khoj.processor.tools.online_search import read_webpages, search_online
18
+ from khoj.processor.tools.run_code import run_code
19
+ from khoj.routers.api import extract_references_and_questions
20
+ from khoj.routers.helpers import (
21
+ ChatEvent,
22
+ generate_summary_from_files,
23
+ send_message_to_model_wrapper,
24
+ )
25
+ from khoj.utils.helpers import (
26
+ ConversationCommand,
27
+ function_calling_description_for_llm,
28
+ is_none_or_empty,
29
+ timer,
30
+ truncate_code_context,
31
+ )
32
+ from khoj.utils.rawconfig import LocationData
33
+
34
+ logger = logging.getLogger(__name__)
35
+
36
+
37
+ async def apick_next_tool(
38
+ query: str,
39
+ conversation_history: dict,
40
+ user: KhojUser = None,
41
+ query_images: List[str] = [],
42
+ location: LocationData = None,
43
+ user_name: str = None,
44
+ agent: Agent = None,
45
+ previous_iterations: List[InformationCollectionIteration] = [],
46
+ max_iterations: int = 5,
47
+ send_status_func: Optional[Callable] = None,
48
+ tracer: dict = {},
49
+ query_files: str = None,
50
+ ):
51
+ """Given a query, determine which of the available tools the agent should use in order to answer appropriately."""
52
+
53
+ # Construct tool options for the agent to choose from
54
+ tool_options = dict()
55
+ tool_options_str = ""
56
+ agent_tools = agent.input_tools if agent else []
57
+ for tool, description in function_calling_description_for_llm.items():
58
+ tool_options[tool.value] = description
59
+ if len(agent_tools) == 0 or tool.value in agent_tools:
60
+ tool_options_str += f'- "{tool.value}": "{description}"\n'
61
+
62
+ # Construct chat history with user and iteration history with researcher agent for context
63
+ chat_history = construct_chat_history(conversation_history, agent_name=agent.name if agent else "Khoj")
64
+ previous_iterations_history = construct_iteration_history(previous_iterations, prompts.previous_iteration)
65
+
66
+ if query_images:
67
+ query = f"[placeholder for user attached images]\n{query}"
68
+
69
+ today = datetime.today()
70
+ location_data = f"{location}" if location else "Unknown"
71
+ personality_context = (
72
+ prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
73
+ )
74
+
75
+ function_planning_prompt = prompts.plan_function_execution.format(
76
+ tools=tool_options_str,
77
+ chat_history=chat_history,
78
+ personality_context=personality_context,
79
+ current_date=today.strftime("%Y-%m-%d"),
80
+ day_of_week=today.strftime("%A"),
81
+ username=user_name or "Unknown",
82
+ location=location_data,
83
+ previous_iterations=previous_iterations_history,
84
+ max_iterations=max_iterations,
85
+ )
86
+
87
+ try:
88
+ with timer("Chat actor: Infer information sources to refer", logger):
89
+ response = await send_message_to_model_wrapper(
90
+ query=query,
91
+ context=function_planning_prompt,
92
+ response_type="json_object",
93
+ user=user,
94
+ query_images=query_images,
95
+ query_files=query_files,
96
+ tracer=tracer,
97
+ )
98
+ except Exception as e:
99
+ logger.error(f"Failed to infer information sources to refer: {e}", exc_info=True)
100
+ yield InformationCollectionIteration(
101
+ tool=None,
102
+ query=None,
103
+ warning="Failed to infer information sources to refer. Skipping iteration. Try again.",
104
+ )
105
+ return
106
+
107
+ try:
108
+ response = load_complex_json(response)
109
+ selected_tool = response.get("tool", None)
110
+ generated_query = response.get("query", None)
111
+ scratchpad = response.get("scratchpad", None)
112
+ warning = None
113
+ logger.info(f"Response for determining relevant tools: {response}")
114
+
115
+ # Detect selection of previously used query, tool combination.
116
+ previous_tool_query_combinations = {(i.tool, i.query) for i in previous_iterations if i.warning is None}
117
+ if (selected_tool, generated_query) in previous_tool_query_combinations:
118
+ warning = f"Repeated tool, query combination detected. Skipping iteration. Try something different."
119
+ # Only send client status updates if we'll execute this iteration
120
+ elif send_status_func:
121
+ determined_tool_message = "**Determined Tool**: "
122
+ determined_tool_message += f"{selected_tool}({generated_query})." if selected_tool else "respond."
123
+ determined_tool_message += f"\nReason: {scratchpad}" if scratchpad else ""
124
+ async for event in send_status_func(f"{scratchpad}"):
125
+ yield {ChatEvent.STATUS: event}
126
+
127
+ yield InformationCollectionIteration(
128
+ tool=selected_tool,
129
+ query=generated_query,
130
+ warning=warning,
131
+ )
132
+ except Exception as e:
133
+ logger.error(f"Invalid response for determining relevant tools: {response}. {e}", exc_info=True)
134
+ yield InformationCollectionIteration(
135
+ tool=None,
136
+ query=None,
137
+ warning=f"Invalid response for determining relevant tools: {response}. Skipping iteration. Fix error: {e}",
138
+ )
139
+
140
+
141
+ async def execute_information_collection(
142
+ request: Request,
143
+ user: KhojUser,
144
+ query: str,
145
+ conversation_id: str,
146
+ conversation_history: dict,
147
+ query_images: List[str],
148
+ agent: Agent = None,
149
+ send_status_func: Optional[Callable] = None,
150
+ user_name: str = None,
151
+ location: LocationData = None,
152
+ file_filters: List[str] = [],
153
+ tracer: dict = {},
154
+ query_files: str = None,
155
+ ):
156
+ current_iteration = 0
157
+ MAX_ITERATIONS = 5
158
+ previous_iterations: List[InformationCollectionIteration] = []
159
+ while current_iteration < MAX_ITERATIONS:
160
+ online_results: Dict = dict()
161
+ code_results: Dict = dict()
162
+ document_results: List[Dict[str, str]] = []
163
+ summarize_files: str = ""
164
+ this_iteration = InformationCollectionIteration(tool=None, query=query)
165
+
166
+ async for result in apick_next_tool(
167
+ query,
168
+ conversation_history,
169
+ user,
170
+ query_images,
171
+ location,
172
+ user_name,
173
+ agent,
174
+ previous_iterations,
175
+ MAX_ITERATIONS,
176
+ send_status_func,
177
+ tracer=tracer,
178
+ query_files=query_files,
179
+ ):
180
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
181
+ yield result[ChatEvent.STATUS]
182
+ elif isinstance(result, InformationCollectionIteration):
183
+ this_iteration = result
184
+
185
+ # Skip running iteration if warning present in iteration
186
+ if this_iteration.warning:
187
+ logger.warning(f"Research mode: {this_iteration.warning}.")
188
+
189
+ elif this_iteration.tool == ConversationCommand.Notes:
190
+ this_iteration.context = []
191
+ document_results = []
192
+ previous_inferred_queries = {
193
+ c["query"] for iteration in previous_iterations if iteration.context for c in iteration.context
194
+ }
195
+ async for result in extract_references_and_questions(
196
+ request,
197
+ construct_tool_chat_history(previous_iterations, ConversationCommand.Notes),
198
+ this_iteration.query,
199
+ 7,
200
+ None,
201
+ conversation_id,
202
+ [ConversationCommand.Default],
203
+ location,
204
+ send_status_func,
205
+ query_images,
206
+ previous_inferred_queries=previous_inferred_queries,
207
+ agent=agent,
208
+ tracer=tracer,
209
+ query_files=query_files,
210
+ ):
211
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
212
+ yield result[ChatEvent.STATUS]
213
+ elif isinstance(result, tuple):
214
+ document_results = result[0]
215
+ this_iteration.context += document_results
216
+
217
+ if not is_none_or_empty(document_results):
218
+ try:
219
+ distinct_files = {d["file"] for d in document_results}
220
+ distinct_headings = set([d["compiled"].split("\n")[0] for d in document_results if "compiled" in d])
221
+ # Strip only leading # from headings
222
+ headings_str = "\n- " + "\n- ".join(distinct_headings).replace("#", "")
223
+ async for result in send_status_func(
224
+ f"**Found {len(distinct_headings)} Notes Across {len(distinct_files)} Files**: {headings_str}"
225
+ ):
226
+ yield result
227
+ except Exception as e:
228
+ this_iteration.warning = f"Error extracting document references: {e}"
229
+ logger.error(this_iteration.warning, exc_info=True)
230
+
231
+ elif this_iteration.tool == ConversationCommand.Online:
232
+ previous_subqueries = {
233
+ subquery
234
+ for iteration in previous_iterations
235
+ if iteration.onlineContext
236
+ for subquery in iteration.onlineContext.keys()
237
+ }
238
+ try:
239
+ async for result in search_online(
240
+ this_iteration.query,
241
+ construct_tool_chat_history(previous_iterations, ConversationCommand.Online),
242
+ location,
243
+ user,
244
+ send_status_func,
245
+ [],
246
+ max_webpages_to_read=0,
247
+ query_images=query_images,
248
+ previous_subqueries=previous_subqueries,
249
+ agent=agent,
250
+ tracer=tracer,
251
+ ):
252
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
253
+ yield result[ChatEvent.STATUS]
254
+ elif is_none_or_empty(result):
255
+ this_iteration.warning = "Detected previously run online search queries. Skipping iteration. Try something different."
256
+ else:
257
+ online_results: Dict[str, Dict] = result # type: ignore
258
+ this_iteration.onlineContext = online_results
259
+ except Exception as e:
260
+ this_iteration.warning = f"Error searching online: {e}"
261
+ logger.error(this_iteration.warning, exc_info=True)
262
+
263
+ elif this_iteration.tool == ConversationCommand.Webpage:
264
+ try:
265
+ async for result in read_webpages(
266
+ this_iteration.query,
267
+ construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
268
+ location,
269
+ user,
270
+ send_status_func,
271
+ query_images=query_images,
272
+ agent=agent,
273
+ tracer=tracer,
274
+ query_files=query_files,
275
+ ):
276
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
277
+ yield result[ChatEvent.STATUS]
278
+ else:
279
+ direct_web_pages: Dict[str, Dict] = result # type: ignore
280
+
281
+ webpages = []
282
+ for web_query in direct_web_pages:
283
+ if online_results.get(web_query):
284
+ online_results[web_query]["webpages"] = direct_web_pages[web_query]["webpages"]
285
+ else:
286
+ online_results[web_query] = {"webpages": direct_web_pages[web_query]["webpages"]}
287
+
288
+ for webpage in direct_web_pages[web_query]["webpages"]:
289
+ webpages.append(webpage["link"])
290
+ this_iteration.onlineContext = online_results
291
+ except Exception as e:
292
+ this_iteration.warning = f"Error reading webpages: {e}"
293
+ logger.error(this_iteration.warning, exc_info=True)
294
+
295
+ elif this_iteration.tool == ConversationCommand.Code:
296
+ try:
297
+ async for result in run_code(
298
+ this_iteration.query,
299
+ construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
300
+ "",
301
+ location,
302
+ user,
303
+ send_status_func,
304
+ query_images=query_images,
305
+ agent=agent,
306
+ query_files=query_files,
307
+ tracer=tracer,
308
+ ):
309
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
310
+ yield result[ChatEvent.STATUS]
311
+ else:
312
+ code_results: Dict[str, Dict] = result # type: ignore
313
+ this_iteration.codeContext = code_results
314
+ async for result in send_status_func(f"**Ran code snippets**: {len(this_iteration.codeContext)}"):
315
+ yield result
316
+ except ValueError as e:
317
+ this_iteration.warning = f"Error running code: {e}"
318
+ logger.warning(this_iteration.warning, exc_info=True)
319
+
320
+ elif this_iteration.tool == ConversationCommand.Summarize:
321
+ try:
322
+ async for result in generate_summary_from_files(
323
+ this_iteration.query,
324
+ user,
325
+ file_filters,
326
+ construct_tool_chat_history(previous_iterations),
327
+ query_images=query_images,
328
+ agent=agent,
329
+ send_status_func=send_status_func,
330
+ query_files=query_files,
331
+ ):
332
+ if isinstance(result, dict) and ChatEvent.STATUS in result:
333
+ yield result[ChatEvent.STATUS]
334
+ else:
335
+ summarize_files = result # type: ignore
336
+ except Exception as e:
337
+ this_iteration.warning = f"Error summarizing files: {e}"
338
+ logger.error(this_iteration.warning, exc_info=True)
339
+
340
+ else:
341
+ # No valid tools. This is our exit condition.
342
+ current_iteration = MAX_ITERATIONS
343
+
344
+ current_iteration += 1
345
+
346
+ if document_results or online_results or code_results or summarize_files or this_iteration.warning:
347
+ results_data = f"\n<iteration>{current_iteration}\n<tool>{this_iteration.tool}</tool>\n<query>{this_iteration.query}</query>\n<results>"
348
+ if document_results:
349
+ results_data += f"\n<document_references>\n{yaml.dump(document_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</document_references>"
350
+ if online_results:
351
+ results_data += f"\n<online_results>\n{yaml.dump(online_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</online_results>"
352
+ if code_results:
353
+ results_data += f"\n<code_results>\n{yaml.dump(truncate_code_context(code_results), allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</code_results>"
354
+ if summarize_files:
355
+ results_data += f"\n<summarized_files>\n{yaml.dump(summarize_files, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</summarized_files>"
356
+ if this_iteration.warning:
357
+ results_data += f"\n<warning>\n{this_iteration.warning}\n</warning>"
358
+ results_data += "\n</results>\n</iteration>"
359
+
360
+ # intermediate_result = await extract_relevant_info(this_iteration.query, results_data, agent)
361
+ this_iteration.summarizedResult = results_data
362
+
363
+ previous_iterations.append(this_iteration)
364
+ yield this_iteration
@@ -0,0 +1,63 @@
1
+ import logging
2
+ import os
3
+ import uuid
4
+
5
+ logger = logging.getLogger(__name__)
6
+
7
+ AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY")
8
+ AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY")
9
+ # S3 supports serving assets via your domain. Khoj expects this to be used in production. To enable it:
10
+ # 1. Your bucket name for images should be of the form sub.domain.tld. For example, generated.khoj.dev
11
+ # 2. Add CNAME entry to your domain's DNS records pointing to the S3 bucket. For example, CNAME generated.khoj.dev generated-khoj-dev.s3.amazonaws.com
12
+ AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET")
13
+
14
+ aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None
15
+
16
+ if aws_enabled:
17
+ from boto3 import client
18
+
19
+ s3_client = client("s3", aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY)
20
+
21
+
22
+ def upload_image(image: bytes, user_id: uuid.UUID):
23
+ """Upload the image to the S3 bucket"""
24
+ if not aws_enabled:
25
+ logger.info("AWS is not enabled. Skipping image upload")
26
+ return None
27
+
28
+ image_key = f"{user_id}/{uuid.uuid4()}.webp"
29
+ try:
30
+ s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read")
31
+ url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}/{image_key}"
32
+ return url
33
+ except Exception as e:
34
+ logger.error(f"Failed to upload image to S3: {e}")
35
+ return None
36
+
37
+
38
+ AWS_USER_UPLOADED_IMAGES_BUCKET_NAME = os.getenv("AWS_USER_UPLOADED_IMAGES_BUCKET_NAME")
39
+
40
+
41
+ def upload_image_to_bucket(image: bytes, user_id: uuid.UUID):
42
+ """Upload the image to the S3 bucket"""
43
+ if not aws_enabled:
44
+ logger.info("AWS is not enabled. Skipping image upload")
45
+ return None
46
+
47
+ image_key = f"{user_id}/{uuid.uuid4()}.webp"
48
+ if not AWS_USER_UPLOADED_IMAGES_BUCKET_NAME:
49
+ logger.error("AWS_USER_UPLOADED_IMAGES_BUCKET_NAME is not set")
50
+ return None
51
+
52
+ try:
53
+ s3_client.put_object(
54
+ Bucket=AWS_USER_UPLOADED_IMAGES_BUCKET_NAME,
55
+ Key=image_key,
56
+ Body=image,
57
+ ACL="public-read",
58
+ ContentType="image/webp",
59
+ )
60
+ return f"https://{AWS_USER_UPLOADED_IMAGES_BUCKET_NAME}/{image_key}"
61
+ except Exception as e:
62
+ logger.error(f"Failed to upload image to S3: {e}")
63
+ return None
khoj/routers/twilio.py ADDED
@@ -0,0 +1,36 @@
1
+ import logging
2
+ import os
3
+
4
+ from khoj.database.models import KhojUser
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+ account_sid = os.getenv("TWILIO_ACCOUNT_SID")
9
+ auth_token = os.getenv("TWILIO_AUTH_TOKEN")
10
+ verification_service_sid = os.getenv("TWILIO_VERIFICATION_SID")
11
+
12
+ twilio_enabled = account_sid is not None and auth_token is not None and verification_service_sid is not None
13
+ if twilio_enabled:
14
+ from twilio.rest import Client
15
+
16
+ client = Client(account_sid, auth_token)
17
+
18
+
19
+ def is_twilio_enabled():
20
+ return twilio_enabled
21
+
22
+
23
+ def create_otp(user: KhojUser):
24
+ """Create a new OTP for the user"""
25
+ verification = client.verify.v2.services(verification_service_sid).verifications.create(
26
+ to=str(user.phone_number), channel="whatsapp"
27
+ )
28
+ return verification.sid is not None
29
+
30
+
31
+ def verify_otp(user: KhojUser, code: str):
32
+ """Verify the OTP for the user"""
33
+ verification_check = client.verify.v2.services(verification_service_sid).verification_checks.create(
34
+ to=str(user.phone_number), code=code
35
+ )
36
+ return verification_check.status == "approved"
@@ -0,0 +1,141 @@
1
+ # System Packages
2
+ import json
3
+ import os
4
+
5
+ from fastapi import APIRouter, Request
6
+ from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
7
+ from fastapi.templating import Jinja2Templates
8
+ from starlette.authentication import requires
9
+
10
+ from khoj.database.adapters import get_user_github_config
11
+ from khoj.routers.helpers import get_next_url, get_user_config
12
+ from khoj.utils import constants
13
+ from khoj.utils.rawconfig import GithubContentConfig, GithubRepoConfig
14
+
15
+ # Initialize Router
16
+ web_client = APIRouter()
17
+ templates = Jinja2Templates([constants.web_directory, constants.next_js_directory, constants.pypi_static_directory])
18
+
19
+
20
+ # Create Routes
21
+ @web_client.get("/", response_class=FileResponse)
22
+ def index(request: Request):
23
+ return templates.TemplateResponse("index.html", context={"request": request})
24
+
25
+
26
+ @web_client.post("/", response_class=FileResponse)
27
+ @requires(["authenticated"], redirect="login_page")
28
+ def index_post(request: Request):
29
+ return templates.TemplateResponse("index.html", context={"request": request})
30
+
31
+
32
+ @web_client.get("/search", response_class=FileResponse)
33
+ @requires(["authenticated"], redirect="login_page")
34
+ def search_page(request: Request):
35
+ return templates.TemplateResponse(
36
+ "search/index.html",
37
+ context={
38
+ "request": request,
39
+ },
40
+ )
41
+
42
+
43
+ @web_client.get("/chat", response_class=FileResponse)
44
+ @requires(["authenticated"], redirect="login_page")
45
+ def chat_page(request: Request):
46
+ return templates.TemplateResponse(
47
+ "chat/index.html",
48
+ context={
49
+ "request": request,
50
+ },
51
+ )
52
+
53
+
54
+ @web_client.get("/login", response_class=FileResponse)
55
+ def login_page(request: Request):
56
+ next_url = get_next_url(request)
57
+ if request.user.is_authenticated:
58
+ return RedirectResponse(url=next_url)
59
+ google_client_id = os.environ.get("GOOGLE_CLIENT_ID")
60
+ redirect_uri = str(request.app.url_path_for("auth_post"))
61
+ return templates.TemplateResponse(
62
+ "login.html",
63
+ context={
64
+ "request": request,
65
+ "google_client_id": google_client_id,
66
+ "redirect_uri": f"{redirect_uri}?next={next_url}",
67
+ },
68
+ )
69
+
70
+
71
+ @web_client.get("/agents", response_class=HTMLResponse)
72
+ def agents_page(request: Request):
73
+ return templates.TemplateResponse(
74
+ "agents/index.html",
75
+ context={
76
+ "request": request,
77
+ },
78
+ )
79
+
80
+
81
+ @web_client.get("/settings", response_class=HTMLResponse)
82
+ @requires(["authenticated"], redirect="login_page")
83
+ def config_page(request: Request):
84
+ return templates.TemplateResponse("settings/index.html", context={"request": request})
85
+
86
+
87
+ @web_client.get("/settings/content/github", response_class=HTMLResponse)
88
+ @requires(["authenticated"], redirect="login_page")
89
+ def github_config_page(request: Request):
90
+ user = request.user.object
91
+ user_config = get_user_config(user, request)
92
+ current_github_config = get_user_github_config(user)
93
+
94
+ if current_github_config:
95
+ raw_repos = current_github_config.githubrepoconfig.all()
96
+ repos = []
97
+ for repo in raw_repos:
98
+ repos.append(
99
+ GithubRepoConfig(
100
+ name=repo.name,
101
+ owner=repo.owner,
102
+ branch=repo.branch,
103
+ )
104
+ )
105
+ current_config = GithubContentConfig(
106
+ pat_token=current_github_config.pat_token,
107
+ repos=repos,
108
+ )
109
+ current_config = json.loads(current_config.json())
110
+ else:
111
+ current_config = {} # type: ignore
112
+
113
+ user_config["current_config"] = current_config
114
+ return templates.TemplateResponse("content_source_github_input.html", context=user_config)
115
+
116
+
117
+ @web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse)
118
+ def view_public_conversation(request: Request):
119
+ return templates.TemplateResponse(
120
+ "share/chat/index.html",
121
+ context={
122
+ "request": request,
123
+ },
124
+ )
125
+
126
+
127
+ @web_client.get("/automations", response_class=HTMLResponse)
128
+ def automations_config_page(
129
+ request: Request,
130
+ ):
131
+ return templates.TemplateResponse(
132
+ "automations/index.html",
133
+ context={
134
+ "request": request,
135
+ },
136
+ )
137
+
138
+
139
+ @web_client.get("/.well-known/assetlinks.json", response_class=FileResponse)
140
+ def assetlinks(request: Request):
141
+ return FileResponse(constants.assetlinks_file_path)
File without changes