khoj 1.33.3.dev32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khoj/__init__.py +0 -0
- khoj/app/README.md +94 -0
- khoj/app/__init__.py +0 -0
- khoj/app/asgi.py +16 -0
- khoj/app/settings.py +218 -0
- khoj/app/urls.py +25 -0
- khoj/configure.py +452 -0
- khoj/database/__init__.py +0 -0
- khoj/database/adapters/__init__.py +1821 -0
- khoj/database/admin.py +417 -0
- khoj/database/apps.py +6 -0
- khoj/database/management/__init__.py +0 -0
- khoj/database/management/commands/__init__.py +0 -0
- khoj/database/management/commands/change_default_model.py +116 -0
- khoj/database/management/commands/change_generated_images_url.py +61 -0
- khoj/database/management/commands/convert_images_png_to_webp.py +99 -0
- khoj/database/migrations/0001_khojuser.py +98 -0
- khoj/database/migrations/0002_googleuser.py +32 -0
- khoj/database/migrations/0003_vector_extension.py +10 -0
- khoj/database/migrations/0004_content_types_and_more.py +181 -0
- khoj/database/migrations/0005_embeddings_corpus_id.py +19 -0
- khoj/database/migrations/0006_embeddingsdates.py +33 -0
- khoj/database/migrations/0007_add_conversation.py +27 -0
- khoj/database/migrations/0008_alter_conversation_conversation_log.py +17 -0
- khoj/database/migrations/0009_khojapiuser.py +24 -0
- khoj/database/migrations/0010_chatmodeloptions_and_more.py +83 -0
- khoj/database/migrations/0010_rename_embeddings_entry_and_more.py +30 -0
- khoj/database/migrations/0011_merge_20231102_0138.py +14 -0
- khoj/database/migrations/0012_entry_file_source.py +21 -0
- khoj/database/migrations/0013_subscription.py +37 -0
- khoj/database/migrations/0014_alter_googleuser_picture.py +17 -0
- khoj/database/migrations/0015_alter_subscription_user.py +21 -0
- khoj/database/migrations/0016_alter_subscription_renewal_date.py +17 -0
- khoj/database/migrations/0017_searchmodel.py +32 -0
- khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py +30 -0
- khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py +27 -0
- khoj/database/migrations/0020_reflectivequestion.py +36 -0
- khoj/database/migrations/0021_speechtotextmodeloptions_and_more.py +42 -0
- khoj/database/migrations/0022_texttoimagemodelconfig.py +25 -0
- khoj/database/migrations/0023_usersearchmodelconfig.py +33 -0
- khoj/database/migrations/0024_alter_entry_embeddings.py +18 -0
- khoj/database/migrations/0025_clientapplication_khojuser_phone_number_and_more.py +46 -0
- khoj/database/migrations/0025_searchmodelconfig_embeddings_inference_endpoint_and_more.py +22 -0
- khoj/database/migrations/0026_searchmodelconfig_cross_encoder_inference_endpoint_and_more.py +22 -0
- khoj/database/migrations/0027_merge_20240118_1324.py +13 -0
- khoj/database/migrations/0028_khojuser_verified_phone_number.py +17 -0
- khoj/database/migrations/0029_userrequests.py +27 -0
- khoj/database/migrations/0030_conversation_slug_and_title.py +38 -0
- khoj/database/migrations/0031_agent_conversation_agent.py +53 -0
- khoj/database/migrations/0031_alter_googleuser_locale.py +30 -0
- khoj/database/migrations/0032_merge_20240322_0427.py +14 -0
- khoj/database/migrations/0033_rename_tuning_agent_personality.py +17 -0
- khoj/database/migrations/0034_alter_chatmodeloptions_chat_model.py +32 -0
- khoj/database/migrations/0035_processlock.py +26 -0
- khoj/database/migrations/0036_alter_processlock_name.py +19 -0
- khoj/database/migrations/0036_delete_offlinechatprocessorconversationconfig.py +15 -0
- khoj/database/migrations/0036_publicconversation.py +42 -0
- khoj/database/migrations/0037_chatmodeloptions_openai_config_and_more.py +51 -0
- khoj/database/migrations/0037_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +32 -0
- khoj/database/migrations/0038_merge_20240425_0857.py +14 -0
- khoj/database/migrations/0038_merge_20240426_1640.py +12 -0
- khoj/database/migrations/0039_merge_20240501_0301.py +12 -0
- khoj/database/migrations/0040_alter_processlock_name.py +26 -0
- khoj/database/migrations/0040_merge_20240504_1010.py +14 -0
- khoj/database/migrations/0041_merge_20240505_1234.py +14 -0
- khoj/database/migrations/0042_serverchatsettings.py +46 -0
- khoj/database/migrations/0043_alter_chatmodeloptions_model_type.py +21 -0
- khoj/database/migrations/0044_conversation_file_filters.py +17 -0
- khoj/database/migrations/0045_fileobject.py +37 -0
- khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py +22 -0
- khoj/database/migrations/0047_alter_entry_file_type.py +31 -0
- khoj/database/migrations/0048_voicemodeloption_uservoicemodelconfig.py +52 -0
- khoj/database/migrations/0049_datastore.py +38 -0
- khoj/database/migrations/0049_texttoimagemodelconfig_api_key_and_more.py +58 -0
- khoj/database/migrations/0050_alter_processlock_name.py +25 -0
- khoj/database/migrations/0051_merge_20240702_1220.py +14 -0
- khoj/database/migrations/0052_alter_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +27 -0
- khoj/database/migrations/0053_agent_style_color_agent_style_icon.py +61 -0
- khoj/database/migrations/0054_alter_agent_style_color.py +38 -0
- khoj/database/migrations/0055_alter_agent_style_icon.py +37 -0
- khoj/database/migrations/0056_chatmodeloptions_vision_enabled.py +17 -0
- khoj/database/migrations/0056_searchmodelconfig_cross_encoder_model_config.py +17 -0
- khoj/database/migrations/0057_merge_20240816_1409.py +13 -0
- khoj/database/migrations/0057_remove_serverchatsettings_default_model_and_more.py +51 -0
- khoj/database/migrations/0058_alter_chatmodeloptions_chat_model.py +17 -0
- khoj/database/migrations/0059_searchmodelconfig_bi_encoder_confidence_threshold.py +17 -0
- khoj/database/migrations/0060_merge_20240905_1828.py +14 -0
- khoj/database/migrations/0061_alter_chatmodeloptions_model_type.py +26 -0
- khoj/database/migrations/0061_alter_texttoimagemodelconfig_model_type.py +21 -0
- khoj/database/migrations/0062_merge_20240913_0222.py +14 -0
- khoj/database/migrations/0063_conversation_temp_id.py +36 -0
- khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +86 -0
- khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
- khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
- khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
- khoj/database/migrations/0068_alter_agent_output_modes.py +24 -0
- khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py +89 -0
- khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +46 -0
- khoj/database/migrations/0071_subscription_enabled_trial_at_and_more.py +32 -0
- khoj/database/migrations/0072_entry_search_model.py +24 -0
- khoj/database/migrations/0073_delete_usersearchmodelconfig.py +15 -0
- khoj/database/migrations/0074_alter_conversation_title.py +17 -0
- khoj/database/migrations/0075_migrate_generated_assets_and_validate.py +85 -0
- khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py +26 -0
- khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py +62 -0
- khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py +17 -0
- khoj/database/migrations/__init__.py +0 -0
- khoj/database/models/__init__.py +725 -0
- khoj/database/tests.py +3 -0
- khoj/interface/compiled/404/index.html +1 -0
- khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_buildManifest.js +1 -0
- khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_ssgManifest.js +1 -0
- khoj/interface/compiled/_next/static/chunks/1010-8f39bb4648b5ba10.js +1 -0
- khoj/interface/compiled/_next/static/chunks/182-f1c48a203dc91e0e.js +20 -0
- khoj/interface/compiled/_next/static/chunks/1915-d3c36ad6ce697ce7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/2117-165ef4747a5b836b.js +2 -0
- khoj/interface/compiled/_next/static/chunks/2581-455000f8aeb08fc3.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3727.dcea8f2193111552.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3789-a09e37a819171a9d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/4124-6c28322ce218d2d5.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5427-b52d95253e692bfa.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5473-b1cf56dedac6577a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5477-0bbddb79c25a54a7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/6065-64db9ad305ba0bcd.js +1 -0
- khoj/interface/compiled/_next/static/chunks/6293-469dd16402ea8a6f.js +3 -0
- khoj/interface/compiled/_next/static/chunks/688-b5b4391bbc0376f1.js +1 -0
- khoj/interface/compiled/_next/static/chunks/8667-b6bf63c72b2d76eb.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9259-1172dbaca0515237.js +1 -0
- khoj/interface/compiled/_next/static/chunks/94ca1967.1d9b42d929a1ee8c.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9597.83583248dfbf6e73.js +1 -0
- khoj/interface/compiled/_next/static/chunks/964ecbae.51d6faf8801d15e6.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9665-391df1e5c51c960a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/_not-found/page-a834eddae3e235df.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/layout-e00fb81dca656a10.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/page-28ce086a1129bca2.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/layout-1fe1537449f43496.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/page-bf365a60829d347f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/chat/layout-33934fc2d6ae6838.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/chat/page-0e476e57eb2015e3.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/layout-30e7fda7262713ce.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/page-a5515ea71aec5ef0.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/layout-c02531d586972d7d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/page-9140541e67ea307d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/settings/layout-d09d6510a45cd4bd.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/settings/page-951ba40b5b94b23a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/layout-e8e5db7830bf3f47.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-1beb80d8d741c932.js +1 -0
- khoj/interface/compiled/_next/static/chunks/d3ac728e-44ebd2a0c99b12a0.js +1 -0
- khoj/interface/compiled/_next/static/chunks/fd9d1056-4482b99a36fd1673.js +1 -0
- khoj/interface/compiled/_next/static/chunks/framework-8e0e0f4a6b83a956.js +1 -0
- khoj/interface/compiled/_next/static/chunks/main-app-de1f09df97a3cfc7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/main-db4bfac6b0a8d00b.js +1 -0
- khoj/interface/compiled/_next/static/chunks/pages/_app-3c9ca398d360b709.js +1 -0
- khoj/interface/compiled/_next/static/chunks/pages/_error-cf5ca766ac8f493f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/webpack-a03962458328b163.js +1 -0
- khoj/interface/compiled/_next/static/css/089de1d8526b96e9.css +1 -0
- khoj/interface/compiled/_next/static/css/37a73b87f02df402.css +1 -0
- khoj/interface/compiled/_next/static/css/4e4e6a4a1c920d06.css +1 -0
- khoj/interface/compiled/_next/static/css/8d02837c730f8d13.css +25 -0
- khoj/interface/compiled/_next/static/css/8e6a3ca11a60b189.css +1 -0
- khoj/interface/compiled/_next/static/css/9c164d9727dd8092.css +1 -0
- khoj/interface/compiled/_next/static/css/dac88c17aaee5fcf.css +1 -0
- khoj/interface/compiled/_next/static/css/df4b47a2d0d85eae.css +1 -0
- khoj/interface/compiled/_next/static/css/e4eb883b5265d372.css +1 -0
- khoj/interface/compiled/_next/static/media/1d8a05b60287ae6c-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/6f22fce21a7c433c-s.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/77c207b095007c34-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/82ef96de0e8f4d8c-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.1608a09b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.4aafdb68.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.a79f1c31.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.b6770918.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.cce5b8ec.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.ec17d132.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.07ef19e7.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.55fac258.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.dad44a7f.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.9f256b85.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.b18f59e1.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.d42a5579.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.7c187121.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.d3c882a6.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.ed38e79f.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.b74a1a8b.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.c3fb5ac2.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.d181c465.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.6f2bb1df.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.70d8b0a5.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.e3f82f9d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.47373d1e.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.8916142b.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.9024d815.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.0462f03b.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.7f51fe03.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.b7f8fe9b.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.572d331f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.a879cf83.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.f1035d8d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.5295ba48.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.939bc644.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.f28c23ac.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.8c5b5494.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.94e1e8dc.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.bf59d231.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.3b1e59b3.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.7c9bc82b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.b4c20c84.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.74048478.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.ba21ed5f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.d4d7ba48.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.03e9641d.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.07505710.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.fe9cbbe1.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.e1e279cb.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.eae34984.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.fabc004a.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.57727022.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.5916a24f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.d6b476ec.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.9acaf01c.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.a144ef58.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.b4230e7e.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.10d95fd3.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.7a996c9d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.fbccdabe.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.6258592b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.a8709e36.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.d97aaf4a.ttf +0 -0
- khoj/interface/compiled/_next/static/media/a6ecd16fa044d500-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/bd82c78e5b7b3fe9-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/c32c8052c071fc42-s.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/c4250770ab8708b6-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/e098aaaecc9cfbb2-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/flags.3afdda2f.webp +0 -0
- khoj/interface/compiled/_next/static/media/flags@2x.5fbe9fc1.webp +0 -0
- khoj/interface/compiled/_next/static/media/globe.98e105ca.webp +0 -0
- khoj/interface/compiled/_next/static/media/globe@2x.974df6f8.webp +0 -0
- khoj/interface/compiled/agents/index.html +1 -0
- khoj/interface/compiled/agents/index.txt +7 -0
- khoj/interface/compiled/agents.svg +6 -0
- khoj/interface/compiled/assets/icons/khoj_lantern.ico +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern.svg +100 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_1200x1200.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_128x128.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_128x128_dark.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_256x256.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_512x512.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_logomarktype_1200x630.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-browse-draw-sample.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-plain-chat-sample.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-remember-plan-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-browse-draw-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-plain-chat-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-remember-plan-sample.png +0 -0
- khoj/interface/compiled/automation.svg +37 -0
- khoj/interface/compiled/automations/index.html +1 -0
- khoj/interface/compiled/automations/index.txt +8 -0
- khoj/interface/compiled/chat/index.html +1 -0
- khoj/interface/compiled/chat/index.txt +7 -0
- khoj/interface/compiled/chat.svg +24 -0
- khoj/interface/compiled/close.svg +5 -0
- khoj/interface/compiled/copy-button-success.svg +6 -0
- khoj/interface/compiled/copy-button.svg +5 -0
- khoj/interface/compiled/index.html +1 -0
- khoj/interface/compiled/index.txt +7 -0
- khoj/interface/compiled/khoj.webmanifest +76 -0
- khoj/interface/compiled/logo.svg +24 -0
- khoj/interface/compiled/search/index.html +1 -0
- khoj/interface/compiled/search/index.txt +7 -0
- khoj/interface/compiled/send.svg +1 -0
- khoj/interface/compiled/settings/index.html +1 -0
- khoj/interface/compiled/settings/index.txt +9 -0
- khoj/interface/compiled/share/chat/index.html +1 -0
- khoj/interface/compiled/share/chat/index.txt +7 -0
- khoj/interface/compiled/share.svg +8 -0
- khoj/interface/compiled/thumbs-down.svg +6 -0
- khoj/interface/compiled/thumbs-up.svg +6 -0
- khoj/interface/email/feedback.html +34 -0
- khoj/interface/email/magic_link.html +40 -0
- khoj/interface/email/task.html +37 -0
- khoj/interface/email/welcome.html +90 -0
- khoj/interface/web/.well-known/assetlinks.json +11 -0
- khoj/interface/web/assets/icons/agents.svg +19 -0
- khoj/interface/web/assets/icons/automation.svg +43 -0
- khoj/interface/web/assets/icons/chat.svg +24 -0
- khoj/interface/web/assets/icons/github.svg +1 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways-200.png +0 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways-500.png +0 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways.svg +32 -0
- khoj/interface/web/assets/icons/khoj.svg +26 -0
- khoj/interface/web/assets/icons/logotype.svg +1 -0
- khoj/interface/web/assets/icons/search.svg +57 -0
- khoj/interface/web/assets/icons/sync.svg +4 -0
- khoj/interface/web/assets/khoj.css +237 -0
- khoj/interface/web/assets/utils.js +33 -0
- khoj/interface/web/base_config.html +445 -0
- khoj/interface/web/content_source_github_input.html +208 -0
- khoj/interface/web/login.html +310 -0
- khoj/interface/web/utils.html +48 -0
- khoj/main.py +249 -0
- khoj/manage.py +22 -0
- khoj/migrations/__init__.py +0 -0
- khoj/migrations/migrate_offline_chat_default_model.py +69 -0
- khoj/migrations/migrate_offline_chat_default_model_2.py +71 -0
- khoj/migrations/migrate_offline_chat_schema.py +83 -0
- khoj/migrations/migrate_offline_model.py +29 -0
- khoj/migrations/migrate_processor_config_openai.py +67 -0
- khoj/migrations/migrate_server_pg.py +132 -0
- khoj/migrations/migrate_version.py +17 -0
- khoj/processor/__init__.py +0 -0
- khoj/processor/content/__init__.py +0 -0
- khoj/processor/content/docx/__init__.py +0 -0
- khoj/processor/content/docx/docx_to_entries.py +111 -0
- khoj/processor/content/github/__init__.py +0 -0
- khoj/processor/content/github/github_to_entries.py +226 -0
- khoj/processor/content/images/__init__.py +0 -0
- khoj/processor/content/images/image_to_entries.py +117 -0
- khoj/processor/content/markdown/__init__.py +0 -0
- khoj/processor/content/markdown/markdown_to_entries.py +160 -0
- khoj/processor/content/notion/notion_to_entries.py +259 -0
- khoj/processor/content/org_mode/__init__.py +0 -0
- khoj/processor/content/org_mode/org_to_entries.py +226 -0
- khoj/processor/content/org_mode/orgnode.py +532 -0
- khoj/processor/content/pdf/__init__.py +0 -0
- khoj/processor/content/pdf/pdf_to_entries.py +119 -0
- khoj/processor/content/plaintext/__init__.py +0 -0
- khoj/processor/content/plaintext/plaintext_to_entries.py +117 -0
- khoj/processor/content/text_to_entries.py +296 -0
- khoj/processor/conversation/__init__.py +0 -0
- khoj/processor/conversation/anthropic/__init__.py +0 -0
- khoj/processor/conversation/anthropic/anthropic_chat.py +243 -0
- khoj/processor/conversation/anthropic/utils.py +217 -0
- khoj/processor/conversation/google/__init__.py +0 -0
- khoj/processor/conversation/google/gemini_chat.py +253 -0
- khoj/processor/conversation/google/utils.py +260 -0
- khoj/processor/conversation/offline/__init__.py +0 -0
- khoj/processor/conversation/offline/chat_model.py +308 -0
- khoj/processor/conversation/offline/utils.py +80 -0
- khoj/processor/conversation/offline/whisper.py +15 -0
- khoj/processor/conversation/openai/__init__.py +0 -0
- khoj/processor/conversation/openai/gpt.py +243 -0
- khoj/processor/conversation/openai/utils.py +232 -0
- khoj/processor/conversation/openai/whisper.py +13 -0
- khoj/processor/conversation/prompts.py +1188 -0
- khoj/processor/conversation/utils.py +867 -0
- khoj/processor/embeddings.py +122 -0
- khoj/processor/image/generate.py +215 -0
- khoj/processor/speech/__init__.py +0 -0
- khoj/processor/speech/text_to_speech.py +51 -0
- khoj/processor/tools/__init__.py +0 -0
- khoj/processor/tools/online_search.py +472 -0
- khoj/processor/tools/run_code.py +179 -0
- khoj/routers/__init__.py +0 -0
- khoj/routers/api.py +760 -0
- khoj/routers/api_agents.py +295 -0
- khoj/routers/api_chat.py +1273 -0
- khoj/routers/api_content.py +634 -0
- khoj/routers/api_model.py +123 -0
- khoj/routers/api_phone.py +86 -0
- khoj/routers/api_subscription.py +144 -0
- khoj/routers/auth.py +307 -0
- khoj/routers/email.py +135 -0
- khoj/routers/helpers.py +2333 -0
- khoj/routers/notion.py +85 -0
- khoj/routers/research.py +364 -0
- khoj/routers/storage.py +63 -0
- khoj/routers/twilio.py +36 -0
- khoj/routers/web_client.py +141 -0
- khoj/search_filter/__init__.py +0 -0
- khoj/search_filter/base_filter.py +15 -0
- khoj/search_filter/date_filter.py +215 -0
- khoj/search_filter/file_filter.py +32 -0
- khoj/search_filter/word_filter.py +29 -0
- khoj/search_type/__init__.py +0 -0
- khoj/search_type/text_search.py +255 -0
- khoj/utils/__init__.py +0 -0
- khoj/utils/cli.py +101 -0
- khoj/utils/config.py +81 -0
- khoj/utils/constants.py +51 -0
- khoj/utils/fs_syncer.py +252 -0
- khoj/utils/helpers.py +627 -0
- khoj/utils/initialization.py +301 -0
- khoj/utils/jsonl.py +43 -0
- khoj/utils/models.py +47 -0
- khoj/utils/rawconfig.py +208 -0
- khoj/utils/state.py +48 -0
- khoj/utils/yaml.py +47 -0
- khoj-1.33.3.dev32.dist-info/METADATA +190 -0
- khoj-1.33.3.dev32.dist-info/RECORD +393 -0
- khoj-1.33.3.dev32.dist-info/WHEEL +4 -0
- khoj-1.33.3.dev32.dist-info/entry_points.txt +2 -0
- khoj-1.33.3.dev32.dist-info/licenses/LICENSE +661 -0
khoj/configure.py
ADDED
@@ -0,0 +1,452 @@
|
|
1
|
+
import json
|
2
|
+
import logging
|
3
|
+
import os
|
4
|
+
from datetime import datetime
|
5
|
+
from enum import Enum
|
6
|
+
from functools import wraps
|
7
|
+
from typing import Optional
|
8
|
+
|
9
|
+
import openai
|
10
|
+
import requests
|
11
|
+
import schedule
|
12
|
+
from asgiref.sync import sync_to_async
|
13
|
+
from django.conf import settings
|
14
|
+
from django.db import close_old_connections, connections
|
15
|
+
from django.utils.timezone import make_aware
|
16
|
+
from fastapi import Response
|
17
|
+
from starlette.authentication import (
|
18
|
+
AuthCredentials,
|
19
|
+
AuthenticationBackend,
|
20
|
+
SimpleUser,
|
21
|
+
UnauthenticatedUser,
|
22
|
+
)
|
23
|
+
from starlette.concurrency import run_in_threadpool
|
24
|
+
from starlette.middleware import Middleware
|
25
|
+
from starlette.middleware.authentication import AuthenticationMiddleware
|
26
|
+
from starlette.middleware.base import BaseHTTPMiddleware
|
27
|
+
from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware
|
28
|
+
from starlette.middleware.sessions import SessionMiddleware
|
29
|
+
from starlette.requests import HTTPConnection
|
30
|
+
from starlette.types import ASGIApp, Receive, Scope, Send
|
31
|
+
|
32
|
+
from khoj.database.adapters import (
|
33
|
+
AgentAdapters,
|
34
|
+
ClientApplicationAdapters,
|
35
|
+
ConversationAdapters,
|
36
|
+
ProcessLockAdapters,
|
37
|
+
aget_or_create_user_by_phone_number,
|
38
|
+
aget_user_by_phone_number,
|
39
|
+
ais_user_subscribed,
|
40
|
+
delete_user_requests,
|
41
|
+
get_all_users,
|
42
|
+
get_or_create_search_models,
|
43
|
+
)
|
44
|
+
from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
|
45
|
+
from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
|
46
|
+
from khoj.routers.api_content import configure_content, configure_search
|
47
|
+
from khoj.routers.twilio import is_twilio_enabled
|
48
|
+
from khoj.utils import constants, state
|
49
|
+
from khoj.utils.config import SearchType
|
50
|
+
from khoj.utils.fs_syncer import collect_files
|
51
|
+
from khoj.utils.helpers import is_none_or_empty, telemetry_disabled
|
52
|
+
from khoj.utils.rawconfig import FullConfig
|
53
|
+
|
54
|
+
logger = logging.getLogger(__name__)
|
55
|
+
|
56
|
+
|
57
|
+
class AuthenticatedKhojUser(SimpleUser):
|
58
|
+
def __init__(self, user, client_app: Optional[ClientApplication] = None):
|
59
|
+
self.object = user
|
60
|
+
self.client_app = client_app
|
61
|
+
super().__init__(user.username)
|
62
|
+
|
63
|
+
|
64
|
+
class AsyncCloseConnectionsMiddleware(BaseHTTPMiddleware):
|
65
|
+
"""
|
66
|
+
Using this middleware to call close_old_connections() twice is a pretty yucky hack,
|
67
|
+
as it appears that run_in_threadpool (used by Starlette/FastAPI) and sync_to_async
|
68
|
+
(used by Django) have divergent behavior, ultimately acquiring the incorrect thread
|
69
|
+
in mixed sync/async which has the effect of duplicating connections.
|
70
|
+
We could fix the duplicate connections too if we normalized the thread behavior,
|
71
|
+
but at minimum we need to clean up connections in each case to prevent persistent
|
72
|
+
"InterfaceError: connection already closed" errors when the database connection is
|
73
|
+
reset via a database restart or something -- so here we are!
|
74
|
+
If we always use smart_sync_to_async(), this double calling isn't necessary, but
|
75
|
+
depending on what levels of abstraction we introduce, we might silently break the
|
76
|
+
assumptions. Better to be safe than sorry!
|
77
|
+
Attribution: https://gist.github.com/bryanhelmig/6fb091f23c1a4b7462dddce51cfaa1ca
|
78
|
+
"""
|
79
|
+
|
80
|
+
async def dispatch(self, request, call_next):
|
81
|
+
await run_in_threadpool(close_old_connections)
|
82
|
+
await sync_to_async(close_old_connections)()
|
83
|
+
try:
|
84
|
+
response = await call_next(request)
|
85
|
+
finally:
|
86
|
+
# in tests, use @override_settings(CLOSE_CONNECTIONS_AFTER_REQUEST=True)
|
87
|
+
if getattr(settings, "CLOSE_CONNECTIONS_AFTER_REQUEST", False):
|
88
|
+
await run_in_threadpool(connections.close_all)
|
89
|
+
await sync_to_async(connections.close_all)()
|
90
|
+
return response
|
91
|
+
|
92
|
+
|
93
|
+
class UserAuthenticationBackend(AuthenticationBackend):
|
94
|
+
def __init__(
|
95
|
+
self,
|
96
|
+
):
|
97
|
+
from khoj.database.models import KhojApiUser, KhojUser
|
98
|
+
|
99
|
+
self.khojuser_manager = KhojUser.objects
|
100
|
+
self.khojapiuser_manager = KhojApiUser.objects
|
101
|
+
self._initialize_default_user()
|
102
|
+
super().__init__()
|
103
|
+
|
104
|
+
def _initialize_default_user(self):
|
105
|
+
if not self.khojuser_manager.filter(username="default").exists():
|
106
|
+
default_user = self.khojuser_manager.create_user(
|
107
|
+
username="default",
|
108
|
+
email="default@example.com",
|
109
|
+
password="default",
|
110
|
+
)
|
111
|
+
renewal_date = make_aware(datetime.strptime("2100-04-01", "%Y-%m-%d"))
|
112
|
+
Subscription.objects.create(user=default_user, type=Subscription.Type.STANDARD, renewal_date=renewal_date)
|
113
|
+
|
114
|
+
async def authenticate(self, request: HTTPConnection):
|
115
|
+
current_user = request.session.get("user")
|
116
|
+
if current_user and current_user.get("email"):
|
117
|
+
user = (
|
118
|
+
await self.khojuser_manager.filter(email=current_user.get("email"))
|
119
|
+
.prefetch_related("subscription")
|
120
|
+
.afirst()
|
121
|
+
)
|
122
|
+
if user:
|
123
|
+
subscribed = await ais_user_subscribed(user)
|
124
|
+
if subscribed:
|
125
|
+
return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user)
|
126
|
+
return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user)
|
127
|
+
|
128
|
+
# Request from Desktop, Emacs, Obsidian clients
|
129
|
+
if len(request.headers.get("Authorization", "").split("Bearer ")) == 2:
|
130
|
+
# Get bearer token from header
|
131
|
+
bearer_token = request.headers["Authorization"].split("Bearer ")[1]
|
132
|
+
# Get user owning token
|
133
|
+
user_with_token = (
|
134
|
+
await self.khojapiuser_manager.filter(token=bearer_token)
|
135
|
+
.select_related("user")
|
136
|
+
.prefetch_related("user__subscription")
|
137
|
+
.afirst()
|
138
|
+
)
|
139
|
+
if user_with_token:
|
140
|
+
subscribed = await ais_user_subscribed(user_with_token.user)
|
141
|
+
if subscribed:
|
142
|
+
return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user_with_token.user)
|
143
|
+
return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user_with_token.user)
|
144
|
+
|
145
|
+
# Request from Whatsapp client
|
146
|
+
client_id = request.query_params.get("client_id")
|
147
|
+
if client_id:
|
148
|
+
# Get the client secret, which is passed in the Authorization header
|
149
|
+
client_secret = request.headers["Authorization"].split("Bearer ")[1]
|
150
|
+
if not client_secret:
|
151
|
+
return Response(
|
152
|
+
status_code=401,
|
153
|
+
content="Please provide a client secret in the Authorization header with a client_id query param.",
|
154
|
+
)
|
155
|
+
|
156
|
+
# Get the client application
|
157
|
+
client_application = await ClientApplicationAdapters.aget_client_application_by_id(client_id, client_secret)
|
158
|
+
if client_application is None:
|
159
|
+
return AuthCredentials(), UnauthenticatedUser()
|
160
|
+
# Get the identifier used for the user
|
161
|
+
phone_number = request.query_params.get("phone_number")
|
162
|
+
if is_none_or_empty(phone_number):
|
163
|
+
return AuthCredentials(), UnauthenticatedUser()
|
164
|
+
|
165
|
+
if not phone_number.startswith("+"):
|
166
|
+
phone_number = f"+{phone_number}"
|
167
|
+
|
168
|
+
create_if_not_exists = request.query_params.get("create_if_not_exists")
|
169
|
+
if create_if_not_exists:
|
170
|
+
user, is_new = await aget_or_create_user_by_phone_number(phone_number)
|
171
|
+
if user and is_new:
|
172
|
+
logger.log(logging.INFO, f"🥳 New User Created: {user.uuid}")
|
173
|
+
else:
|
174
|
+
user = await aget_user_by_phone_number(phone_number)
|
175
|
+
|
176
|
+
if user is None:
|
177
|
+
return AuthCredentials(), UnauthenticatedUser()
|
178
|
+
|
179
|
+
subscribed = await ais_user_subscribed(user)
|
180
|
+
|
181
|
+
if subscribed:
|
182
|
+
return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user, client_application)
|
183
|
+
return AuthCredentials(["authenticated"]), AuthenticatedKhojUser(user, client_application)
|
184
|
+
|
185
|
+
# No auth required if server in anonymous mode
|
186
|
+
if state.anonymous_mode:
|
187
|
+
user = await self.khojuser_manager.filter(username="default").prefetch_related("subscription").afirst()
|
188
|
+
if user:
|
189
|
+
return AuthCredentials(["authenticated", "premium"]), AuthenticatedKhojUser(user)
|
190
|
+
|
191
|
+
return AuthCredentials(), UnauthenticatedUser()
|
192
|
+
|
193
|
+
|
194
|
+
def initialize_server(config: Optional[FullConfig]):
|
195
|
+
try:
|
196
|
+
configure_server(config, init=True)
|
197
|
+
except Exception as e:
|
198
|
+
logger.error(f"🚨 Failed to configure server on app load: {e}", exc_info=True)
|
199
|
+
raise e
|
200
|
+
|
201
|
+
|
202
|
+
def clean_connections(func):
|
203
|
+
"""
|
204
|
+
A decorator that ensures that Django database connections that have become unusable, or are obsolete, are closed
|
205
|
+
before and after a method is executed (see: https://docs.djangoproject.com/en/dev/ref/databases/#general-notes
|
206
|
+
for background).
|
207
|
+
"""
|
208
|
+
|
209
|
+
@wraps(func)
|
210
|
+
def func_wrapper(*args, **kwargs):
|
211
|
+
close_old_connections()
|
212
|
+
try:
|
213
|
+
result = func(*args, **kwargs)
|
214
|
+
finally:
|
215
|
+
close_old_connections()
|
216
|
+
|
217
|
+
return result
|
218
|
+
|
219
|
+
return func_wrapper
|
220
|
+
|
221
|
+
|
222
|
+
def configure_server(
|
223
|
+
config: FullConfig,
|
224
|
+
regenerate: bool = False,
|
225
|
+
search_type: Optional[SearchType] = None,
|
226
|
+
init=False,
|
227
|
+
user: KhojUser = None,
|
228
|
+
):
|
229
|
+
# Update Config
|
230
|
+
if config == None:
|
231
|
+
logger.info(f"Initializing with default config.")
|
232
|
+
config = FullConfig()
|
233
|
+
state.config = config
|
234
|
+
|
235
|
+
if ConversationAdapters.has_valid_ai_model_api():
|
236
|
+
ai_model_api = ConversationAdapters.get_ai_model_api()
|
237
|
+
state.openai_client = openai.OpenAI(api_key=ai_model_api.api_key)
|
238
|
+
|
239
|
+
# Initialize Search Models from Config and initialize content
|
240
|
+
try:
|
241
|
+
search_models = get_or_create_search_models()
|
242
|
+
state.embeddings_model = dict()
|
243
|
+
state.cross_encoder_model = dict()
|
244
|
+
|
245
|
+
for model in search_models:
|
246
|
+
state.embeddings_model.update(
|
247
|
+
{
|
248
|
+
model.name: EmbeddingsModel(
|
249
|
+
model.bi_encoder,
|
250
|
+
model.embeddings_inference_endpoint,
|
251
|
+
model.embeddings_inference_endpoint_api_key,
|
252
|
+
query_encode_kwargs=model.bi_encoder_query_encode_config,
|
253
|
+
docs_encode_kwargs=model.bi_encoder_docs_encode_config,
|
254
|
+
model_kwargs=model.bi_encoder_model_config,
|
255
|
+
)
|
256
|
+
}
|
257
|
+
)
|
258
|
+
state.cross_encoder_model.update(
|
259
|
+
{
|
260
|
+
model.name: CrossEncoderModel(
|
261
|
+
model.cross_encoder,
|
262
|
+
model.cross_encoder_inference_endpoint,
|
263
|
+
model.cross_encoder_inference_endpoint_api_key,
|
264
|
+
model_kwargs=model.cross_encoder_model_config,
|
265
|
+
)
|
266
|
+
}
|
267
|
+
)
|
268
|
+
|
269
|
+
state.SearchType = configure_search_types()
|
270
|
+
state.search_models = configure_search(state.search_models, state.config.search_type)
|
271
|
+
setup_default_agent(user)
|
272
|
+
|
273
|
+
message = (
|
274
|
+
"📡 Telemetry disabled"
|
275
|
+
if telemetry_disabled(state.config.app, state.telemetry_disabled)
|
276
|
+
else "📡 Telemetry enabled"
|
277
|
+
)
|
278
|
+
logger.info(message)
|
279
|
+
|
280
|
+
if not init:
|
281
|
+
initialize_content(user, regenerate, search_type)
|
282
|
+
|
283
|
+
except Exception as e:
|
284
|
+
logger.error(f"Failed to load some search models: {e}", exc_info=True)
|
285
|
+
|
286
|
+
|
287
|
+
def setup_default_agent(user: KhojUser):
|
288
|
+
AgentAdapters.create_default_agent(user)
|
289
|
+
|
290
|
+
|
291
|
+
def initialize_content(user: KhojUser, regenerate: bool, search_type: Optional[SearchType] = None):
|
292
|
+
# Initialize Content from Config
|
293
|
+
if state.search_models:
|
294
|
+
try:
|
295
|
+
logger.info("📬 Updating content index...")
|
296
|
+
all_files = collect_files(user=user)
|
297
|
+
status = configure_content(
|
298
|
+
user,
|
299
|
+
all_files,
|
300
|
+
regenerate,
|
301
|
+
search_type,
|
302
|
+
)
|
303
|
+
if not status:
|
304
|
+
raise RuntimeError("Failed to update content index")
|
305
|
+
except Exception as e:
|
306
|
+
raise e
|
307
|
+
|
308
|
+
|
309
|
+
def configure_routes(app):
|
310
|
+
# Import APIs here to setup search types before while configuring server
|
311
|
+
from khoj.routers.api import api
|
312
|
+
from khoj.routers.api_agents import api_agents
|
313
|
+
from khoj.routers.api_chat import api_chat
|
314
|
+
from khoj.routers.api_content import api_content
|
315
|
+
from khoj.routers.api_model import api_model
|
316
|
+
from khoj.routers.notion import notion_router
|
317
|
+
from khoj.routers.web_client import web_client
|
318
|
+
|
319
|
+
app.include_router(api, prefix="/api")
|
320
|
+
app.include_router(api_chat, prefix="/api/chat")
|
321
|
+
app.include_router(api_agents, prefix="/api/agents")
|
322
|
+
app.include_router(api_model, prefix="/api/model")
|
323
|
+
app.include_router(api_content, prefix="/api/content")
|
324
|
+
app.include_router(notion_router, prefix="/api/notion")
|
325
|
+
app.include_router(web_client)
|
326
|
+
|
327
|
+
if not state.anonymous_mode:
|
328
|
+
from khoj.routers.auth import auth_router
|
329
|
+
|
330
|
+
app.include_router(auth_router, prefix="/auth")
|
331
|
+
logger.info("🔑 Enabled Authentication")
|
332
|
+
|
333
|
+
if state.billing_enabled:
|
334
|
+
from khoj.routers.api_subscription import subscription_router
|
335
|
+
|
336
|
+
app.include_router(subscription_router, prefix="/api/subscription")
|
337
|
+
logger.info("💳 Enabled Billing")
|
338
|
+
|
339
|
+
if is_twilio_enabled():
|
340
|
+
from khoj.routers.api_phone import api_phone
|
341
|
+
|
342
|
+
app.include_router(api_phone, prefix="/api/phone")
|
343
|
+
logger.info("📞 Enabled Twilio")
|
344
|
+
|
345
|
+
|
346
|
+
def configure_middleware(app, ssl_enabled: bool = False):
|
347
|
+
class NextJsMiddleware(Middleware):
|
348
|
+
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
349
|
+
if scope["type"] == "http" and scope["path"].startswith("/_next"):
|
350
|
+
scope["path"] = "/static" + scope["path"]
|
351
|
+
await self.app(scope, receive, send)
|
352
|
+
|
353
|
+
def __init__(self, app: ASGIApp) -> None:
|
354
|
+
super().__init__(app)
|
355
|
+
self.app = app
|
356
|
+
|
357
|
+
if ssl_enabled:
|
358
|
+
app.add_middleware(HTTPSRedirectMiddleware)
|
359
|
+
app.add_middleware(AsyncCloseConnectionsMiddleware)
|
360
|
+
app.add_middleware(AuthenticationMiddleware, backend=UserAuthenticationBackend())
|
361
|
+
app.add_middleware(NextJsMiddleware)
|
362
|
+
app.add_middleware(SessionMiddleware, secret_key=os.environ.get("KHOJ_DJANGO_SECRET_KEY", "!secret"))
|
363
|
+
|
364
|
+
|
365
|
+
def update_content_index():
|
366
|
+
for user in get_all_users():
|
367
|
+
all_files = collect_files(user=user)
|
368
|
+
success = configure_content(user, all_files)
|
369
|
+
if not success:
|
370
|
+
raise RuntimeError("Failed to update content index")
|
371
|
+
logger.info("📪 Content index updated via Scheduler")
|
372
|
+
|
373
|
+
|
374
|
+
@schedule.repeat(schedule.every(22).to(25).hours)
|
375
|
+
@clean_connections
|
376
|
+
def update_content_index_regularly():
|
377
|
+
ProcessLockAdapters.run_with_lock(
|
378
|
+
update_content_index, ProcessLock.Operation.INDEX_CONTENT, max_duration_in_seconds=60 * 60 * 2
|
379
|
+
)
|
380
|
+
|
381
|
+
|
382
|
+
def configure_search_types():
|
383
|
+
# Extract core search types
|
384
|
+
core_search_types = {e.name: e.value for e in SearchType}
|
385
|
+
|
386
|
+
# Dynamically generate search type enum by merging core search types with configured plugin search types
|
387
|
+
return Enum("SearchType", core_search_types)
|
388
|
+
|
389
|
+
|
390
|
+
@schedule.repeat(schedule.every(2).minutes)
|
391
|
+
@clean_connections
|
392
|
+
def upload_telemetry():
|
393
|
+
if telemetry_disabled(state.config.app, state.telemetry_disabled) or not state.telemetry:
|
394
|
+
return
|
395
|
+
|
396
|
+
try:
|
397
|
+
logger.info(f"📡 Uploading telemetry to {constants.telemetry_server}...")
|
398
|
+
logger.debug(f"Telemetry state:\n{state.telemetry}")
|
399
|
+
for log in state.telemetry:
|
400
|
+
for field in log:
|
401
|
+
# Check if the value for the field is JSON serializable
|
402
|
+
if log[field] is None:
|
403
|
+
log[field] = ""
|
404
|
+
try:
|
405
|
+
json.dumps(log[field])
|
406
|
+
except TypeError:
|
407
|
+
log[field] = str(log[field])
|
408
|
+
response = requests.post(constants.telemetry_server, json=state.telemetry)
|
409
|
+
response.raise_for_status()
|
410
|
+
except Exception as e:
|
411
|
+
logger.error(f"📡 Error uploading telemetry: {e}", exc_info=True)
|
412
|
+
else:
|
413
|
+
state.telemetry = []
|
414
|
+
|
415
|
+
|
416
|
+
@schedule.repeat(schedule.every(31).minutes)
|
417
|
+
@clean_connections
|
418
|
+
def delete_old_user_requests():
|
419
|
+
num_deleted = delete_user_requests()
|
420
|
+
logger.debug(f"🗑️ Deleted {num_deleted[0]} day-old user requests")
|
421
|
+
|
422
|
+
|
423
|
+
@schedule.repeat(schedule.every(17).minutes)
|
424
|
+
@clean_connections
|
425
|
+
def wakeup_scheduler():
|
426
|
+
# Wake up the scheduler to ensure it runs the scheduled tasks. This is because the elected leader may not always be aware of tasks scheduled on other workers.
|
427
|
+
TWELVE_HOURS = 43200
|
428
|
+
|
429
|
+
# If the worker currently possesses a process lock, check if it is valid.
|
430
|
+
|
431
|
+
if state.schedule_leader_process_lock:
|
432
|
+
if not ProcessLockAdapters.is_process_locked(state.schedule_leader_process_lock):
|
433
|
+
state.schedule_leader_process_lock = None
|
434
|
+
state.scheduler.pause()
|
435
|
+
|
436
|
+
# Get the current process lock
|
437
|
+
schedule_leader_process_lock = ProcessLockAdapters.get_process_lock(ProcessLock.Operation.SCHEDULE_LEADER)
|
438
|
+
|
439
|
+
# Check if the process lock is still active. If not, create a new process lock. This worker will become the scheduler leader.
|
440
|
+
if not schedule_leader_process_lock or not ProcessLockAdapters.is_process_locked(schedule_leader_process_lock):
|
441
|
+
schedule_leader_process_lock = ProcessLockAdapters.set_process_lock(
|
442
|
+
ProcessLock.Operation.SCHEDULE_LEADER, max_duration_in_seconds=TWELVE_HOURS
|
443
|
+
)
|
444
|
+
state.schedule_leader_process_lock = schedule_leader_process_lock
|
445
|
+
state.scheduler.resume()
|
446
|
+
logger.info("🔔 Scheduler leader process lock acquired")
|
447
|
+
|
448
|
+
if state.schedule_leader_process_lock:
|
449
|
+
state.scheduler.wakeup()
|
450
|
+
else:
|
451
|
+
# Make sure the other workers don't run the scheduled tasks
|
452
|
+
state.scheduler.pause()
|
File without changes
|