khoj 1.33.3.dev32__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khoj/__init__.py +0 -0
- khoj/app/README.md +94 -0
- khoj/app/__init__.py +0 -0
- khoj/app/asgi.py +16 -0
- khoj/app/settings.py +218 -0
- khoj/app/urls.py +25 -0
- khoj/configure.py +452 -0
- khoj/database/__init__.py +0 -0
- khoj/database/adapters/__init__.py +1821 -0
- khoj/database/admin.py +417 -0
- khoj/database/apps.py +6 -0
- khoj/database/management/__init__.py +0 -0
- khoj/database/management/commands/__init__.py +0 -0
- khoj/database/management/commands/change_default_model.py +116 -0
- khoj/database/management/commands/change_generated_images_url.py +61 -0
- khoj/database/management/commands/convert_images_png_to_webp.py +99 -0
- khoj/database/migrations/0001_khojuser.py +98 -0
- khoj/database/migrations/0002_googleuser.py +32 -0
- khoj/database/migrations/0003_vector_extension.py +10 -0
- khoj/database/migrations/0004_content_types_and_more.py +181 -0
- khoj/database/migrations/0005_embeddings_corpus_id.py +19 -0
- khoj/database/migrations/0006_embeddingsdates.py +33 -0
- khoj/database/migrations/0007_add_conversation.py +27 -0
- khoj/database/migrations/0008_alter_conversation_conversation_log.py +17 -0
- khoj/database/migrations/0009_khojapiuser.py +24 -0
- khoj/database/migrations/0010_chatmodeloptions_and_more.py +83 -0
- khoj/database/migrations/0010_rename_embeddings_entry_and_more.py +30 -0
- khoj/database/migrations/0011_merge_20231102_0138.py +14 -0
- khoj/database/migrations/0012_entry_file_source.py +21 -0
- khoj/database/migrations/0013_subscription.py +37 -0
- khoj/database/migrations/0014_alter_googleuser_picture.py +17 -0
- khoj/database/migrations/0015_alter_subscription_user.py +21 -0
- khoj/database/migrations/0016_alter_subscription_renewal_date.py +17 -0
- khoj/database/migrations/0017_searchmodel.py +32 -0
- khoj/database/migrations/0018_searchmodelconfig_delete_searchmodel.py +30 -0
- khoj/database/migrations/0019_alter_googleuser_family_name_and_more.py +27 -0
- khoj/database/migrations/0020_reflectivequestion.py +36 -0
- khoj/database/migrations/0021_speechtotextmodeloptions_and_more.py +42 -0
- khoj/database/migrations/0022_texttoimagemodelconfig.py +25 -0
- khoj/database/migrations/0023_usersearchmodelconfig.py +33 -0
- khoj/database/migrations/0024_alter_entry_embeddings.py +18 -0
- khoj/database/migrations/0025_clientapplication_khojuser_phone_number_and_more.py +46 -0
- khoj/database/migrations/0025_searchmodelconfig_embeddings_inference_endpoint_and_more.py +22 -0
- khoj/database/migrations/0026_searchmodelconfig_cross_encoder_inference_endpoint_and_more.py +22 -0
- khoj/database/migrations/0027_merge_20240118_1324.py +13 -0
- khoj/database/migrations/0028_khojuser_verified_phone_number.py +17 -0
- khoj/database/migrations/0029_userrequests.py +27 -0
- khoj/database/migrations/0030_conversation_slug_and_title.py +38 -0
- khoj/database/migrations/0031_agent_conversation_agent.py +53 -0
- khoj/database/migrations/0031_alter_googleuser_locale.py +30 -0
- khoj/database/migrations/0032_merge_20240322_0427.py +14 -0
- khoj/database/migrations/0033_rename_tuning_agent_personality.py +17 -0
- khoj/database/migrations/0034_alter_chatmodeloptions_chat_model.py +32 -0
- khoj/database/migrations/0035_processlock.py +26 -0
- khoj/database/migrations/0036_alter_processlock_name.py +19 -0
- khoj/database/migrations/0036_delete_offlinechatprocessorconversationconfig.py +15 -0
- khoj/database/migrations/0036_publicconversation.py +42 -0
- khoj/database/migrations/0037_chatmodeloptions_openai_config_and_more.py +51 -0
- khoj/database/migrations/0037_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +32 -0
- khoj/database/migrations/0038_merge_20240425_0857.py +14 -0
- khoj/database/migrations/0038_merge_20240426_1640.py +12 -0
- khoj/database/migrations/0039_merge_20240501_0301.py +12 -0
- khoj/database/migrations/0040_alter_processlock_name.py +26 -0
- khoj/database/migrations/0040_merge_20240504_1010.py +14 -0
- khoj/database/migrations/0041_merge_20240505_1234.py +14 -0
- khoj/database/migrations/0042_serverchatsettings.py +46 -0
- khoj/database/migrations/0043_alter_chatmodeloptions_model_type.py +21 -0
- khoj/database/migrations/0044_conversation_file_filters.py +17 -0
- khoj/database/migrations/0045_fileobject.py +37 -0
- khoj/database/migrations/0046_khojuser_email_verification_code_and_more.py +22 -0
- khoj/database/migrations/0047_alter_entry_file_type.py +31 -0
- khoj/database/migrations/0048_voicemodeloption_uservoicemodelconfig.py +52 -0
- khoj/database/migrations/0049_datastore.py +38 -0
- khoj/database/migrations/0049_texttoimagemodelconfig_api_key_and_more.py +58 -0
- khoj/database/migrations/0050_alter_processlock_name.py +25 -0
- khoj/database/migrations/0051_merge_20240702_1220.py +14 -0
- khoj/database/migrations/0052_alter_searchmodelconfig_bi_encoder_docs_encode_config_and_more.py +27 -0
- khoj/database/migrations/0053_agent_style_color_agent_style_icon.py +61 -0
- khoj/database/migrations/0054_alter_agent_style_color.py +38 -0
- khoj/database/migrations/0055_alter_agent_style_icon.py +37 -0
- khoj/database/migrations/0056_chatmodeloptions_vision_enabled.py +17 -0
- khoj/database/migrations/0056_searchmodelconfig_cross_encoder_model_config.py +17 -0
- khoj/database/migrations/0057_merge_20240816_1409.py +13 -0
- khoj/database/migrations/0057_remove_serverchatsettings_default_model_and_more.py +51 -0
- khoj/database/migrations/0058_alter_chatmodeloptions_chat_model.py +17 -0
- khoj/database/migrations/0059_searchmodelconfig_bi_encoder_confidence_threshold.py +17 -0
- khoj/database/migrations/0060_merge_20240905_1828.py +14 -0
- khoj/database/migrations/0061_alter_chatmodeloptions_model_type.py +26 -0
- khoj/database/migrations/0061_alter_texttoimagemodelconfig_model_type.py +21 -0
- khoj/database/migrations/0062_merge_20240913_0222.py +14 -0
- khoj/database/migrations/0063_conversation_temp_id.py +36 -0
- khoj/database/migrations/0064_remove_conversation_temp_id_alter_conversation_id.py +86 -0
- khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
- khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
- khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
- khoj/database/migrations/0068_alter_agent_output_modes.py +24 -0
- khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py +89 -0
- khoj/database/migrations/0070_alter_agent_input_tools_alter_agent_output_modes.py +46 -0
- khoj/database/migrations/0071_subscription_enabled_trial_at_and_more.py +32 -0
- khoj/database/migrations/0072_entry_search_model.py +24 -0
- khoj/database/migrations/0073_delete_usersearchmodelconfig.py +15 -0
- khoj/database/migrations/0074_alter_conversation_title.py +17 -0
- khoj/database/migrations/0075_migrate_generated_assets_and_validate.py +85 -0
- khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py +26 -0
- khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py +62 -0
- khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py +17 -0
- khoj/database/migrations/__init__.py +0 -0
- khoj/database/models/__init__.py +725 -0
- khoj/database/tests.py +3 -0
- khoj/interface/compiled/404/index.html +1 -0
- khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_buildManifest.js +1 -0
- khoj/interface/compiled/_next/static/Tg-vU1p1B-YKT5Qv8KSHt/_ssgManifest.js +1 -0
- khoj/interface/compiled/_next/static/chunks/1010-8f39bb4648b5ba10.js +1 -0
- khoj/interface/compiled/_next/static/chunks/182-f1c48a203dc91e0e.js +20 -0
- khoj/interface/compiled/_next/static/chunks/1915-d3c36ad6ce697ce7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/2117-165ef4747a5b836b.js +2 -0
- khoj/interface/compiled/_next/static/chunks/2581-455000f8aeb08fc3.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3727.dcea8f2193111552.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3789-a09e37a819171a9d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/4124-6c28322ce218d2d5.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5427-b52d95253e692bfa.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5473-b1cf56dedac6577a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5477-0bbddb79c25a54a7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/6065-64db9ad305ba0bcd.js +1 -0
- khoj/interface/compiled/_next/static/chunks/6293-469dd16402ea8a6f.js +3 -0
- khoj/interface/compiled/_next/static/chunks/688-b5b4391bbc0376f1.js +1 -0
- khoj/interface/compiled/_next/static/chunks/8667-b6bf63c72b2d76eb.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9259-1172dbaca0515237.js +1 -0
- khoj/interface/compiled/_next/static/chunks/94ca1967.1d9b42d929a1ee8c.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9597.83583248dfbf6e73.js +1 -0
- khoj/interface/compiled/_next/static/chunks/964ecbae.51d6faf8801d15e6.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9665-391df1e5c51c960a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/_not-found/page-a834eddae3e235df.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/layout-e00fb81dca656a10.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/page-28ce086a1129bca2.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/layout-1fe1537449f43496.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/page-bf365a60829d347f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/chat/layout-33934fc2d6ae6838.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/chat/page-0e476e57eb2015e3.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/layout-30e7fda7262713ce.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/page-a5515ea71aec5ef0.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/layout-c02531d586972d7d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/page-9140541e67ea307d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/settings/layout-d09d6510a45cd4bd.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/settings/page-951ba40b5b94b23a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/layout-e8e5db7830bf3f47.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-1beb80d8d741c932.js +1 -0
- khoj/interface/compiled/_next/static/chunks/d3ac728e-44ebd2a0c99b12a0.js +1 -0
- khoj/interface/compiled/_next/static/chunks/fd9d1056-4482b99a36fd1673.js +1 -0
- khoj/interface/compiled/_next/static/chunks/framework-8e0e0f4a6b83a956.js +1 -0
- khoj/interface/compiled/_next/static/chunks/main-app-de1f09df97a3cfc7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/main-db4bfac6b0a8d00b.js +1 -0
- khoj/interface/compiled/_next/static/chunks/pages/_app-3c9ca398d360b709.js +1 -0
- khoj/interface/compiled/_next/static/chunks/pages/_error-cf5ca766ac8f493f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/polyfills-42372ed130431b0a.js +1 -0
- khoj/interface/compiled/_next/static/chunks/webpack-a03962458328b163.js +1 -0
- khoj/interface/compiled/_next/static/css/089de1d8526b96e9.css +1 -0
- khoj/interface/compiled/_next/static/css/37a73b87f02df402.css +1 -0
- khoj/interface/compiled/_next/static/css/4e4e6a4a1c920d06.css +1 -0
- khoj/interface/compiled/_next/static/css/8d02837c730f8d13.css +25 -0
- khoj/interface/compiled/_next/static/css/8e6a3ca11a60b189.css +1 -0
- khoj/interface/compiled/_next/static/css/9c164d9727dd8092.css +1 -0
- khoj/interface/compiled/_next/static/css/dac88c17aaee5fcf.css +1 -0
- khoj/interface/compiled/_next/static/css/df4b47a2d0d85eae.css +1 -0
- khoj/interface/compiled/_next/static/css/e4eb883b5265d372.css +1 -0
- khoj/interface/compiled/_next/static/media/1d8a05b60287ae6c-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/6f22fce21a7c433c-s.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/77c207b095007c34-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/82ef96de0e8f4d8c-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.1608a09b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.4aafdb68.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_AMS-Regular.a79f1c31.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.b6770918.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.cce5b8ec.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Bold.ec17d132.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.07ef19e7.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.55fac258.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Caligraphic-Regular.dad44a7f.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.9f256b85.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.b18f59e1.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Bold.d42a5579.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.7c187121.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.d3c882a6.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Fraktur-Regular.ed38e79f.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.b74a1a8b.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.c3fb5ac2.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Bold.d181c465.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.6f2bb1df.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.70d8b0a5.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-BoldItalic.e3f82f9d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.47373d1e.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.8916142b.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Italic.9024d815.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.0462f03b.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.7f51fe03.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Main-Regular.b7f8fe9b.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.572d331f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.a879cf83.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-BoldItalic.f1035d8d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.5295ba48.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.939bc644.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Math-Italic.f28c23ac.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.8c5b5494.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.94e1e8dc.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Bold.bf59d231.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.3b1e59b3.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.7c9bc82b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Italic.b4c20c84.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.74048478.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.ba21ed5f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_SansSerif-Regular.d4d7ba48.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.03e9641d.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.07505710.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Script-Regular.fe9cbbe1.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.e1e279cb.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.eae34984.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size1-Regular.fabc004a.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.57727022.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.5916a24f.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size2-Regular.d6b476ec.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.9acaf01c.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.a144ef58.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size3-Regular.b4230e7e.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.10d95fd3.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.7a996c9d.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Size4-Regular.fbccdabe.ttf +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.6258592b.woff +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.a8709e36.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/KaTeX_Typewriter-Regular.d97aaf4a.ttf +0 -0
- khoj/interface/compiled/_next/static/media/a6ecd16fa044d500-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/bd82c78e5b7b3fe9-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/c32c8052c071fc42-s.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/c4250770ab8708b6-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/e098aaaecc9cfbb2-s.p.woff2 +0 -0
- khoj/interface/compiled/_next/static/media/flags.3afdda2f.webp +0 -0
- khoj/interface/compiled/_next/static/media/flags@2x.5fbe9fc1.webp +0 -0
- khoj/interface/compiled/_next/static/media/globe.98e105ca.webp +0 -0
- khoj/interface/compiled/_next/static/media/globe@2x.974df6f8.webp +0 -0
- khoj/interface/compiled/agents/index.html +1 -0
- khoj/interface/compiled/agents/index.txt +7 -0
- khoj/interface/compiled/agents.svg +6 -0
- khoj/interface/compiled/assets/icons/khoj_lantern.ico +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern.svg +100 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_1200x1200.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_128x128.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_128x128_dark.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_256x256.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_512x512.png +0 -0
- khoj/interface/compiled/assets/icons/khoj_lantern_logomarktype_1200x630.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-browse-draw-sample.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-plain-chat-sample.png +0 -0
- khoj/interface/compiled/assets/samples/desktop-remember-plan-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-browse-draw-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-plain-chat-sample.png +0 -0
- khoj/interface/compiled/assets/samples/phone-remember-plan-sample.png +0 -0
- khoj/interface/compiled/automation.svg +37 -0
- khoj/interface/compiled/automations/index.html +1 -0
- khoj/interface/compiled/automations/index.txt +8 -0
- khoj/interface/compiled/chat/index.html +1 -0
- khoj/interface/compiled/chat/index.txt +7 -0
- khoj/interface/compiled/chat.svg +24 -0
- khoj/interface/compiled/close.svg +5 -0
- khoj/interface/compiled/copy-button-success.svg +6 -0
- khoj/interface/compiled/copy-button.svg +5 -0
- khoj/interface/compiled/index.html +1 -0
- khoj/interface/compiled/index.txt +7 -0
- khoj/interface/compiled/khoj.webmanifest +76 -0
- khoj/interface/compiled/logo.svg +24 -0
- khoj/interface/compiled/search/index.html +1 -0
- khoj/interface/compiled/search/index.txt +7 -0
- khoj/interface/compiled/send.svg +1 -0
- khoj/interface/compiled/settings/index.html +1 -0
- khoj/interface/compiled/settings/index.txt +9 -0
- khoj/interface/compiled/share/chat/index.html +1 -0
- khoj/interface/compiled/share/chat/index.txt +7 -0
- khoj/interface/compiled/share.svg +8 -0
- khoj/interface/compiled/thumbs-down.svg +6 -0
- khoj/interface/compiled/thumbs-up.svg +6 -0
- khoj/interface/email/feedback.html +34 -0
- khoj/interface/email/magic_link.html +40 -0
- khoj/interface/email/task.html +37 -0
- khoj/interface/email/welcome.html +90 -0
- khoj/interface/web/.well-known/assetlinks.json +11 -0
- khoj/interface/web/assets/icons/agents.svg +19 -0
- khoj/interface/web/assets/icons/automation.svg +43 -0
- khoj/interface/web/assets/icons/chat.svg +24 -0
- khoj/interface/web/assets/icons/github.svg +1 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways-200.png +0 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways-500.png +0 -0
- khoj/interface/web/assets/icons/khoj-logo-sideways.svg +32 -0
- khoj/interface/web/assets/icons/khoj.svg +26 -0
- khoj/interface/web/assets/icons/logotype.svg +1 -0
- khoj/interface/web/assets/icons/search.svg +57 -0
- khoj/interface/web/assets/icons/sync.svg +4 -0
- khoj/interface/web/assets/khoj.css +237 -0
- khoj/interface/web/assets/utils.js +33 -0
- khoj/interface/web/base_config.html +445 -0
- khoj/interface/web/content_source_github_input.html +208 -0
- khoj/interface/web/login.html +310 -0
- khoj/interface/web/utils.html +48 -0
- khoj/main.py +249 -0
- khoj/manage.py +22 -0
- khoj/migrations/__init__.py +0 -0
- khoj/migrations/migrate_offline_chat_default_model.py +69 -0
- khoj/migrations/migrate_offline_chat_default_model_2.py +71 -0
- khoj/migrations/migrate_offline_chat_schema.py +83 -0
- khoj/migrations/migrate_offline_model.py +29 -0
- khoj/migrations/migrate_processor_config_openai.py +67 -0
- khoj/migrations/migrate_server_pg.py +132 -0
- khoj/migrations/migrate_version.py +17 -0
- khoj/processor/__init__.py +0 -0
- khoj/processor/content/__init__.py +0 -0
- khoj/processor/content/docx/__init__.py +0 -0
- khoj/processor/content/docx/docx_to_entries.py +111 -0
- khoj/processor/content/github/__init__.py +0 -0
- khoj/processor/content/github/github_to_entries.py +226 -0
- khoj/processor/content/images/__init__.py +0 -0
- khoj/processor/content/images/image_to_entries.py +117 -0
- khoj/processor/content/markdown/__init__.py +0 -0
- khoj/processor/content/markdown/markdown_to_entries.py +160 -0
- khoj/processor/content/notion/notion_to_entries.py +259 -0
- khoj/processor/content/org_mode/__init__.py +0 -0
- khoj/processor/content/org_mode/org_to_entries.py +226 -0
- khoj/processor/content/org_mode/orgnode.py +532 -0
- khoj/processor/content/pdf/__init__.py +0 -0
- khoj/processor/content/pdf/pdf_to_entries.py +119 -0
- khoj/processor/content/plaintext/__init__.py +0 -0
- khoj/processor/content/plaintext/plaintext_to_entries.py +117 -0
- khoj/processor/content/text_to_entries.py +296 -0
- khoj/processor/conversation/__init__.py +0 -0
- khoj/processor/conversation/anthropic/__init__.py +0 -0
- khoj/processor/conversation/anthropic/anthropic_chat.py +243 -0
- khoj/processor/conversation/anthropic/utils.py +217 -0
- khoj/processor/conversation/google/__init__.py +0 -0
- khoj/processor/conversation/google/gemini_chat.py +253 -0
- khoj/processor/conversation/google/utils.py +260 -0
- khoj/processor/conversation/offline/__init__.py +0 -0
- khoj/processor/conversation/offline/chat_model.py +308 -0
- khoj/processor/conversation/offline/utils.py +80 -0
- khoj/processor/conversation/offline/whisper.py +15 -0
- khoj/processor/conversation/openai/__init__.py +0 -0
- khoj/processor/conversation/openai/gpt.py +243 -0
- khoj/processor/conversation/openai/utils.py +232 -0
- khoj/processor/conversation/openai/whisper.py +13 -0
- khoj/processor/conversation/prompts.py +1188 -0
- khoj/processor/conversation/utils.py +867 -0
- khoj/processor/embeddings.py +122 -0
- khoj/processor/image/generate.py +215 -0
- khoj/processor/speech/__init__.py +0 -0
- khoj/processor/speech/text_to_speech.py +51 -0
- khoj/processor/tools/__init__.py +0 -0
- khoj/processor/tools/online_search.py +472 -0
- khoj/processor/tools/run_code.py +179 -0
- khoj/routers/__init__.py +0 -0
- khoj/routers/api.py +760 -0
- khoj/routers/api_agents.py +295 -0
- khoj/routers/api_chat.py +1273 -0
- khoj/routers/api_content.py +634 -0
- khoj/routers/api_model.py +123 -0
- khoj/routers/api_phone.py +86 -0
- khoj/routers/api_subscription.py +144 -0
- khoj/routers/auth.py +307 -0
- khoj/routers/email.py +135 -0
- khoj/routers/helpers.py +2333 -0
- khoj/routers/notion.py +85 -0
- khoj/routers/research.py +364 -0
- khoj/routers/storage.py +63 -0
- khoj/routers/twilio.py +36 -0
- khoj/routers/web_client.py +141 -0
- khoj/search_filter/__init__.py +0 -0
- khoj/search_filter/base_filter.py +15 -0
- khoj/search_filter/date_filter.py +215 -0
- khoj/search_filter/file_filter.py +32 -0
- khoj/search_filter/word_filter.py +29 -0
- khoj/search_type/__init__.py +0 -0
- khoj/search_type/text_search.py +255 -0
- khoj/utils/__init__.py +0 -0
- khoj/utils/cli.py +101 -0
- khoj/utils/config.py +81 -0
- khoj/utils/constants.py +51 -0
- khoj/utils/fs_syncer.py +252 -0
- khoj/utils/helpers.py +627 -0
- khoj/utils/initialization.py +301 -0
- khoj/utils/jsonl.py +43 -0
- khoj/utils/models.py +47 -0
- khoj/utils/rawconfig.py +208 -0
- khoj/utils/state.py +48 -0
- khoj/utils/yaml.py +47 -0
- khoj-1.33.3.dev32.dist-info/METADATA +190 -0
- khoj-1.33.3.dev32.dist-info/RECORD +393 -0
- khoj-1.33.3.dev32.dist-info/WHEEL +4 -0
- khoj-1.33.3.dev32.dist-info/entry_points.txt +2 -0
- khoj-1.33.3.dev32.dist-info/licenses/LICENSE +661 -0
khoj/routers/notion.py
ADDED
@@ -0,0 +1,85 @@
|
|
1
|
+
import asyncio
|
2
|
+
import base64
|
3
|
+
import json
|
4
|
+
import logging
|
5
|
+
import os
|
6
|
+
from concurrent.futures import ThreadPoolExecutor
|
7
|
+
|
8
|
+
import requests
|
9
|
+
from fastapi import APIRouter, BackgroundTasks, Request, Response
|
10
|
+
from starlette.responses import RedirectResponse
|
11
|
+
|
12
|
+
from khoj.database.adapters import aget_user_by_uuid
|
13
|
+
from khoj.database.models import KhojUser, NotionConfig
|
14
|
+
from khoj.routers.helpers import configure_content
|
15
|
+
from khoj.utils.state import SearchType
|
16
|
+
|
17
|
+
NOTION_OAUTH_CLIENT_ID = os.getenv("NOTION_OAUTH_CLIENT_ID")
|
18
|
+
NOTION_OAUTH_CLIENT_SECRET = os.getenv("NOTION_OAUTH_CLIENT_SECRET")
|
19
|
+
NOTION_REDIRECT_URI = os.getenv("NOTION_REDIRECT_URI")
|
20
|
+
|
21
|
+
notion_router = APIRouter()
|
22
|
+
|
23
|
+
executor = ThreadPoolExecutor()
|
24
|
+
|
25
|
+
logger = logging.getLogger(__name__)
|
26
|
+
|
27
|
+
|
28
|
+
async def run_in_executor(func, *args):
|
29
|
+
loop = asyncio.get_event_loop()
|
30
|
+
return await loop.run_in_executor(executor, func, *args)
|
31
|
+
|
32
|
+
|
33
|
+
@notion_router.get("/auth/callback")
|
34
|
+
async def notion_auth_callback(request: Request, background_tasks: BackgroundTasks):
|
35
|
+
code = request.query_params.get("code")
|
36
|
+
state = request.query_params.get("state")
|
37
|
+
if not code or not state:
|
38
|
+
return Response("Missing code or state", status_code=400)
|
39
|
+
|
40
|
+
user: KhojUser = await aget_user_by_uuid(state)
|
41
|
+
|
42
|
+
await NotionConfig.objects.filter(user=user).adelete()
|
43
|
+
|
44
|
+
if not user:
|
45
|
+
raise Exception("User not found")
|
46
|
+
|
47
|
+
bearer_token = f"{NOTION_OAUTH_CLIENT_ID}:{NOTION_OAUTH_CLIENT_SECRET}"
|
48
|
+
base64_encoded_token = base64.b64encode(bearer_token.encode()).decode()
|
49
|
+
|
50
|
+
headers = {
|
51
|
+
"Accept": "application/json",
|
52
|
+
"Content-Type": "application/json",
|
53
|
+
"Authorization": f"Basic {base64_encoded_token}",
|
54
|
+
}
|
55
|
+
|
56
|
+
data = {
|
57
|
+
"grant_type": "authorization_code",
|
58
|
+
"code": code,
|
59
|
+
"redirect_uri": NOTION_REDIRECT_URI,
|
60
|
+
}
|
61
|
+
|
62
|
+
response = requests.post("https://api.notion.com/v1/oauth/token", data=json.dumps(data), headers=headers)
|
63
|
+
|
64
|
+
final_response = response.json()
|
65
|
+
|
66
|
+
logger.info(f"Notion auth callback response: {final_response}")
|
67
|
+
|
68
|
+
access_token = final_response.get("access_token")
|
69
|
+
await NotionConfig.objects.acreate(token=access_token, user=user)
|
70
|
+
|
71
|
+
owner = final_response.get("owner")
|
72
|
+
workspace_id = final_response.get("workspace_id")
|
73
|
+
workspace_name = final_response.get("workspace_name")
|
74
|
+
bot_id = final_response.get("bot_id")
|
75
|
+
|
76
|
+
logger.info(
|
77
|
+
f"Notion integration. Owner: {owner}, Workspace ID: {workspace_id}, Workspace Name: {workspace_name}, Bot ID: {bot_id}"
|
78
|
+
)
|
79
|
+
|
80
|
+
notion_redirect = str(request.app.url_path_for("config_page"))
|
81
|
+
|
82
|
+
# Trigger an async job to configure_content. Let it run without blocking the response.
|
83
|
+
background_tasks.add_task(run_in_executor, configure_content, user, {}, False, SearchType.Notion)
|
84
|
+
|
85
|
+
return RedirectResponse(notion_redirect)
|
khoj/routers/research.py
ADDED
@@ -0,0 +1,364 @@
|
|
1
|
+
import logging
|
2
|
+
from datetime import datetime
|
3
|
+
from typing import Callable, Dict, List, Optional
|
4
|
+
|
5
|
+
import yaml
|
6
|
+
from fastapi import Request
|
7
|
+
|
8
|
+
from khoj.database.models import Agent, KhojUser
|
9
|
+
from khoj.processor.conversation import prompts
|
10
|
+
from khoj.processor.conversation.utils import (
|
11
|
+
InformationCollectionIteration,
|
12
|
+
construct_chat_history,
|
13
|
+
construct_iteration_history,
|
14
|
+
construct_tool_chat_history,
|
15
|
+
load_complex_json,
|
16
|
+
)
|
17
|
+
from khoj.processor.tools.online_search import read_webpages, search_online
|
18
|
+
from khoj.processor.tools.run_code import run_code
|
19
|
+
from khoj.routers.api import extract_references_and_questions
|
20
|
+
from khoj.routers.helpers import (
|
21
|
+
ChatEvent,
|
22
|
+
generate_summary_from_files,
|
23
|
+
send_message_to_model_wrapper,
|
24
|
+
)
|
25
|
+
from khoj.utils.helpers import (
|
26
|
+
ConversationCommand,
|
27
|
+
function_calling_description_for_llm,
|
28
|
+
is_none_or_empty,
|
29
|
+
timer,
|
30
|
+
truncate_code_context,
|
31
|
+
)
|
32
|
+
from khoj.utils.rawconfig import LocationData
|
33
|
+
|
34
|
+
logger = logging.getLogger(__name__)
|
35
|
+
|
36
|
+
|
37
|
+
async def apick_next_tool(
|
38
|
+
query: str,
|
39
|
+
conversation_history: dict,
|
40
|
+
user: KhojUser = None,
|
41
|
+
query_images: List[str] = [],
|
42
|
+
location: LocationData = None,
|
43
|
+
user_name: str = None,
|
44
|
+
agent: Agent = None,
|
45
|
+
previous_iterations: List[InformationCollectionIteration] = [],
|
46
|
+
max_iterations: int = 5,
|
47
|
+
send_status_func: Optional[Callable] = None,
|
48
|
+
tracer: dict = {},
|
49
|
+
query_files: str = None,
|
50
|
+
):
|
51
|
+
"""Given a query, determine which of the available tools the agent should use in order to answer appropriately."""
|
52
|
+
|
53
|
+
# Construct tool options for the agent to choose from
|
54
|
+
tool_options = dict()
|
55
|
+
tool_options_str = ""
|
56
|
+
agent_tools = agent.input_tools if agent else []
|
57
|
+
for tool, description in function_calling_description_for_llm.items():
|
58
|
+
tool_options[tool.value] = description
|
59
|
+
if len(agent_tools) == 0 or tool.value in agent_tools:
|
60
|
+
tool_options_str += f'- "{tool.value}": "{description}"\n'
|
61
|
+
|
62
|
+
# Construct chat history with user and iteration history with researcher agent for context
|
63
|
+
chat_history = construct_chat_history(conversation_history, agent_name=agent.name if agent else "Khoj")
|
64
|
+
previous_iterations_history = construct_iteration_history(previous_iterations, prompts.previous_iteration)
|
65
|
+
|
66
|
+
if query_images:
|
67
|
+
query = f"[placeholder for user attached images]\n{query}"
|
68
|
+
|
69
|
+
today = datetime.today()
|
70
|
+
location_data = f"{location}" if location else "Unknown"
|
71
|
+
personality_context = (
|
72
|
+
prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
|
73
|
+
)
|
74
|
+
|
75
|
+
function_planning_prompt = prompts.plan_function_execution.format(
|
76
|
+
tools=tool_options_str,
|
77
|
+
chat_history=chat_history,
|
78
|
+
personality_context=personality_context,
|
79
|
+
current_date=today.strftime("%Y-%m-%d"),
|
80
|
+
day_of_week=today.strftime("%A"),
|
81
|
+
username=user_name or "Unknown",
|
82
|
+
location=location_data,
|
83
|
+
previous_iterations=previous_iterations_history,
|
84
|
+
max_iterations=max_iterations,
|
85
|
+
)
|
86
|
+
|
87
|
+
try:
|
88
|
+
with timer("Chat actor: Infer information sources to refer", logger):
|
89
|
+
response = await send_message_to_model_wrapper(
|
90
|
+
query=query,
|
91
|
+
context=function_planning_prompt,
|
92
|
+
response_type="json_object",
|
93
|
+
user=user,
|
94
|
+
query_images=query_images,
|
95
|
+
query_files=query_files,
|
96
|
+
tracer=tracer,
|
97
|
+
)
|
98
|
+
except Exception as e:
|
99
|
+
logger.error(f"Failed to infer information sources to refer: {e}", exc_info=True)
|
100
|
+
yield InformationCollectionIteration(
|
101
|
+
tool=None,
|
102
|
+
query=None,
|
103
|
+
warning="Failed to infer information sources to refer. Skipping iteration. Try again.",
|
104
|
+
)
|
105
|
+
return
|
106
|
+
|
107
|
+
try:
|
108
|
+
response = load_complex_json(response)
|
109
|
+
selected_tool = response.get("tool", None)
|
110
|
+
generated_query = response.get("query", None)
|
111
|
+
scratchpad = response.get("scratchpad", None)
|
112
|
+
warning = None
|
113
|
+
logger.info(f"Response for determining relevant tools: {response}")
|
114
|
+
|
115
|
+
# Detect selection of previously used query, tool combination.
|
116
|
+
previous_tool_query_combinations = {(i.tool, i.query) for i in previous_iterations if i.warning is None}
|
117
|
+
if (selected_tool, generated_query) in previous_tool_query_combinations:
|
118
|
+
warning = f"Repeated tool, query combination detected. Skipping iteration. Try something different."
|
119
|
+
# Only send client status updates if we'll execute this iteration
|
120
|
+
elif send_status_func:
|
121
|
+
determined_tool_message = "**Determined Tool**: "
|
122
|
+
determined_tool_message += f"{selected_tool}({generated_query})." if selected_tool else "respond."
|
123
|
+
determined_tool_message += f"\nReason: {scratchpad}" if scratchpad else ""
|
124
|
+
async for event in send_status_func(f"{scratchpad}"):
|
125
|
+
yield {ChatEvent.STATUS: event}
|
126
|
+
|
127
|
+
yield InformationCollectionIteration(
|
128
|
+
tool=selected_tool,
|
129
|
+
query=generated_query,
|
130
|
+
warning=warning,
|
131
|
+
)
|
132
|
+
except Exception as e:
|
133
|
+
logger.error(f"Invalid response for determining relevant tools: {response}. {e}", exc_info=True)
|
134
|
+
yield InformationCollectionIteration(
|
135
|
+
tool=None,
|
136
|
+
query=None,
|
137
|
+
warning=f"Invalid response for determining relevant tools: {response}. Skipping iteration. Fix error: {e}",
|
138
|
+
)
|
139
|
+
|
140
|
+
|
141
|
+
async def execute_information_collection(
|
142
|
+
request: Request,
|
143
|
+
user: KhojUser,
|
144
|
+
query: str,
|
145
|
+
conversation_id: str,
|
146
|
+
conversation_history: dict,
|
147
|
+
query_images: List[str],
|
148
|
+
agent: Agent = None,
|
149
|
+
send_status_func: Optional[Callable] = None,
|
150
|
+
user_name: str = None,
|
151
|
+
location: LocationData = None,
|
152
|
+
file_filters: List[str] = [],
|
153
|
+
tracer: dict = {},
|
154
|
+
query_files: str = None,
|
155
|
+
):
|
156
|
+
current_iteration = 0
|
157
|
+
MAX_ITERATIONS = 5
|
158
|
+
previous_iterations: List[InformationCollectionIteration] = []
|
159
|
+
while current_iteration < MAX_ITERATIONS:
|
160
|
+
online_results: Dict = dict()
|
161
|
+
code_results: Dict = dict()
|
162
|
+
document_results: List[Dict[str, str]] = []
|
163
|
+
summarize_files: str = ""
|
164
|
+
this_iteration = InformationCollectionIteration(tool=None, query=query)
|
165
|
+
|
166
|
+
async for result in apick_next_tool(
|
167
|
+
query,
|
168
|
+
conversation_history,
|
169
|
+
user,
|
170
|
+
query_images,
|
171
|
+
location,
|
172
|
+
user_name,
|
173
|
+
agent,
|
174
|
+
previous_iterations,
|
175
|
+
MAX_ITERATIONS,
|
176
|
+
send_status_func,
|
177
|
+
tracer=tracer,
|
178
|
+
query_files=query_files,
|
179
|
+
):
|
180
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
181
|
+
yield result[ChatEvent.STATUS]
|
182
|
+
elif isinstance(result, InformationCollectionIteration):
|
183
|
+
this_iteration = result
|
184
|
+
|
185
|
+
# Skip running iteration if warning present in iteration
|
186
|
+
if this_iteration.warning:
|
187
|
+
logger.warning(f"Research mode: {this_iteration.warning}.")
|
188
|
+
|
189
|
+
elif this_iteration.tool == ConversationCommand.Notes:
|
190
|
+
this_iteration.context = []
|
191
|
+
document_results = []
|
192
|
+
previous_inferred_queries = {
|
193
|
+
c["query"] for iteration in previous_iterations if iteration.context for c in iteration.context
|
194
|
+
}
|
195
|
+
async for result in extract_references_and_questions(
|
196
|
+
request,
|
197
|
+
construct_tool_chat_history(previous_iterations, ConversationCommand.Notes),
|
198
|
+
this_iteration.query,
|
199
|
+
7,
|
200
|
+
None,
|
201
|
+
conversation_id,
|
202
|
+
[ConversationCommand.Default],
|
203
|
+
location,
|
204
|
+
send_status_func,
|
205
|
+
query_images,
|
206
|
+
previous_inferred_queries=previous_inferred_queries,
|
207
|
+
agent=agent,
|
208
|
+
tracer=tracer,
|
209
|
+
query_files=query_files,
|
210
|
+
):
|
211
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
212
|
+
yield result[ChatEvent.STATUS]
|
213
|
+
elif isinstance(result, tuple):
|
214
|
+
document_results = result[0]
|
215
|
+
this_iteration.context += document_results
|
216
|
+
|
217
|
+
if not is_none_or_empty(document_results):
|
218
|
+
try:
|
219
|
+
distinct_files = {d["file"] for d in document_results}
|
220
|
+
distinct_headings = set([d["compiled"].split("\n")[0] for d in document_results if "compiled" in d])
|
221
|
+
# Strip only leading # from headings
|
222
|
+
headings_str = "\n- " + "\n- ".join(distinct_headings).replace("#", "")
|
223
|
+
async for result in send_status_func(
|
224
|
+
f"**Found {len(distinct_headings)} Notes Across {len(distinct_files)} Files**: {headings_str}"
|
225
|
+
):
|
226
|
+
yield result
|
227
|
+
except Exception as e:
|
228
|
+
this_iteration.warning = f"Error extracting document references: {e}"
|
229
|
+
logger.error(this_iteration.warning, exc_info=True)
|
230
|
+
|
231
|
+
elif this_iteration.tool == ConversationCommand.Online:
|
232
|
+
previous_subqueries = {
|
233
|
+
subquery
|
234
|
+
for iteration in previous_iterations
|
235
|
+
if iteration.onlineContext
|
236
|
+
for subquery in iteration.onlineContext.keys()
|
237
|
+
}
|
238
|
+
try:
|
239
|
+
async for result in search_online(
|
240
|
+
this_iteration.query,
|
241
|
+
construct_tool_chat_history(previous_iterations, ConversationCommand.Online),
|
242
|
+
location,
|
243
|
+
user,
|
244
|
+
send_status_func,
|
245
|
+
[],
|
246
|
+
max_webpages_to_read=0,
|
247
|
+
query_images=query_images,
|
248
|
+
previous_subqueries=previous_subqueries,
|
249
|
+
agent=agent,
|
250
|
+
tracer=tracer,
|
251
|
+
):
|
252
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
253
|
+
yield result[ChatEvent.STATUS]
|
254
|
+
elif is_none_or_empty(result):
|
255
|
+
this_iteration.warning = "Detected previously run online search queries. Skipping iteration. Try something different."
|
256
|
+
else:
|
257
|
+
online_results: Dict[str, Dict] = result # type: ignore
|
258
|
+
this_iteration.onlineContext = online_results
|
259
|
+
except Exception as e:
|
260
|
+
this_iteration.warning = f"Error searching online: {e}"
|
261
|
+
logger.error(this_iteration.warning, exc_info=True)
|
262
|
+
|
263
|
+
elif this_iteration.tool == ConversationCommand.Webpage:
|
264
|
+
try:
|
265
|
+
async for result in read_webpages(
|
266
|
+
this_iteration.query,
|
267
|
+
construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
|
268
|
+
location,
|
269
|
+
user,
|
270
|
+
send_status_func,
|
271
|
+
query_images=query_images,
|
272
|
+
agent=agent,
|
273
|
+
tracer=tracer,
|
274
|
+
query_files=query_files,
|
275
|
+
):
|
276
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
277
|
+
yield result[ChatEvent.STATUS]
|
278
|
+
else:
|
279
|
+
direct_web_pages: Dict[str, Dict] = result # type: ignore
|
280
|
+
|
281
|
+
webpages = []
|
282
|
+
for web_query in direct_web_pages:
|
283
|
+
if online_results.get(web_query):
|
284
|
+
online_results[web_query]["webpages"] = direct_web_pages[web_query]["webpages"]
|
285
|
+
else:
|
286
|
+
online_results[web_query] = {"webpages": direct_web_pages[web_query]["webpages"]}
|
287
|
+
|
288
|
+
for webpage in direct_web_pages[web_query]["webpages"]:
|
289
|
+
webpages.append(webpage["link"])
|
290
|
+
this_iteration.onlineContext = online_results
|
291
|
+
except Exception as e:
|
292
|
+
this_iteration.warning = f"Error reading webpages: {e}"
|
293
|
+
logger.error(this_iteration.warning, exc_info=True)
|
294
|
+
|
295
|
+
elif this_iteration.tool == ConversationCommand.Code:
|
296
|
+
try:
|
297
|
+
async for result in run_code(
|
298
|
+
this_iteration.query,
|
299
|
+
construct_tool_chat_history(previous_iterations, ConversationCommand.Webpage),
|
300
|
+
"",
|
301
|
+
location,
|
302
|
+
user,
|
303
|
+
send_status_func,
|
304
|
+
query_images=query_images,
|
305
|
+
agent=agent,
|
306
|
+
query_files=query_files,
|
307
|
+
tracer=tracer,
|
308
|
+
):
|
309
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
310
|
+
yield result[ChatEvent.STATUS]
|
311
|
+
else:
|
312
|
+
code_results: Dict[str, Dict] = result # type: ignore
|
313
|
+
this_iteration.codeContext = code_results
|
314
|
+
async for result in send_status_func(f"**Ran code snippets**: {len(this_iteration.codeContext)}"):
|
315
|
+
yield result
|
316
|
+
except ValueError as e:
|
317
|
+
this_iteration.warning = f"Error running code: {e}"
|
318
|
+
logger.warning(this_iteration.warning, exc_info=True)
|
319
|
+
|
320
|
+
elif this_iteration.tool == ConversationCommand.Summarize:
|
321
|
+
try:
|
322
|
+
async for result in generate_summary_from_files(
|
323
|
+
this_iteration.query,
|
324
|
+
user,
|
325
|
+
file_filters,
|
326
|
+
construct_tool_chat_history(previous_iterations),
|
327
|
+
query_images=query_images,
|
328
|
+
agent=agent,
|
329
|
+
send_status_func=send_status_func,
|
330
|
+
query_files=query_files,
|
331
|
+
):
|
332
|
+
if isinstance(result, dict) and ChatEvent.STATUS in result:
|
333
|
+
yield result[ChatEvent.STATUS]
|
334
|
+
else:
|
335
|
+
summarize_files = result # type: ignore
|
336
|
+
except Exception as e:
|
337
|
+
this_iteration.warning = f"Error summarizing files: {e}"
|
338
|
+
logger.error(this_iteration.warning, exc_info=True)
|
339
|
+
|
340
|
+
else:
|
341
|
+
# No valid tools. This is our exit condition.
|
342
|
+
current_iteration = MAX_ITERATIONS
|
343
|
+
|
344
|
+
current_iteration += 1
|
345
|
+
|
346
|
+
if document_results or online_results or code_results or summarize_files or this_iteration.warning:
|
347
|
+
results_data = f"\n<iteration>{current_iteration}\n<tool>{this_iteration.tool}</tool>\n<query>{this_iteration.query}</query>\n<results>"
|
348
|
+
if document_results:
|
349
|
+
results_data += f"\n<document_references>\n{yaml.dump(document_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</document_references>"
|
350
|
+
if online_results:
|
351
|
+
results_data += f"\n<online_results>\n{yaml.dump(online_results, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</online_results>"
|
352
|
+
if code_results:
|
353
|
+
results_data += f"\n<code_results>\n{yaml.dump(truncate_code_context(code_results), allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</code_results>"
|
354
|
+
if summarize_files:
|
355
|
+
results_data += f"\n<summarized_files>\n{yaml.dump(summarize_files, allow_unicode=True, sort_keys=False, default_flow_style=False)}\n</summarized_files>"
|
356
|
+
if this_iteration.warning:
|
357
|
+
results_data += f"\n<warning>\n{this_iteration.warning}\n</warning>"
|
358
|
+
results_data += "\n</results>\n</iteration>"
|
359
|
+
|
360
|
+
# intermediate_result = await extract_relevant_info(this_iteration.query, results_data, agent)
|
361
|
+
this_iteration.summarizedResult = results_data
|
362
|
+
|
363
|
+
previous_iterations.append(this_iteration)
|
364
|
+
yield this_iteration
|
khoj/routers/storage.py
ADDED
@@ -0,0 +1,63 @@
|
|
1
|
+
import logging
|
2
|
+
import os
|
3
|
+
import uuid
|
4
|
+
|
5
|
+
logger = logging.getLogger(__name__)
|
6
|
+
|
7
|
+
AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY")
|
8
|
+
AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY")
|
9
|
+
# S3 supports serving assets via your domain. Khoj expects this to be used in production. To enable it:
|
10
|
+
# 1. Your bucket name for images should be of the form sub.domain.tld. For example, generated.khoj.dev
|
11
|
+
# 2. Add CNAME entry to your domain's DNS records pointing to the S3 bucket. For example, CNAME generated.khoj.dev generated-khoj-dev.s3.amazonaws.com
|
12
|
+
AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET")
|
13
|
+
|
14
|
+
aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None
|
15
|
+
|
16
|
+
if aws_enabled:
|
17
|
+
from boto3 import client
|
18
|
+
|
19
|
+
s3_client = client("s3", aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY)
|
20
|
+
|
21
|
+
|
22
|
+
def upload_image(image: bytes, user_id: uuid.UUID):
|
23
|
+
"""Upload the image to the S3 bucket"""
|
24
|
+
if not aws_enabled:
|
25
|
+
logger.info("AWS is not enabled. Skipping image upload")
|
26
|
+
return None
|
27
|
+
|
28
|
+
image_key = f"{user_id}/{uuid.uuid4()}.webp"
|
29
|
+
try:
|
30
|
+
s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read")
|
31
|
+
url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}/{image_key}"
|
32
|
+
return url
|
33
|
+
except Exception as e:
|
34
|
+
logger.error(f"Failed to upload image to S3: {e}")
|
35
|
+
return None
|
36
|
+
|
37
|
+
|
38
|
+
AWS_USER_UPLOADED_IMAGES_BUCKET_NAME = os.getenv("AWS_USER_UPLOADED_IMAGES_BUCKET_NAME")
|
39
|
+
|
40
|
+
|
41
|
+
def upload_image_to_bucket(image: bytes, user_id: uuid.UUID):
|
42
|
+
"""Upload the image to the S3 bucket"""
|
43
|
+
if not aws_enabled:
|
44
|
+
logger.info("AWS is not enabled. Skipping image upload")
|
45
|
+
return None
|
46
|
+
|
47
|
+
image_key = f"{user_id}/{uuid.uuid4()}.webp"
|
48
|
+
if not AWS_USER_UPLOADED_IMAGES_BUCKET_NAME:
|
49
|
+
logger.error("AWS_USER_UPLOADED_IMAGES_BUCKET_NAME is not set")
|
50
|
+
return None
|
51
|
+
|
52
|
+
try:
|
53
|
+
s3_client.put_object(
|
54
|
+
Bucket=AWS_USER_UPLOADED_IMAGES_BUCKET_NAME,
|
55
|
+
Key=image_key,
|
56
|
+
Body=image,
|
57
|
+
ACL="public-read",
|
58
|
+
ContentType="image/webp",
|
59
|
+
)
|
60
|
+
return f"https://{AWS_USER_UPLOADED_IMAGES_BUCKET_NAME}/{image_key}"
|
61
|
+
except Exception as e:
|
62
|
+
logger.error(f"Failed to upload image to S3: {e}")
|
63
|
+
return None
|
khoj/routers/twilio.py
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
import logging
|
2
|
+
import os
|
3
|
+
|
4
|
+
from khoj.database.models import KhojUser
|
5
|
+
|
6
|
+
logger = logging.getLogger(__name__)
|
7
|
+
|
8
|
+
account_sid = os.getenv("TWILIO_ACCOUNT_SID")
|
9
|
+
auth_token = os.getenv("TWILIO_AUTH_TOKEN")
|
10
|
+
verification_service_sid = os.getenv("TWILIO_VERIFICATION_SID")
|
11
|
+
|
12
|
+
twilio_enabled = account_sid is not None and auth_token is not None and verification_service_sid is not None
|
13
|
+
if twilio_enabled:
|
14
|
+
from twilio.rest import Client
|
15
|
+
|
16
|
+
client = Client(account_sid, auth_token)
|
17
|
+
|
18
|
+
|
19
|
+
def is_twilio_enabled():
|
20
|
+
return twilio_enabled
|
21
|
+
|
22
|
+
|
23
|
+
def create_otp(user: KhojUser):
|
24
|
+
"""Create a new OTP for the user"""
|
25
|
+
verification = client.verify.v2.services(verification_service_sid).verifications.create(
|
26
|
+
to=str(user.phone_number), channel="whatsapp"
|
27
|
+
)
|
28
|
+
return verification.sid is not None
|
29
|
+
|
30
|
+
|
31
|
+
def verify_otp(user: KhojUser, code: str):
|
32
|
+
"""Verify the OTP for the user"""
|
33
|
+
verification_check = client.verify.v2.services(verification_service_sid).verification_checks.create(
|
34
|
+
to=str(user.phone_number), code=code
|
35
|
+
)
|
36
|
+
return verification_check.status == "approved"
|
@@ -0,0 +1,141 @@
|
|
1
|
+
# System Packages
|
2
|
+
import json
|
3
|
+
import os
|
4
|
+
|
5
|
+
from fastapi import APIRouter, Request
|
6
|
+
from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
|
7
|
+
from fastapi.templating import Jinja2Templates
|
8
|
+
from starlette.authentication import requires
|
9
|
+
|
10
|
+
from khoj.database.adapters import get_user_github_config
|
11
|
+
from khoj.routers.helpers import get_next_url, get_user_config
|
12
|
+
from khoj.utils import constants
|
13
|
+
from khoj.utils.rawconfig import GithubContentConfig, GithubRepoConfig
|
14
|
+
|
15
|
+
# Initialize Router
|
16
|
+
web_client = APIRouter()
|
17
|
+
templates = Jinja2Templates([constants.web_directory, constants.next_js_directory, constants.pypi_static_directory])
|
18
|
+
|
19
|
+
|
20
|
+
# Create Routes
|
21
|
+
@web_client.get("/", response_class=FileResponse)
|
22
|
+
def index(request: Request):
|
23
|
+
return templates.TemplateResponse("index.html", context={"request": request})
|
24
|
+
|
25
|
+
|
26
|
+
@web_client.post("/", response_class=FileResponse)
|
27
|
+
@requires(["authenticated"], redirect="login_page")
|
28
|
+
def index_post(request: Request):
|
29
|
+
return templates.TemplateResponse("index.html", context={"request": request})
|
30
|
+
|
31
|
+
|
32
|
+
@web_client.get("/search", response_class=FileResponse)
|
33
|
+
@requires(["authenticated"], redirect="login_page")
|
34
|
+
def search_page(request: Request):
|
35
|
+
return templates.TemplateResponse(
|
36
|
+
"search/index.html",
|
37
|
+
context={
|
38
|
+
"request": request,
|
39
|
+
},
|
40
|
+
)
|
41
|
+
|
42
|
+
|
43
|
+
@web_client.get("/chat", response_class=FileResponse)
|
44
|
+
@requires(["authenticated"], redirect="login_page")
|
45
|
+
def chat_page(request: Request):
|
46
|
+
return templates.TemplateResponse(
|
47
|
+
"chat/index.html",
|
48
|
+
context={
|
49
|
+
"request": request,
|
50
|
+
},
|
51
|
+
)
|
52
|
+
|
53
|
+
|
54
|
+
@web_client.get("/login", response_class=FileResponse)
|
55
|
+
def login_page(request: Request):
|
56
|
+
next_url = get_next_url(request)
|
57
|
+
if request.user.is_authenticated:
|
58
|
+
return RedirectResponse(url=next_url)
|
59
|
+
google_client_id = os.environ.get("GOOGLE_CLIENT_ID")
|
60
|
+
redirect_uri = str(request.app.url_path_for("auth_post"))
|
61
|
+
return templates.TemplateResponse(
|
62
|
+
"login.html",
|
63
|
+
context={
|
64
|
+
"request": request,
|
65
|
+
"google_client_id": google_client_id,
|
66
|
+
"redirect_uri": f"{redirect_uri}?next={next_url}",
|
67
|
+
},
|
68
|
+
)
|
69
|
+
|
70
|
+
|
71
|
+
@web_client.get("/agents", response_class=HTMLResponse)
|
72
|
+
def agents_page(request: Request):
|
73
|
+
return templates.TemplateResponse(
|
74
|
+
"agents/index.html",
|
75
|
+
context={
|
76
|
+
"request": request,
|
77
|
+
},
|
78
|
+
)
|
79
|
+
|
80
|
+
|
81
|
+
@web_client.get("/settings", response_class=HTMLResponse)
|
82
|
+
@requires(["authenticated"], redirect="login_page")
|
83
|
+
def config_page(request: Request):
|
84
|
+
return templates.TemplateResponse("settings/index.html", context={"request": request})
|
85
|
+
|
86
|
+
|
87
|
+
@web_client.get("/settings/content/github", response_class=HTMLResponse)
|
88
|
+
@requires(["authenticated"], redirect="login_page")
|
89
|
+
def github_config_page(request: Request):
|
90
|
+
user = request.user.object
|
91
|
+
user_config = get_user_config(user, request)
|
92
|
+
current_github_config = get_user_github_config(user)
|
93
|
+
|
94
|
+
if current_github_config:
|
95
|
+
raw_repos = current_github_config.githubrepoconfig.all()
|
96
|
+
repos = []
|
97
|
+
for repo in raw_repos:
|
98
|
+
repos.append(
|
99
|
+
GithubRepoConfig(
|
100
|
+
name=repo.name,
|
101
|
+
owner=repo.owner,
|
102
|
+
branch=repo.branch,
|
103
|
+
)
|
104
|
+
)
|
105
|
+
current_config = GithubContentConfig(
|
106
|
+
pat_token=current_github_config.pat_token,
|
107
|
+
repos=repos,
|
108
|
+
)
|
109
|
+
current_config = json.loads(current_config.json())
|
110
|
+
else:
|
111
|
+
current_config = {} # type: ignore
|
112
|
+
|
113
|
+
user_config["current_config"] = current_config
|
114
|
+
return templates.TemplateResponse("content_source_github_input.html", context=user_config)
|
115
|
+
|
116
|
+
|
117
|
+
@web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse)
|
118
|
+
def view_public_conversation(request: Request):
|
119
|
+
return templates.TemplateResponse(
|
120
|
+
"share/chat/index.html",
|
121
|
+
context={
|
122
|
+
"request": request,
|
123
|
+
},
|
124
|
+
)
|
125
|
+
|
126
|
+
|
127
|
+
@web_client.get("/automations", response_class=HTMLResponse)
|
128
|
+
def automations_config_page(
|
129
|
+
request: Request,
|
130
|
+
):
|
131
|
+
return templates.TemplateResponse(
|
132
|
+
"automations/index.html",
|
133
|
+
context={
|
134
|
+
"request": request,
|
135
|
+
},
|
136
|
+
)
|
137
|
+
|
138
|
+
|
139
|
+
@web_client.get("/.well-known/assetlinks.json", response_class=FileResponse)
|
140
|
+
def assetlinks(request: Request):
|
141
|
+
return FileResponse(constants.assetlinks_file_path)
|
File without changes
|