khoj 1.30.11.dev64__py3-none-any.whl → 1.32.3.dev34__py3-none-any.whl

khoj/configure.py

@@ -24,6 +24,7 @@ from starlette.concurrency import run_in_threadpool
 from starlette.middleware import Middleware
 from starlette.middleware.authentication import AuthenticationMiddleware
 from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.middleware.httpsredirect import HTTPSRedirectMiddleware
 from starlette.middleware.sessions import SessionMiddleware
 from starlette.requests import HTTPConnection
 from starlette.types import ASGIApp, Receive, Scope, Send
@@ -43,7 +44,6 @@ from khoj.database.adapters import (
 from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.api_content import configure_content, configure_search
-from khoj.routers.helpers import update_telemetry_state
 from khoj.routers.twilio import is_twilio_enabled
 from khoj.utils import constants, state
 from khoj.utils.config import SearchType
@@ -343,7 +343,7 @@ def configure_routes(app):
         logger.info("📞 Enabled Twilio")
 
 
-def configure_middleware(app):
+def configure_middleware(app, ssl_enabled: bool = False):
     class NextJsMiddleware(Middleware):
         async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
             if scope["type"] == "http" and scope["path"].startswith("/_next"):
@@ -354,6 +354,8 @@ def configure_middleware(app):
             super().__init__(app)
             self.app = app
 
+    if ssl_enabled:
+        app.add_middleware(HTTPSRedirectMiddleware)
     app.add_middleware(AsyncCloseConnectionsMiddleware)
     app.add_middleware(AuthenticationMiddleware, backend=UserAuthenticationBackend())
     app.add_middleware(NextJsMiddleware)

khoj/database/adapters/__init__.py

@@ -36,7 +36,7 @@ from torch import Tensor
 from khoj.database.models import (
     Agent,
     AiModelApi,
-    ChatModelOptions,
+    ChatModel,
     ClientApplication,
     Conversation,
     Entry,
@@ -238,7 +238,9 @@ async def aget_or_create_user_by_email(email: str) -> tuple[KhojUser, bool]:
     await user.asave()
 
     if user:
-        user.email_verification_code = secrets.token_urlsafe(18)
+        # Generate a secure 6-digit numeric code
+        user.email_verification_code = f"{secrets.randbelow(1000000):06}"
+        user.email_verification_code_expiry = datetime.now(tz=timezone.utc) + timedelta(minutes=5)
         await user.asave()
 
     user_subscription = await Subscription.objects.filter(user=user).afirst()
@@ -267,16 +269,19 @@ async def astart_trial_subscription(user: KhojUser) -> Subscription:
     return subscription
 
 
-async def aget_user_validated_by_email_verification_code(code: str) -> KhojUser:
-    user = await KhojUser.objects.filter(email_verification_code=code).afirst()
+async def aget_user_validated_by_email_verification_code(code: str, email: str) -> tuple[Optional[KhojUser], bool]:
+    user = await KhojUser.objects.filter(email_verification_code=code, email=email).afirst()
     if not user:
-        return None
+        return None, False
+
+    if user.email_verification_code_expiry < datetime.now(tz=timezone.utc):
+        return user, True
 
     user.email_verification_code = None
     user.verified_email = True
     await user.asave()
 
-    return user
+    return user, False
 
 
 async def create_user_by_google_token(token: dict) -> KhojUser:
@@ -432,10 +437,14 @@ def is_user_subscribed(user: KhojUser) -> bool:
     return subscribed
 
 
-async def get_user_by_email(email: str) -> KhojUser:
+async def aget_user_by_email(email: str) -> KhojUser:
     return await KhojUser.objects.filter(email=email).afirst()
 
 
+def get_user_by_email(email: str) -> KhojUser:
+    return KhojUser.objects.filter(email=email).first()
+
+
 async def aget_user_by_uuid(uuid: str) -> KhojUser:
     return await KhojUser.objects.filter(uuid=uuid).afirst()
 
@@ -736,8 +745,8 @@ class AgentAdapters:
 
     @staticmethod
     def create_default_agent(user: KhojUser):
-        default_conversation_config = ConversationAdapters.get_default_conversation_config(user)
-        if default_conversation_config is None:
+        default_chat_model = ConversationAdapters.get_default_chat_model(user)
+        if default_chat_model is None:
             logger.info("No default conversation config found, skipping default agent creation")
             return None
         default_personality = prompts.personality.format(current_date="placeholder", day_of_week="placeholder")
@@ -746,7 +755,7 @@ class AgentAdapters:
 
         if agent:
             agent.personality = default_personality
-            agent.chat_model = default_conversation_config
+            agent.chat_model = default_chat_model
             agent.slug = AgentAdapters.DEFAULT_AGENT_SLUG
             agent.name = AgentAdapters.DEFAULT_AGENT_NAME
             agent.privacy_level = Agent.PrivacyLevel.PUBLIC
@@ -760,7 +769,7 @@ class AgentAdapters:
                 name=AgentAdapters.DEFAULT_AGENT_NAME,
                 privacy_level=Agent.PrivacyLevel.PUBLIC,
                 managed_by_admin=True,
-                chat_model=default_conversation_config,
+                chat_model=default_chat_model,
                 personality=default_personality,
                 slug=AgentAdapters.DEFAULT_AGENT_SLUG,
             )
@@ -787,7 +796,7 @@ class AgentAdapters:
         output_modes: List[str],
         slug: Optional[str] = None,
     ):
-        chat_model_option = await ChatModelOptions.objects.filter(chat_model=chat_model).afirst()
+        chat_model_option = await ChatModel.objects.filter(name=chat_model).afirst()
 
         # Slug will be None for new agents, which will trigger a new agent creation with a generated, immutable slug
         agent, created = await Agent.objects.filter(slug=slug, creator=user).aupdate_or_create(
@@ -857,7 +866,7 @@ class ConversationAdapters:
             agent=conversation.agent,
             conversation_log=conversation.conversation_log,
             slug=conversation.slug,
-            title=conversation.title,
+            title=conversation.title if conversation.title else conversation.slug,
         )
 
     @staticmethod
@@ -972,29 +981,29 @@ class ConversationAdapters:
 
     @staticmethod
     @require_valid_user
-    def has_any_conversation_config(user: KhojUser):
-        return ChatModelOptions.objects.filter(user=user).exists()
+    def has_any_chat_model(user: KhojUser):
+        return ChatModel.objects.filter(user=user).exists()
 
     @staticmethod
-    def get_all_conversation_configs():
-        return ChatModelOptions.objects.all()
+    def get_all_chat_models():
+        return ChatModel.objects.all()
 
     @staticmethod
-    async def aget_all_conversation_configs():
-        return await sync_to_async(list)(ChatModelOptions.objects.prefetch_related("ai_model_api").all())
+    async def aget_all_chat_models():
+        return await sync_to_async(list)(ChatModel.objects.prefetch_related("ai_model_api").all())
 
     @staticmethod
     def get_vision_enabled_config():
-        conversation_configurations = ConversationAdapters.get_all_conversation_configs()
-        for config in conversation_configurations:
+        chat_models = ConversationAdapters.get_all_chat_models()
+        for config in chat_models:
             if config.vision_enabled:
                 return config
         return None
 
     @staticmethod
     async def aget_vision_enabled_config():
-        conversation_configurations = await ConversationAdapters.aget_all_conversation_configs()
-        for config in conversation_configurations:
+        chat_models = await ConversationAdapters.aget_all_chat_models()
+        for config in chat_models:
             if config.vision_enabled:
                 return config
         return None
@@ -1010,7 +1019,7 @@ class ConversationAdapters:
     @staticmethod
     @arequire_valid_user
     async def aset_user_conversation_processor(user: KhojUser, conversation_processor_config_id: int):
-        config = await ChatModelOptions.objects.filter(id=conversation_processor_config_id).afirst()
+        config = await ChatModel.objects.filter(id=conversation_processor_config_id).afirst()
         if not config:
             return None
         new_config = await UserConversationConfig.objects.aupdate_or_create(user=user, defaults={"setting": config})
@@ -1026,24 +1035,24 @@ class ConversationAdapters:
         return new_config
 
     @staticmethod
-    def get_conversation_config(user: KhojUser):
+    def get_chat_model(user: KhojUser):
         subscribed = is_user_subscribed(user)
         if not subscribed:
-            return ConversationAdapters.get_default_conversation_config(user)
+            return ConversationAdapters.get_default_chat_model(user)
         config = UserConversationConfig.objects.filter(user=user).first()
         if config:
             return config.setting
-        return ConversationAdapters.get_advanced_conversation_config(user)
+        return ConversationAdapters.get_advanced_chat_model(user)
 
     @staticmethod
-    async def aget_conversation_config(user: KhojUser):
+    async def aget_chat_model(user: KhojUser):
         subscribed = await ais_user_subscribed(user)
         if not subscribed:
-            return await ConversationAdapters.aget_default_conversation_config(user)
+            return await ConversationAdapters.aget_default_chat_model(user)
         config = await UserConversationConfig.objects.filter(user=user).prefetch_related("setting").afirst()
         if config:
             return config.setting
-        return ConversationAdapters.aget_advanced_conversation_config(user)
+        return ConversationAdapters.aget_advanced_chat_model(user)
 
     @staticmethod
     async def aget_voice_model_config(user: KhojUser) -> Optional[VoiceModelOption]:
@@ -1064,7 +1073,7 @@ class ConversationAdapters:
         return VoiceModelOption.objects.first()
 
     @staticmethod
-    def get_default_conversation_config(user: KhojUser = None):
+    def get_default_chat_model(user: KhojUser = None):
         """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
         # Get the server chat settings
         server_chat_settings = ServerChatSettings.objects.first()
@@ -1084,10 +1093,10 @@ class ConversationAdapters:
             return user_chat_settings.setting
 
         # Get the first chat model if even the user chat settings are not set
-        return ChatModelOptions.objects.filter().first()
+        return ChatModel.objects.filter().first()
 
     @staticmethod
-    async def aget_default_conversation_config(user: KhojUser = None):
+    async def aget_default_chat_model(user: KhojUser = None):
         """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
         # Get the server chat settings
         server_chat_settings: ServerChatSettings = (
@@ -1117,17 +1126,17 @@ class ConversationAdapters:
             return user_chat_settings.setting
 
         # Get the first chat model if even the user chat settings are not set
-        return await ChatModelOptions.objects.filter().prefetch_related("ai_model_api").afirst()
+        return await ChatModel.objects.filter().prefetch_related("ai_model_api").afirst()
 
     @staticmethod
-    def get_advanced_conversation_config(user: KhojUser):
+    def get_advanced_chat_model(user: KhojUser):
         server_chat_settings = ServerChatSettings.objects.first()
         if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
             return server_chat_settings.chat_advanced
-        return ConversationAdapters.get_default_conversation_config(user)
+        return ConversationAdapters.get_default_chat_model(user)
 
     @staticmethod
-    async def aget_advanced_conversation_config(user: KhojUser = None):
+    async def aget_advanced_chat_model(user: KhojUser = None):
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related("chat_advanced", "chat_advanced__ai_model_api")
@@ -1135,7 +1144,7 @@ class ConversationAdapters:
         )
         if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
             return server_chat_settings.chat_advanced
-        return await ConversationAdapters.aget_default_conversation_config(user)
+        return await ConversationAdapters.aget_default_chat_model(user)
 
     @staticmethod
     async def aget_server_webscraper():
@@ -1247,16 +1256,16 @@ class ConversationAdapters:
 
     @staticmethod
     def get_conversation_processor_options():
-        return ChatModelOptions.objects.all()
+        return ChatModel.objects.all()
 
     @staticmethod
-    def set_conversation_processor_config(user: KhojUser, new_config: ChatModelOptions):
+    def set_user_chat_model(user: KhojUser, chat_model: ChatModel):
         user_conversation_config, _ = UserConversationConfig.objects.get_or_create(user=user)
-        user_conversation_config.setting = new_config
+        user_conversation_config.setting = chat_model
         user_conversation_config.save()
 
     @staticmethod
-    async def aget_user_conversation_config(user: KhojUser):
+    async def aget_user_chat_model(user: KhojUser):
         config = (
             await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__ai_model_api").afirst()
         )
@@ -1288,33 +1297,33 @@ class ConversationAdapters:
         return random.sample(all_questions, max_results)
 
     @staticmethod
-    def get_valid_conversation_config(user: KhojUser, conversation: Conversation):
+    def get_valid_chat_model(user: KhojUser, conversation: Conversation):
         agent: Agent = conversation.agent if AgentAdapters.get_default_agent() != conversation.agent else None
         if agent and agent.chat_model:
-            conversation_config = conversation.agent.chat_model
+            chat_model = conversation.agent.chat_model
         else:
-            conversation_config = ConversationAdapters.get_conversation_config(user)
+            chat_model = ConversationAdapters.get_chat_model(user)
 
-        if conversation_config is None:
-            conversation_config = ConversationAdapters.get_default_conversation_config()
+        if chat_model is None:
+            chat_model = ConversationAdapters.get_default_chat_model()
 
-        if conversation_config.model_type == ChatModelOptions.ModelType.OFFLINE:
+        if chat_model.model_type == ChatModel.ModelType.OFFLINE:
             if state.offline_chat_processor_config is None or state.offline_chat_processor_config.loaded_model is None:
-                chat_model = conversation_config.chat_model
-                max_tokens = conversation_config.max_prompt_size
-                state.offline_chat_processor_config = OfflineChatProcessorModel(chat_model, max_tokens)
+                chat_model_name = chat_model.name
+                max_tokens = chat_model.max_prompt_size
+                state.offline_chat_processor_config = OfflineChatProcessorModel(chat_model_name, max_tokens)
 
-            return conversation_config
+            return chat_model
 
         if (
-            conversation_config.model_type
+            chat_model.model_type
             in [
-                ChatModelOptions.ModelType.ANTHROPIC,
-                ChatModelOptions.ModelType.OPENAI,
-                ChatModelOptions.ModelType.GOOGLE,
+                ChatModel.ModelType.ANTHROPIC,
+                ChatModel.ModelType.OPENAI,
+                ChatModel.ModelType.GOOGLE,
             ]
-        ) and conversation_config.ai_model_api:
-            return conversation_config
+        ) and chat_model.ai_model_api:
+            return chat_model
 
         else:
             raise ValueError("Invalid conversation config - either configure offline chat or openai chat")

khoj/database/admin.py

@@ -16,7 +16,7 @@ from unfold import admin as unfold_admin
 from khoj.database.models import (
     Agent,
     AiModelApi,
-    ChatModelOptions,
+    ChatModel,
     ClientApplication,
     Conversation,
     Entry,
@@ -147,7 +147,7 @@ class KhojUserAdmin(UserAdmin, unfold_admin.ModelAdmin):
             if user.email:
                 host = request.get_host()
                 unique_id = user.email_verification_code
-                login_url = f"{host}/auth/magic?code={unique_id}"
+                login_url = f"{host}/auth/magic?code={unique_id}&email={user.email}"
                 messages.info(request, f"Email login URL for {user.email}: {login_url}")
 
     get_email_login_url.short_description = "Get email login URL"  # type: ignore
@@ -212,15 +212,15 @@ class KhojUserSubscription(unfold_admin.ModelAdmin):
     list_filter = ("type",)
 
 
-@admin.register(ChatModelOptions)
-class ChatModelOptionsAdmin(unfold_admin.ModelAdmin):
+@admin.register(ChatModel)
+class ChatModelAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
-        "chat_model",
+        "name",
         "ai_model_api",
         "max_prompt_size",
     )
-    search_fields = ("id", "chat_model", "ai_model_api__name")
+    search_fields = ("id", "name", "ai_model_api__name")
 
 
 @admin.register(TextToImageModelConfig)
@@ -385,7 +385,7 @@ class UserConversationConfigAdmin(unfold_admin.ModelAdmin):
         "get_chat_model",
         "get_subscription_type",
     )
-    search_fields = ("id", "user__email", "setting__chat_model", "user__subscription__type")
+    search_fields = ("id", "user__email", "setting__name", "user__subscription__type")
     ordering = ("-updated_at",)
 
     def get_user_email(self, obj):
@@ -395,10 +395,10 @@ class UserConversationConfigAdmin(unfold_admin.ModelAdmin):
     get_user_email.admin_order_field = "user__email"  # type: ignore
 
     def get_chat_model(self, obj):
-        return obj.setting.chat_model if obj.setting else None
+        return obj.setting.name if obj.setting else None
 
     get_chat_model.short_description = "Chat Model"  # type: ignore
-    get_chat_model.admin_order_field = "setting__chat_model"  # type: ignore
+    get_chat_model.admin_order_field = "setting__name"  # type: ignore
 
     def get_subscription_type(self, obj):
         if hasattr(obj.user, "subscription"):

khoj/database/migrations/0077_chatmodel_alter_agent_chat_model_and_more.py

@@ -0,0 +1,62 @@
+# Generated by Django 5.0.9 on 2024-12-09 04:21
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="ChatModelOptions",
+            new_name="ChatModel",
+        ),
+        migrations.RenameField(
+            model_name="chatmodel",
+            old_name="chat_model",
+            new_name="name",
+        ),
+        migrations.AlterField(
+            model_name="agent",
+            name="chat_model",
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to="database.chatmodel"),
+        ),
+        migrations.AlterField(
+            model_name="serverchatsettings",
+            name="chat_advanced",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="chat_advanced",
+                to="database.chatmodel",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="serverchatsettings",
+            name="chat_default",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="chat_default",
+                to="database.chatmodel",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="userconversationconfig",
+            name="setting",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="database.chatmodel",
+            ),
+        ),
+    ]

khoj/database/migrations/0078_khojuser_email_verification_code_expiry.py

@@ -0,0 +1,17 @@
+# Generated by Django 5.0.9 on 2024-12-14 18:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0077_chatmodel_alter_agent_chat_model_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="khojuser",
+            name="email_verification_code_expiry",
+            field=models.DateTimeField(blank=True, default=None, null=True),
+        ),
+    ]

khoj/database/models/__init__.py

@@ -138,6 +138,7 @@ class KhojUser(AbstractUser):
     verified_phone_number = models.BooleanField(default=False)
     verified_email = models.BooleanField(default=False)
     email_verification_code = models.CharField(max_length=200, null=True, default=None, blank=True)
+    email_verification_code_expiry = models.DateTimeField(null=True, default=None, blank=True)
 
     def save(self, *args, **kwargs):
         if not self.uuid:
@@ -193,7 +194,7 @@ class AiModelApi(DbBaseModel):
         return self.name
 
 
-class ChatModelOptions(DbBaseModel):
+class ChatModel(DbBaseModel):
     class ModelType(models.TextChoices):
         OPENAI = "openai"
         OFFLINE = "offline"
@@ -203,13 +204,13 @@ class ChatModelOptions(DbBaseModel):
     max_prompt_size = models.IntegerField(default=None, null=True, blank=True)
     subscribed_max_prompt_size = models.IntegerField(default=None, null=True, blank=True)
     tokenizer = models.CharField(max_length=200, default=None, null=True, blank=True)
-    chat_model = models.CharField(max_length=200, default="bartowski/Meta-Llama-3.1-8B-Instruct-GGUF")
+    name = models.CharField(max_length=200, default="bartowski/Meta-Llama-3.1-8B-Instruct-GGUF")
     model_type = models.CharField(max_length=200, choices=ModelType.choices, default=ModelType.OFFLINE)
     vision_enabled = models.BooleanField(default=False)
     ai_model_api = models.ForeignKey(AiModelApi, on_delete=models.CASCADE, default=None, null=True, blank=True)
 
     def __str__(self):
-        return self.chat_model
+        return self.name
 
 
 class VoiceModelOption(DbBaseModel):
@@ -297,7 +298,7 @@ class Agent(DbBaseModel):
         models.CharField(max_length=200, choices=OutputModeOptions.choices), default=list, null=True, blank=True
     )
     managed_by_admin = models.BooleanField(default=False)
-    chat_model = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE)
+    chat_model = models.ForeignKey(ChatModel, on_delete=models.CASCADE)
     slug = models.CharField(max_length=200, unique=True)
     style_color = models.CharField(max_length=200, choices=StyleColorTypes.choices, default=StyleColorTypes.BLUE)
     style_icon = models.CharField(max_length=200, choices=StyleIconTypes.choices, default=StyleIconTypes.LIGHTBULB)
@@ -438,10 +439,10 @@ class WebScraper(DbBaseModel):
 
 class ServerChatSettings(DbBaseModel):
     chat_default = models.ForeignKey(
-        ChatModelOptions, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name="chat_default"
+        ChatModel, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name="chat_default"
     )
     chat_advanced = models.ForeignKey(
-        ChatModelOptions, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name="chat_advanced"
+        ChatModel, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name="chat_advanced"
     )
     web_scraper = models.ForeignKey(
         WebScraper, on_delete=models.CASCADE, default=None, null=True, blank=True, related_name="web_scraper"
@@ -563,7 +564,7 @@ class SpeechToTextModelOptions(DbBaseModel):
 
 class UserConversationConfig(DbBaseModel):
     user = models.OneToOneField(KhojUser, on_delete=models.CASCADE)
-    setting = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE, default=None, null=True, blank=True)
+    setting = models.ForeignKey(ChatModel, on_delete=models.CASCADE, default=None, null=True, blank=True)
 
 
 class UserVoiceModelConfig(DbBaseModel):
@@ -638,7 +639,7 @@ def verify_public_conversation(sender, instance, **kwargs):
 
     # check if this is a new instance
     if instance._state.adding:
-        slug = re.sub(r"\W+", "-", instance.slug.lower())[:50]
+        slug = re.sub(r"\W+", "-", instance.slug.lower())[:50] if instance.slug else uuid.uuid4().hex
         observed_random_id = set()
         while PublicConversation.objects.filter(slug=slug).exists():
             try: