khoj 1.30.11.dev13__py3-none-any.whl → 1.30.11.dev56__py3-none-any.whl

khoj/app/settings.py

@@ -13,6 +13,8 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
 import os
 from pathlib import Path
 
+from django.templatetags.static import static
+
 from khoj.utils.helpers import in_debug_mode, is_env_var_true
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -72,6 +74,7 @@ INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "khoj.database.apps.DatabaseConfig",
+    "unfold",
     "django.contrib.admin",
     "django.contrib.sessions",
     "django.contrib.messages",
@@ -195,3 +198,21 @@ APSCHEDULER_DATETIME_FORMAT = "N j, Y, f:s a"
 # that supports multiple background worker processes instead (e.g. Dramatiq, Celery, Django-RQ,
 # etc. See: https://djangopackages.org/grids/g/workers-queues-tasks/ for popular options).
 APSCHEDULER_RUN_NOW_TIMEOUT = 240  # Seconds
+
+UNFOLD = {
+    "SITE_TITLE": "Khoj Admin Panel",
+    "SITE_HEADER": "Khoj Admin Panel",
+    "SITE_URL": "/",
+    "SITE_ICON": {
+        "light": lambda request: static("assets/icons/khoj_lantern_128x128.png"),
+        "dark": lambda request: static("assets/icons/khoj_lantern_128x128_dark.png"),
+    },
+    "SITE_FAVICONS": [
+        {
+            "rel": "icon",
+            "sizes": "32x32",
+            "type": "image/svg+xml",
+            "href": lambda request: static("assets/icons/khoj_lantern.svg"),
+        },
+    ],
+}

khoj/configure.py

@@ -232,9 +232,9 @@ def configure_server(
         config = FullConfig()
     state.config = config
 
-    if ConversationAdapters.has_valid_openai_conversation_config():
-        openai_config = ConversationAdapters.get_openai_conversation_config()
-        state.openai_client = openai.OpenAI(api_key=openai_config.api_key)
+    if ConversationAdapters.has_valid_ai_model_api():
+        ai_model_api = ConversationAdapters.get_ai_model_api()
+        state.openai_client = openai.OpenAI(api_key=ai_model_api.api_key)
 
     # Initialize Search Models from Config and initialize content
     try:

khoj/database/adapters/__init__.py

@@ -35,6 +35,7 @@ from torch import Tensor
 
 from khoj.database.models import (
     Agent,
+    AiModelApi,
     ChatModelOptions,
     ClientApplication,
     Conversation,
@@ -46,7 +47,6 @@ from khoj.database.models import (
     KhojApiUser,
     KhojUser,
     NotionConfig,
-    OpenAIProcessorConversationConfig,
     ProcessLock,
     PublicConversation,
     ReflectiveQuestion,
@@ -981,7 +981,7 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_all_conversation_configs():
-        return await sync_to_async(list)(ChatModelOptions.objects.prefetch_related("openai_config").all())
+        return await sync_to_async(list)(ChatModelOptions.objects.prefetch_related("ai_model_api").all())
 
     @staticmethod
     def get_vision_enabled_config():
@@ -1000,12 +1000,12 @@ class ConversationAdapters:
         return None
 
     @staticmethod
-    def get_openai_conversation_config():
-        return OpenAIProcessorConversationConfig.objects.filter().first()
+    def get_ai_model_api():
+        return AiModelApi.objects.filter().first()
 
     @staticmethod
-    def has_valid_openai_conversation_config():
-        return OpenAIProcessorConversationConfig.objects.filter().exists()
+    def has_valid_ai_model_api():
+        return AiModelApi.objects.filter().exists()
 
     @staticmethod
     @arequire_valid_user
@@ -1093,7 +1093,7 @@ class ConversationAdapters:
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related(
-                "chat_default", "chat_default__openai_config", "chat_advanced", "chat_advanced__openai_config"
+                "chat_default", "chat_default__ai_model_api", "chat_advanced", "chat_advanced__ai_model_api"
             )
             .afirst()
         )
@@ -1109,7 +1109,7 @@ class ConversationAdapters:
 
         # Get the user's chat settings, if the server chat settings are not set
         user_chat_settings = (
-            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst())
+            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__ai_model_api").afirst())
             if user
             else None
         )
@@ -1117,7 +1117,7 @@ class ConversationAdapters:
             return user_chat_settings.setting
 
         # Get the first chat model if even the user chat settings are not set
-        return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
+        return await ChatModelOptions.objects.filter().prefetch_related("ai_model_api").afirst()
 
     @staticmethod
     def get_advanced_conversation_config(user: KhojUser):
@@ -1130,7 +1130,7 @@ class ConversationAdapters:
     async def aget_advanced_conversation_config(user: KhojUser = None):
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
-            .prefetch_related("chat_advanced", "chat_advanced__openai_config")
+            .prefetch_related("chat_advanced", "chat_advanced__ai_model_api")
             .afirst()
         )
         if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
@@ -1258,7 +1258,7 @@ class ConversationAdapters:
     @staticmethod
     async def aget_user_conversation_config(user: KhojUser):
         config = (
-            await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst()
+            await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__ai_model_api").afirst()
         )
         if not config:
             return None
@@ -1313,7 +1313,7 @@ class ConversationAdapters:
                 ChatModelOptions.ModelType.OPENAI,
                 ChatModelOptions.ModelType.GOOGLE,
             ]
-        ) and conversation_config.openai_config:
+        ) and conversation_config.ai_model_api:
             return conversation_config
 
         else:
@@ -1321,7 +1321,7 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_text_to_image_model_config():
-        return await TextToImageModelConfig.objects.filter().prefetch_related("openai_config").afirst()
+        return await TextToImageModelConfig.objects.filter().prefetch_related("ai_model_api").afirst()
 
     @staticmethod
     def get_text_to_image_model_config():
@@ -1343,9 +1343,9 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_user_text_to_image_model(user: KhojUser) -> Optional[TextToImageModelConfig]:
-        # Create a custom queryset for prefetching settings__openai_config, handling null cases
+        # Create a custom queryset for prefetching settings__ai_model_api, handling null cases
         settings_prefetch = Prefetch(
-            "setting", queryset=TextToImageModelConfig.objects.prefetch_related("openai_config")
+            "setting", queryset=TextToImageModelConfig.objects.prefetch_related("ai_model_api")
         )
 
         config = await UserTextToImageModelConfig.objects.filter(user=user).prefetch_related(settings_prefetch).afirst()

khoj/database/admin.py

@@ -1,17 +1,21 @@
 import csv
 import json
-from datetime import date, datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 from apscheduler.job import Job
 from django.contrib import admin, messages
-from django.contrib.auth.admin import UserAdmin
+from django.contrib.auth.admin import GroupAdmin as BaseGroupAdmin
+from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
+from django.contrib.auth.models import Group
 from django.http import HttpResponse
-from django_apscheduler.admin import DjangoJobAdmin
+from django_apscheduler.admin import DjangoJobAdmin, DjangoJobExecutionAdmin
 from django_apscheduler.jobstores import DjangoJobStore
-from django_apscheduler.models import DjangoJob
+from django_apscheduler.models import DjangoJob, DjangoJobExecution
+from unfold import admin as unfold_admin
 
 from khoj.database.models import (
     Agent,
+    AiModelApi,
     ChatModelOptions,
     ClientApplication,
     Conversation,
@@ -19,7 +23,6 @@ from khoj.database.models import (
     GithubConfig,
     KhojUser,
     NotionConfig,
-    OpenAIProcessorConversationConfig,
     ProcessLock,
     ReflectiveQuestion,
     SearchModelConfig,
@@ -35,10 +38,8 @@ from khoj.database.models import (
 )
 from khoj.utils.helpers import ImageIntentType
 
-admin.site.unregister(DjangoJob)
-
 
-class KhojDjangoJobAdmin(DjangoJobAdmin):
+class KhojDjangoJobAdmin(DjangoJobAdmin, unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "next_run_time",
@@ -62,10 +63,25 @@ class KhojDjangoJobAdmin(DjangoJobAdmin):
         return queryset, use_distinct
 
 
+class KhojDjangoJobExecutionAdmin(DjangoJobExecutionAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+admin.site.unregister(DjangoJob)
 admin.site.register(DjangoJob, KhojDjangoJobAdmin)
+admin.site.unregister(DjangoJobExecution)
+admin.site.register(DjangoJobExecution, KhojDjangoJobExecutionAdmin)
 
 
-class KhojUserAdmin(UserAdmin):
+class GroupAdmin(BaseGroupAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+class UserAdmin(BaseUserAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+class KhojUserAdmin(UserAdmin, unfold_admin.ModelAdmin):
     class DateJoinedAfterFilter(admin.SimpleListFilter):
         title = "Joined after"
         parameter_name = "joined_after"
@@ -137,21 +153,22 @@ class KhojUserAdmin(UserAdmin):
     get_email_login_url.short_description = "Get email login URL"  # type: ignore
 
 
+admin.site.unregister(Group)
 admin.site.register(KhojUser, KhojUserAdmin)
 
-admin.site.register(ProcessLock)
-admin.site.register(SpeechToTextModelOptions)
-admin.site.register(ReflectiveQuestion)
-admin.site.register(ClientApplication)
-admin.site.register(GithubConfig)
-admin.site.register(NotionConfig)
-admin.site.register(UserVoiceModelConfig)
-admin.site.register(VoiceModelOption)
-admin.site.register(UserRequests)
+admin.site.register(ProcessLock, unfold_admin.ModelAdmin)
+admin.site.register(SpeechToTextModelOptions, unfold_admin.ModelAdmin)
+admin.site.register(ReflectiveQuestion, unfold_admin.ModelAdmin)
+admin.site.register(ClientApplication, unfold_admin.ModelAdmin)
+admin.site.register(GithubConfig, unfold_admin.ModelAdmin)
+admin.site.register(NotionConfig, unfold_admin.ModelAdmin)
+admin.site.register(UserVoiceModelConfig, unfold_admin.ModelAdmin)
+admin.site.register(VoiceModelOption, unfold_admin.ModelAdmin)
+admin.site.register(UserRequests, unfold_admin.ModelAdmin)
 
 
 @admin.register(Agent)
-class AgentAdmin(admin.ModelAdmin):
+class AgentAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
@@ -161,7 +178,7 @@ class AgentAdmin(admin.ModelAdmin):
 
 
 @admin.register(Entry)
-class EntryAdmin(admin.ModelAdmin):
+class EntryAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "created_at",
@@ -183,7 +200,7 @@ class EntryAdmin(admin.ModelAdmin):
 
 
 @admin.register(Subscription)
-class KhojUserSubscription(admin.ModelAdmin):
+class KhojUserSubscription(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "user",
@@ -195,7 +212,7 @@ class KhojUserSubscription(admin.ModelAdmin):
 
 
 @admin.register(ChatModelOptions)
-class ChatModelOptionsAdmin(admin.ModelAdmin):
+class ChatModelOptionsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "chat_model",
@@ -206,7 +223,7 @@ class ChatModelOptionsAdmin(admin.ModelAdmin):
 
 
 @admin.register(TextToImageModelConfig)
-class TextToImageModelOptionsAdmin(admin.ModelAdmin):
+class TextToImageModelOptionsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "model_name",
@@ -215,8 +232,8 @@ class TextToImageModelOptionsAdmin(admin.ModelAdmin):
     search_fields = ("id", "model_name", "model_type")
 
 
-@admin.register(OpenAIProcessorConversationConfig)
-class OpenAIProcessorConversationConfigAdmin(admin.ModelAdmin):
+@admin.register(AiModelApi)
+class AiModelApiAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
@@ -227,7 +244,7 @@ class OpenAIProcessorConversationConfigAdmin(admin.ModelAdmin):
 
 
 @admin.register(SearchModelConfig)
-class SearchModelConfigAdmin(admin.ModelAdmin):
+class SearchModelConfigAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
@@ -238,7 +255,7 @@ class SearchModelConfigAdmin(admin.ModelAdmin):
 
 
 @admin.register(ServerChatSettings)
-class ServerChatSettingsAdmin(admin.ModelAdmin):
+class ServerChatSettingsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "chat_default",
         "chat_advanced",
@@ -247,7 +264,7 @@ class ServerChatSettingsAdmin(admin.ModelAdmin):
 
 
 @admin.register(WebScraper)
-class WebScraperAdmin(admin.ModelAdmin):
+class WebScraperAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "priority",
         "name",
@@ -261,7 +278,7 @@ class WebScraperAdmin(admin.ModelAdmin):
 
 
 @admin.register(Conversation)
-class ConversationAdmin(admin.ModelAdmin):
+class ConversationAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "user",
@@ -286,17 +303,10 @@ class ConversationAdmin(admin.ModelAdmin):
             modified_log = conversation.conversation_log
             chat_log = modified_log.get("chat", [])
             for idx, log in enumerate(chat_log):
-                if (
-                    log["by"] == "khoj"
-                    and log["intent"]
-                    and log["intent"]["type"]
-                    and (
-                        log["intent"]["type"] == ImageIntentType.TEXT_TO_IMAGE.value
-                        or log["intent"]["type"] == ImageIntentType.TEXT_TO_IMAGE_V3.value
-                    )
-                ):
-                    log["message"] = "inline image redacted for space"
+                if log["by"] == "khoj" and log["images"]:
+                    log["images"] = ["inline image redacted for space"]
                     chat_log[idx] = log
+
             modified_log["chat"] = chat_log
 
             writer.writerow(
@@ -367,7 +377,7 @@ class ConversationAdmin(admin.ModelAdmin):
 
 
 @admin.register(UserConversationConfig)
-class UserConversationConfigAdmin(admin.ModelAdmin):
+class UserConversationConfigAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "get_user_email",

khoj/database/migrations/0075_migrate_generated_assets_and_validate.py

@@ -0,0 +1,85 @@
+# Made manually by sabaimran for use by Django 5.0.9 on 2024-12-01 16:59
+
+from django.db import migrations, models
+
+# This script was written alongside when Pydantic validation was added to the Conversation conversation_log field.
+
+
+def migrate_generated_assets(apps, schema_editor):
+    Conversation = apps.get_model("database", "Conversation")
+
+    # Process conversations in chunks
+    for conversation in Conversation.objects.iterator():
+        try:
+            meta_log = conversation.conversation_log
+            modified = False
+
+            for chat in meta_log.get("chat", []):
+                intent_type = chat.get("intent", {}).get("type")
+
+                if intent_type and chat["by"] == "khoj":
+                    if intent_type and "text-to-image" in intent_type:
+                        # Migrate the generated image to the new format
+                        chat["images"] = [chat.get("message")]
+                        chat["message"] = chat["intent"]["inferred-queries"][0]
+                        modified = True
+
+                    if intent_type and "excalidraw" in intent_type:
+                        # Migrate the generated excalidraw to the new format
+                        chat["excalidrawDiagram"] = chat.get("message")
+                        chat["message"] = chat["intent"]["inferred-queries"][0]
+                        modified = True
+
+            # Only save if changes were made
+            if modified:
+                conversation.conversation_log = meta_log
+                conversation.save()
+
+        except Exception as e:
+            print(f"Error processing conversation {conversation.id}: {str(e)}")
+            continue
+
+
+def reverse_migration(apps, schema_editor):
+    Conversation = apps.get_model("database", "Conversation")
+
+    # Process conversations in chunks
+    for conversation in Conversation.objects.iterator():
+        try:
+            meta_log = conversation.conversation_log
+            modified = False
+
+            for chat in meta_log.get("chat", []):
+                intent_type = chat.get("intent", {}).get("type")
+
+                if intent_type and chat["by"] == "khoj":
+                    if intent_type and "text-to-image" in intent_type:
+                        # Migrate the generated image back to the old format
+                        chat["message"] = chat.get("images", [])[0]
+                        chat.pop("images", None)
+                        modified = True
+
+                    if intent_type and "excalidraw" in intent_type:
+                        # Migrate the generated excalidraw back to the old format
+                        chat["message"] = chat.get("excalidrawDiagram")
+                        chat.pop("excalidrawDiagram", None)
+                        modified = True
+
+            # Only save if changes were made
+            if modified:
+                conversation.conversation_log = meta_log
+                conversation.save()
+
+        except Exception as e:
+            print(f"Error processing conversation {conversation.id}: {str(e)}")
+            continue
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0074_alter_conversation_title"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_generated_assets, reverse_migration),
+    ]

khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0.9 on 2024-12-05 09:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0075_migrate_generated_assets_and_validate"),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="OpenAIProcessorConversationConfig",
+            new_name="AiModelApi",
+        ),
+        migrations.RenameField(
+            model_name="chatmodeloptions",
+            old_name="openai_config",
+            new_name="ai_model_api",
+        ),
+        migrations.RenameField(
+            model_name="texttoimagemodelconfig",
+            old_name="openai_config",
+            new_name="ai_model_api",
+        ),
+    ]