khoj 1.30.7.dev1__tar.gz → 1.30.11.dev70__tar.gz

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/.gitignore

@@ -42,3 +42,13 @@ src/interface/obsidian/main.js
 
 # obsidian
 data.json
+
+# Android
+src/interface/android/.gradle
+src/interface/android/app/build
+src/interface/android/build
+src/interface/android/*.aab
+src/interface/android/*.apk
+src/interface/android/*.apk.idsig
+src/interface/android/*.keystore
+src/interface/android/local.properties

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.30.7.dev1
+Version: 1.30.11.dev70
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
@@ -31,7 +31,8 @@ Requires-Dist: dateparser>=1.1.1
 Requires-Dist: defusedxml==0.7.1
 Requires-Dist: django-apscheduler==0.6.2
 Requires-Dist: django-phonenumber-field==7.3.0
-Requires-Dist: django==5.0.9
+Requires-Dist: django-unfold==0.42.0
+Requires-Dist: django==5.0.10
 Requires-Dist: docx2txt==0.8
 Requires-Dist: einops==0.8.0
 Requires-Dist: fastapi>=0.110.0
@@ -55,6 +56,7 @@ Requires-Dist: pillow~=10.0.0
 Requires-Dist: psutil>=5.8.0
 Requires-Dist: psycopg2-binary==2.9.9
 Requires-Dist: pydantic[email]>=2.0.0
+Requires-Dist: pyjson5==1.6.7
 Requires-Dist: pymupdf==1.24.11
 Requires-Dist: python-multipart>=0.0.7
 Requires-Dist: pytz~=2024.1

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/pyproject.toml

@@ -63,7 +63,8 @@ dependencies = [
     "tenacity == 8.3.0",
     "anyio == 3.7.1",
     "pymupdf == 1.24.11",
-    "django == 5.0.9",
+    "django == 5.0.10",
+    "django-unfold == 0.42.0",
     "authlib == 1.2.1",
     "llama-cpp-python == 0.2.88",
     "itsdangerous == 2.1.2",
@@ -88,6 +89,7 @@ dependencies = [
     "anthropic == 0.26.1",
     "docx2txt == 0.8",
     "google-generativeai == 0.8.3",
+    "pyjson5 == 1.6.7",
 ]
 dynamic = ["version"]
 

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/src/khoj/app/settings.py

@@ -13,6 +13,8 @@ https://docs.djangoproject.com/en/4.2/ref/settings/
 import os
 from pathlib import Path
 
+from django.templatetags.static import static
+
 from khoj.utils.helpers import in_debug_mode, is_env_var_true
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -72,6 +74,7 @@ INSTALLED_APPS = [
     "django.contrib.auth",
     "django.contrib.contenttypes",
     "khoj.database.apps.DatabaseConfig",
+    "unfold",
     "django.contrib.admin",
     "django.contrib.sessions",
     "django.contrib.messages",
@@ -195,3 +198,21 @@ APSCHEDULER_DATETIME_FORMAT = "N j, Y, f:s a"
 # that supports multiple background worker processes instead (e.g. Dramatiq, Celery, Django-RQ,
 # etc. See: https://djangopackages.org/grids/g/workers-queues-tasks/ for popular options).
 APSCHEDULER_RUN_NOW_TIMEOUT = 240  # Seconds
+
+UNFOLD = {
+    "SITE_TITLE": "Khoj Admin Panel",
+    "SITE_HEADER": "Khoj Admin Panel",
+    "SITE_URL": "/",
+    "SITE_ICON": {
+        "light": lambda request: static("assets/icons/khoj_lantern_128x128.png"),
+        "dark": lambda request: static("assets/icons/khoj_lantern_128x128_dark.png"),
+    },
+    "SITE_FAVICONS": [
+        {
+            "rel": "icon",
+            "sizes": "32x32",
+            "type": "image/svg+xml",
+            "href": lambda request: static("assets/icons/khoj_lantern.svg"),
+        },
+    ],
+}

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/src/khoj/configure.py

@@ -232,9 +232,9 @@ def configure_server(
         config = FullConfig()
     state.config = config
 
-    if ConversationAdapters.has_valid_openai_conversation_config():
-        openai_config = ConversationAdapters.get_openai_conversation_config()
-        state.openai_client = openai.OpenAI(api_key=openai_config.api_key)
+    if ConversationAdapters.has_valid_ai_model_api():
+        ai_model_api = ConversationAdapters.get_ai_model_api()
+        state.openai_client = openai.OpenAI(api_key=ai_model_api.api_key)
 
     # Initialize Search Models from Config and initialize content
     try:

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/src/khoj/database/adapters/__init__.py

@@ -35,6 +35,7 @@ from torch import Tensor
 
 from khoj.database.models import (
     Agent,
+    AiModelApi,
     ChatModelOptions,
     ClientApplication,
     Conversation,
@@ -46,7 +47,6 @@ from khoj.database.models import (
     KhojApiUser,
     KhojUser,
     NotionConfig,
-    OpenAIProcessorConversationConfig,
     ProcessLock,
     PublicConversation,
     ReflectiveQuestion,
@@ -224,7 +224,9 @@ async def acreate_user_by_phone_number(phone_number: str) -> KhojUser:
     )
     await user.asave()
 
-    await Subscription.objects.acreate(user=user, type=Subscription.Type.STANDARD)
+    user_subscription = await Subscription.objects.filter(user=user).afirst()
+    if not user_subscription:
+        await Subscription.objects.acreate(user=user, type=Subscription.Type.STANDARD)
 
     return user
 
@@ -296,7 +298,9 @@ async def create_user_by_google_token(token: dict) -> KhojUser:
         user=user,
     )
 
-    await Subscription.objects.acreate(user=user, type=Subscription.Type.STANDARD)
+    user_subscription = await Subscription.objects.filter(user=user).afirst()
+    if not user_subscription:
+        await Subscription.objects.acreate(user=user, type=Subscription.Type.STANDARD)
 
     return user
 
@@ -853,7 +857,7 @@ class ConversationAdapters:
             agent=conversation.agent,
             conversation_log=conversation.conversation_log,
             slug=conversation.slug,
-            title=conversation.title,
+            title=conversation.title if conversation.title else conversation.slug,
         )
 
     @staticmethod
@@ -977,7 +981,7 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_all_conversation_configs():
-        return await sync_to_async(list)(ChatModelOptions.objects.prefetch_related("openai_config").all())
+        return await sync_to_async(list)(ChatModelOptions.objects.prefetch_related("ai_model_api").all())
 
     @staticmethod
     def get_vision_enabled_config():
@@ -996,12 +1000,12 @@ class ConversationAdapters:
         return None
 
     @staticmethod
-    def get_openai_conversation_config():
-        return OpenAIProcessorConversationConfig.objects.filter().first()
+    def get_ai_model_api():
+        return AiModelApi.objects.filter().first()
 
     @staticmethod
-    def has_valid_openai_conversation_config():
-        return OpenAIProcessorConversationConfig.objects.filter().exists()
+    def has_valid_ai_model_api():
+        return AiModelApi.objects.filter().exists()
 
     @staticmethod
     @arequire_valid_user
@@ -1089,7 +1093,7 @@ class ConversationAdapters:
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related(
-                "chat_default", "chat_default__openai_config", "chat_advanced", "chat_advanced__openai_config"
+                "chat_default", "chat_default__ai_model_api", "chat_advanced", "chat_advanced__ai_model_api"
             )
             .afirst()
         )
@@ -1105,7 +1109,7 @@ class ConversationAdapters:
 
         # Get the user's chat settings, if the server chat settings are not set
         user_chat_settings = (
-            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst())
+            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__ai_model_api").afirst())
             if user
             else None
         )
@@ -1113,7 +1117,7 @@ class ConversationAdapters:
             return user_chat_settings.setting
 
         # Get the first chat model if even the user chat settings are not set
-        return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
+        return await ChatModelOptions.objects.filter().prefetch_related("ai_model_api").afirst()
 
     @staticmethod
     def get_advanced_conversation_config(user: KhojUser):
@@ -1126,7 +1130,7 @@ class ConversationAdapters:
     async def aget_advanced_conversation_config(user: KhojUser = None):
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
-            .prefetch_related("chat_advanced", "chat_advanced__openai_config")
+            .prefetch_related("chat_advanced", "chat_advanced__ai_model_api")
             .afirst()
         )
         if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
@@ -1254,7 +1258,7 @@ class ConversationAdapters:
     @staticmethod
     async def aget_user_conversation_config(user: KhojUser):
         config = (
-            await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst()
+            await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__ai_model_api").afirst()
         )
         if not config:
             return None
@@ -1309,7 +1313,7 @@ class ConversationAdapters:
                 ChatModelOptions.ModelType.OPENAI,
                 ChatModelOptions.ModelType.GOOGLE,
             ]
-        ) and conversation_config.openai_config:
+        ) and conversation_config.ai_model_api:
             return conversation_config
 
         else:
@@ -1317,7 +1321,7 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_text_to_image_model_config():
-        return await TextToImageModelConfig.objects.filter().prefetch_related("openai_config").afirst()
+        return await TextToImageModelConfig.objects.filter().prefetch_related("ai_model_api").afirst()
 
     @staticmethod
     def get_text_to_image_model_config():
@@ -1339,9 +1343,9 @@ class ConversationAdapters:
 
     @staticmethod
     async def aget_user_text_to_image_model(user: KhojUser) -> Optional[TextToImageModelConfig]:
-        # Create a custom queryset for prefetching settings__openai_config, handling null cases
+        # Create a custom queryset for prefetching settings__ai_model_api, handling null cases
         settings_prefetch = Prefetch(
-            "setting", queryset=TextToImageModelConfig.objects.prefetch_related("openai_config")
+            "setting", queryset=TextToImageModelConfig.objects.prefetch_related("ai_model_api")
         )
 
         config = await UserTextToImageModelConfig.objects.filter(user=user).prefetch_related(settings_prefetch).afirst()

{khoj-1.30.7.dev1 → khoj-1.30.11.dev70}/src/khoj/database/admin.py

@@ -1,17 +1,21 @@
 import csv
 import json
-from datetime import date, datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
 from apscheduler.job import Job
 from django.contrib import admin, messages
-from django.contrib.auth.admin import UserAdmin
+from django.contrib.auth.admin import GroupAdmin as BaseGroupAdmin
+from django.contrib.auth.admin import UserAdmin as BaseUserAdmin
+from django.contrib.auth.models import Group
 from django.http import HttpResponse
-from django_apscheduler.admin import DjangoJobAdmin
+from django_apscheduler.admin import DjangoJobAdmin, DjangoJobExecutionAdmin
 from django_apscheduler.jobstores import DjangoJobStore
-from django_apscheduler.models import DjangoJob
+from django_apscheduler.models import DjangoJob, DjangoJobExecution
+from unfold import admin as unfold_admin
 
 from khoj.database.models import (
     Agent,
+    AiModelApi,
     ChatModelOptions,
     ClientApplication,
     Conversation,
@@ -19,7 +23,6 @@ from khoj.database.models import (
     GithubConfig,
     KhojUser,
     NotionConfig,
-    OpenAIProcessorConversationConfig,
     ProcessLock,
     ReflectiveQuestion,
     SearchModelConfig,
@@ -35,10 +38,8 @@ from khoj.database.models import (
 )
 from khoj.utils.helpers import ImageIntentType
 
-admin.site.unregister(DjangoJob)
-
 
-class KhojDjangoJobAdmin(DjangoJobAdmin):
+class KhojDjangoJobAdmin(DjangoJobAdmin, unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "next_run_time",
@@ -62,10 +63,25 @@ class KhojDjangoJobAdmin(DjangoJobAdmin):
         return queryset, use_distinct
 
 
+class KhojDjangoJobExecutionAdmin(DjangoJobExecutionAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+admin.site.unregister(DjangoJob)
 admin.site.register(DjangoJob, KhojDjangoJobAdmin)
+admin.site.unregister(DjangoJobExecution)
+admin.site.register(DjangoJobExecution, KhojDjangoJobExecutionAdmin)
 
 
-class KhojUserAdmin(UserAdmin):
+class GroupAdmin(BaseGroupAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+class UserAdmin(BaseUserAdmin, unfold_admin.ModelAdmin):
+    pass
+
+
+class KhojUserAdmin(UserAdmin, unfold_admin.ModelAdmin):
     class DateJoinedAfterFilter(admin.SimpleListFilter):
         title = "Joined after"
         parameter_name = "joined_after"
@@ -137,31 +153,33 @@ class KhojUserAdmin(UserAdmin):
     get_email_login_url.short_description = "Get email login URL"  # type: ignore
 
 
+admin.site.unregister(Group)
 admin.site.register(KhojUser, KhojUserAdmin)
 
-admin.site.register(ProcessLock)
-admin.site.register(SpeechToTextModelOptions)
-admin.site.register(ReflectiveQuestion)
-admin.site.register(ClientApplication)
-admin.site.register(GithubConfig)
-admin.site.register(NotionConfig)
-admin.site.register(UserVoiceModelConfig)
-admin.site.register(VoiceModelOption)
-admin.site.register(UserRequests)
+admin.site.register(ProcessLock, unfold_admin.ModelAdmin)
+admin.site.register(SpeechToTextModelOptions, unfold_admin.ModelAdmin)
+admin.site.register(ReflectiveQuestion, unfold_admin.ModelAdmin)
+admin.site.register(ClientApplication, unfold_admin.ModelAdmin)
+admin.site.register(GithubConfig, unfold_admin.ModelAdmin)
+admin.site.register(NotionConfig, unfold_admin.ModelAdmin)
+admin.site.register(UserVoiceModelConfig, unfold_admin.ModelAdmin)
+admin.site.register(VoiceModelOption, unfold_admin.ModelAdmin)
+admin.site.register(UserRequests, unfold_admin.ModelAdmin)
 
 
 @admin.register(Agent)
-class AgentAdmin(admin.ModelAdmin):
+class AgentAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
     )
     search_fields = ("id", "name")
+    list_filter = ("privacy_level",)
     ordering = ("-created_at",)
 
 
 @admin.register(Entry)
-class EntryAdmin(admin.ModelAdmin):
+class EntryAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "created_at",
@@ -183,7 +201,7 @@ class EntryAdmin(admin.ModelAdmin):
 
 
 @admin.register(Subscription)
-class KhojUserSubscription(admin.ModelAdmin):
+class KhojUserSubscription(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "user",
@@ -195,18 +213,18 @@ class KhojUserSubscription(admin.ModelAdmin):
 
 
 @admin.register(ChatModelOptions)
-class ChatModelOptionsAdmin(admin.ModelAdmin):
+class ChatModelOptionsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "chat_model",
-        "model_type",
+        "ai_model_api",
         "max_prompt_size",
     )
-    search_fields = ("id", "chat_model", "model_type")
+    search_fields = ("id", "chat_model", "ai_model_api__name")
 
 
 @admin.register(TextToImageModelConfig)
-class TextToImageModelOptionsAdmin(admin.ModelAdmin):
+class TextToImageModelOptionsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "model_name",
@@ -215,8 +233,8 @@ class TextToImageModelOptionsAdmin(admin.ModelAdmin):
     search_fields = ("id", "model_name", "model_type")
 
 
-@admin.register(OpenAIProcessorConversationConfig)
-class OpenAIProcessorConversationConfigAdmin(admin.ModelAdmin):
+@admin.register(AiModelApi)
+class AiModelApiAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
@@ -227,7 +245,7 @@ class OpenAIProcessorConversationConfigAdmin(admin.ModelAdmin):
 
 
 @admin.register(SearchModelConfig)
-class SearchModelConfigAdmin(admin.ModelAdmin):
+class SearchModelConfigAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "name",
@@ -238,7 +256,7 @@ class SearchModelConfigAdmin(admin.ModelAdmin):
 
 
 @admin.register(ServerChatSettings)
-class ServerChatSettingsAdmin(admin.ModelAdmin):
+class ServerChatSettingsAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "chat_default",
         "chat_advanced",
@@ -247,7 +265,7 @@ class ServerChatSettingsAdmin(admin.ModelAdmin):
 
 
 @admin.register(WebScraper)
-class WebScraperAdmin(admin.ModelAdmin):
+class WebScraperAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "priority",
         "name",
@@ -261,7 +279,7 @@ class WebScraperAdmin(admin.ModelAdmin):
 
 
 @admin.register(Conversation)
-class ConversationAdmin(admin.ModelAdmin):
+class ConversationAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "user",
@@ -270,7 +288,7 @@ class ConversationAdmin(admin.ModelAdmin):
         "client",
     )
     search_fields = ("id", "user__email", "user__username", "client__name")
-    list_filter = ("agent",)
+    list_filter = ("agent", "client", "user")
     ordering = ("-created_at",)
 
     actions = ["export_selected_objects", "export_selected_minimal_objects"]
@@ -286,17 +304,10 @@ class ConversationAdmin(admin.ModelAdmin):
             modified_log = conversation.conversation_log
             chat_log = modified_log.get("chat", [])
             for idx, log in enumerate(chat_log):
-                if (
-                    log["by"] == "khoj"
-                    and log["intent"]
-                    and log["intent"]["type"]
-                    and (
-                        log["intent"]["type"] == ImageIntentType.TEXT_TO_IMAGE.value
-                        or log["intent"]["type"] == ImageIntentType.TEXT_TO_IMAGE_V3.value
-                    )
-                ):
-                    log["message"] = "inline image redacted for space"
+                if log["by"] == "khoj" and log["images"]:
+                    log["images"] = ["inline image redacted for space"]
                     chat_log[idx] = log
+
             modified_log["chat"] = chat_log
 
             writer.writerow(
@@ -367,7 +378,7 @@ class ConversationAdmin(admin.ModelAdmin):
 
 
 @admin.register(UserConversationConfig)
-class UserConversationConfigAdmin(admin.ModelAdmin):
+class UserConversationConfigAdmin(unfold_admin.ModelAdmin):
     list_display = (
         "id",
         "get_user_email",

khoj-1.30.11.dev70/src/khoj/database/migrations/0075_migrate_generated_assets_and_validate.py

@@ -0,0 +1,85 @@
+# Made manually by sabaimran for use by Django 5.0.9 on 2024-12-01 16:59
+
+from django.db import migrations, models
+
+# This script was written alongside when Pydantic validation was added to the Conversation conversation_log field.
+
+
+def migrate_generated_assets(apps, schema_editor):
+    Conversation = apps.get_model("database", "Conversation")
+
+    # Process conversations in chunks
+    for conversation in Conversation.objects.iterator():
+        try:
+            meta_log = conversation.conversation_log
+            modified = False
+
+            for chat in meta_log.get("chat", []):
+                intent_type = chat.get("intent", {}).get("type")
+
+                if intent_type and chat["by"] == "khoj":
+                    if intent_type and "text-to-image" in intent_type:
+                        # Migrate the generated image to the new format
+                        chat["images"] = [chat.get("message")]
+                        chat["message"] = chat["intent"]["inferred-queries"][0]
+                        modified = True
+
+                    if intent_type and "excalidraw" in intent_type:
+                        # Migrate the generated excalidraw to the new format
+                        chat["excalidrawDiagram"] = chat.get("message")
+                        chat["message"] = chat["intent"]["inferred-queries"][0]
+                        modified = True
+
+            # Only save if changes were made
+            if modified:
+                conversation.conversation_log = meta_log
+                conversation.save()
+
+        except Exception as e:
+            print(f"Error processing conversation {conversation.id}: {str(e)}")
+            continue
+
+
+def reverse_migration(apps, schema_editor):
+    Conversation = apps.get_model("database", "Conversation")
+
+    # Process conversations in chunks
+    for conversation in Conversation.objects.iterator():
+        try:
+            meta_log = conversation.conversation_log
+            modified = False
+
+            for chat in meta_log.get("chat", []):
+                intent_type = chat.get("intent", {}).get("type")
+
+                if intent_type and chat["by"] == "khoj":
+                    if intent_type and "text-to-image" in intent_type:
+                        # Migrate the generated image back to the old format
+                        chat["message"] = chat.get("images", [])[0]
+                        chat.pop("images", None)
+                        modified = True
+
+                    if intent_type and "excalidraw" in intent_type:
+                        # Migrate the generated excalidraw back to the old format
+                        chat["message"] = chat.get("excalidrawDiagram")
+                        chat.pop("excalidrawDiagram", None)
+                        modified = True
+
+            # Only save if changes were made
+            if modified:
+                conversation.conversation_log = meta_log
+                conversation.save()
+
+        except Exception as e:
+            print(f"Error processing conversation {conversation.id}: {str(e)}")
+            continue
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0074_alter_conversation_title"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_generated_assets, reverse_migration),
+    ]

khoj-1.30.11.dev70/src/khoj/database/migrations/0076_rename_openaiprocessorconversationconfig_aimodelapi_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.0.9 on 2024-12-05 09:00
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0075_migrate_generated_assets_and_validate"),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="OpenAIProcessorConversationConfig",
+            new_name="AiModelApi",
+        ),
+        migrations.RenameField(
+            model_name="chatmodeloptions",
+            old_name="openai_config",
+            new_name="ai_model_api",
+        ),
+        migrations.RenameField(
+            model_name="texttoimagemodelconfig",
+            old_name="openai_config",
+            new_name="ai_model_api",
+        ),
+    ]