khoj 1.24.2.dev16__py3-none-any.whl → 1.25.1.dev34__py3-none-any.whl

khoj/configure.py

@@ -42,6 +42,7 @@ from khoj.database.adapters import (
 from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.api_content import configure_content, configure_search
+from khoj.routers.helpers import update_telemetry_state
 from khoj.routers.twilio import is_twilio_enabled
 from khoj.utils import constants, state
 from khoj.utils.config import SearchType
@@ -165,7 +166,15 @@ class UserAuthenticationBackend(AuthenticationBackend):
 
             create_if_not_exists = request.query_params.get("create_if_not_exists")
             if create_if_not_exists:
-                user = await aget_or_create_user_by_phone_number(phone_number)
+                user, is_new = await aget_or_create_user_by_phone_number(phone_number)
+                if user and is_new:
+                    update_telemetry_state(
+                        request=request,
+                        telemetry_type="api",
+                        api="create_user",
+                        metadata={"user_id": str(user.uuid)},
+                    )
+                    logger.log(logging.INFO, f"🥳 New User Created: {user.uuid}")
             else:
                 user = await aget_user_by_phone_number(phone_number)
 
@@ -244,7 +253,7 @@ def configure_server(
 
         state.SearchType = configure_search_types()
         state.search_models = configure_search(state.search_models, state.config.search_type)
-        setup_default_agent()
+        setup_default_agent(user)
 
         message = "📡 Telemetry disabled" if telemetry_disabled(state.config.app) else "📡 Telemetry enabled"
         logger.info(message)
@@ -256,8 +265,8 @@ def configure_server(
         raise e
 
 
-def setup_default_agent():
-    AgentAdapters.create_default_agent()
+def setup_default_agent(user: KhojUser):
+    AgentAdapters.create_default_agent(user)
 
 
 def initialize_content(regenerate: bool, search_type: Optional[SearchType] = None, user: KhojUser = None):

khoj/database/adapters/__init__.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import math
+import os
 import random
 import re
 import secrets
@@ -10,7 +11,6 @@ from enum import Enum
 from typing import Callable, Iterable, List, Optional, Type
 
 import cron_descriptor
-import django
 from apscheduler.job import Job
 from asgiref.sync import sync_to_async
 from django.contrib.sessions.backends.db import SessionStore
@@ -52,6 +52,7 @@ from khoj.database.models import (
     UserTextToImageModelConfig,
     UserVoiceModelConfig,
     VoiceModelOption,
+    WebScraper,
 )
 from khoj.processor.conversation import prompts
 from khoj.search_filter.date_filter import DateFilter
@@ -59,7 +60,12 @@ from khoj.search_filter.file_filter import FileFilter
 from khoj.search_filter.word_filter import WordFilter
 from khoj.utils import state
 from khoj.utils.config import OfflineChatProcessorModel
-from khoj.utils.helpers import generate_random_name, is_none_or_empty, timer
+from khoj.utils.helpers import (
+    generate_random_name,
+    in_debug_mode,
+    is_none_or_empty,
+    timer,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -113,13 +119,15 @@ async def get_or_create_user(token: dict) -> KhojUser:
     return user
 
 
-async def aget_or_create_user_by_phone_number(phone_number: str) -> KhojUser:
+async def aget_or_create_user_by_phone_number(phone_number: str) -> tuple[KhojUser, bool]:
+    is_new = False
     if is_none_or_empty(phone_number):
-        return None
+        return None, is_new
     user = await aget_user_by_phone_number(phone_number)
     if not user:
         user = await acreate_user_by_phone_number(phone_number)
-    return user
+        is_new = True
+    return user, is_new
 
 
 async def aset_user_phone_number(user: KhojUser, phone_number: str) -> KhojUser:
@@ -165,8 +173,10 @@ async def acreate_user_by_phone_number(phone_number: str) -> KhojUser:
     return user
 
 
-async def aget_or_create_user_by_email(email: str) -> KhojUser:
-    user, _ = await KhojUser.objects.filter(email=email).aupdate_or_create(defaults={"username": email, "email": email})
+async def aget_or_create_user_by_email(email: str) -> tuple[KhojUser, bool]:
+    user, is_new = await KhojUser.objects.filter(email=email).aupdate_or_create(
+        defaults={"username": email, "email": email}
+    )
     await user.asave()
 
     if user:
@@ -177,7 +187,7 @@ async def aget_or_create_user_by_email(email: str) -> KhojUser:
     if not user_subscription:
         await Subscription.objects.acreate(user=user, type="trial")
 
-    return user
+    return user, is_new
 
 
 async def aget_user_validated_by_email_verification_code(code: str) -> KhojUser:
@@ -248,9 +258,9 @@ def get_user_subscription(email: str) -> Optional[Subscription]:
 
 async def set_user_subscription(
     email: str, is_recurring=None, renewal_date=None, type="standard"
-) -> Optional[Subscription]:
+) -> tuple[Optional[Subscription], bool]:
     # Get or create the user object and their subscription
-    user = await aget_or_create_user_by_email(email)
+    user, is_new = await aget_or_create_user_by_email(email)
     user_subscription = await Subscription.objects.filter(user=user).afirst()
 
     # Update the user subscription state
@@ -262,7 +272,7 @@ async def set_user_subscription(
     elif renewal_date is not None:
         user_subscription.renewal_date = renewal_date
     await user_subscription.asave()
-    return user_subscription
+    return user_subscription, is_new
 
 
 def subscription_to_state(subscription: Subscription) -> str:
@@ -556,18 +566,26 @@ class AgentAdapters:
 
     @staticmethod
     async def aget_readonly_agent_by_slug(agent_slug: str, user: KhojUser):
-        return await Agent.objects.filter(
-            (Q(slug__iexact=agent_slug.lower()))
-            & (
-                Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
-                | Q(privacy_level=Agent.PrivacyLevel.PROTECTED)
-                | Q(creator=user)
+        return (
+            await Agent.objects.filter(
+                (Q(slug__iexact=agent_slug.lower()))
+                & (
+                    Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
+                    | Q(privacy_level=Agent.PrivacyLevel.PROTECTED)
+                    | Q(creator=user)
+                )
             )
-        ).afirst()
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
 
     @staticmethod
     async def adelete_agent_by_slug(agent_slug: str, user: KhojUser):
         agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
+
+        async for entry in Entry.objects.filter(agent=agent).aiterator():
+            await entry.adelete()
+
         if agent:
             await agent.adelete()
             return True
@@ -575,15 +593,23 @@ class AgentAdapters:
 
     @staticmethod
     async def aget_agent_by_slug(agent_slug: str, user: KhojUser):
-        return await Agent.objects.filter(
-            (Q(slug__iexact=agent_slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
-        ).afirst()
+        return (
+            await Agent.objects.filter(
+                (Q(slug__iexact=agent_slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
+            )
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
 
     @staticmethod
     async def aget_agent_by_name(agent_name: str, user: KhojUser):
-        return await Agent.objects.filter(
-            (Q(name__iexact=agent_name.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
-        ).afirst()
+        return (
+            await Agent.objects.filter(
+                (Q(name__iexact=agent_name.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
+            )
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
 
     @staticmethod
     def get_agent_by_slug(slug: str, user: KhojUser = None):
@@ -627,8 +653,8 @@ class AgentAdapters:
         return Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).first()
 
     @staticmethod
-    def create_default_agent():
-        default_conversation_config = ConversationAdapters.get_default_conversation_config()
+    def create_default_agent(user: KhojUser):
+        default_conversation_config = ConversationAdapters.get_default_conversation_config(user)
         if default_conversation_config is None:
             logger.info("No default conversation config found, skipping default agent creation")
             return None
@@ -676,10 +702,12 @@ class AgentAdapters:
         files: List[str],
         input_tools: List[str],
         output_modes: List[str],
+        slug: Optional[str] = None,
     ):
         chat_model_option = await ChatModelOptions.objects.filter(chat_model=chat_model).afirst()
 
-        agent, created = await Agent.objects.filter(name=name, creator=user).aupdate_or_create(
+        # Slug will be None for new agents, which will trigger a new agent creation with a generated, immutable slug
+        agent, created = await Agent.objects.filter(slug=slug, creator=user).aupdate_or_create(
             defaults={
                 "name": name,
                 "creator": user,
@@ -810,7 +838,7 @@ class ConversationAdapters:
         user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
     ):
         if agent_slug:
-            agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
+            agent = await AgentAdapters.aget_readonly_agent_by_slug(agent_slug, user)
             if agent is None:
                 raise HTTPException(status_code=400, detail="No such agent currently exists.")
             return await Conversation.objects.acreate(user=user, client=client_application, agent=agent, title=title)
@@ -822,7 +850,7 @@ class ConversationAdapters:
         user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
     ):
         if agent_slug:
-            agent = AgentAdapters.get_agent_by_slug(agent_slug, user)
+            agent = AgentAdapters.aget_readonly_agent_by_slug(agent_slug, user)
             if agent is None:
                 raise HTTPException(status_code=400, detail="No such agent currently exists.")
             return Conversation.objects.create(user=user, client=client_application, agent=agent, title=title)
@@ -917,21 +945,21 @@ class ConversationAdapters:
     def get_conversation_config(user: KhojUser):
         subscribed = is_user_subscribed(user)
         if not subscribed:
-            return ConversationAdapters.get_default_conversation_config()
+            return ConversationAdapters.get_default_conversation_config(user)
         config = UserConversationConfig.objects.filter(user=user).first()
         if config:
             return config.setting
-        return ConversationAdapters.get_advanced_conversation_config()
+        return ConversationAdapters.get_advanced_conversation_config(user)
 
     @staticmethod
     async def aget_conversation_config(user: KhojUser):
         subscribed = await ais_user_subscribed(user)
         if not subscribed:
-            return await ConversationAdapters.aget_default_conversation_config()
+            return await ConversationAdapters.aget_default_conversation_config(user)
         config = await UserConversationConfig.objects.filter(user=user).prefetch_related("setting").afirst()
         if config:
             return config.setting
-        return ConversationAdapters.aget_advanced_conversation_config()
+        return ConversationAdapters.aget_advanced_conversation_config(user)
 
     @staticmethod
     async def aget_voice_model_config(user: KhojUser) -> Optional[VoiceModelOption]:
@@ -952,40 +980,126 @@ class ConversationAdapters:
         return VoiceModelOption.objects.first()
 
     @staticmethod
-    def get_default_conversation_config():
+    def get_default_conversation_config(user: KhojUser = None):
+        """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
+        # Get the server chat settings
         server_chat_settings = ServerChatSettings.objects.first()
-        if server_chat_settings is None or server_chat_settings.chat_default is None:
-            return ChatModelOptions.objects.filter().first()
-        return server_chat_settings.chat_default
+        if server_chat_settings is not None and server_chat_settings.chat_default is not None:
+            return server_chat_settings.chat_default
+
+        # Get the user's chat settings, if the server chat settings are not set
+        user_chat_settings = UserConversationConfig.objects.filter(user=user).first() if user else None
+        if user_chat_settings is not None and user_chat_settings.setting is not None:
+            return user_chat_settings.setting
+
+        # Get the first chat model if even the user chat settings are not set
+        return ChatModelOptions.objects.filter().first()
 
     @staticmethod
-    async def aget_default_conversation_config():
+    async def aget_default_conversation_config(user: KhojUser = None):
+        """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
+        # Get the server chat settings
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related("chat_default", "chat_default__openai_config")
             .afirst()
         )
-        if server_chat_settings is None or server_chat_settings.chat_default is None:
-            return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
-        return server_chat_settings.chat_default
+        if server_chat_settings is not None and server_chat_settings.chat_default is not None:
+            return server_chat_settings.chat_default
+
+        # Get the user's chat settings, if the server chat settings are not set
+        user_chat_settings = (
+            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst())
+            if user
+            else None
+        )
+        if user_chat_settings is not None and user_chat_settings.setting is not None:
+            return user_chat_settings.setting
+
+        # Get the first chat model if even the user chat settings are not set
+        return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
 
     @staticmethod
-    def get_advanced_conversation_config():
+    def get_advanced_conversation_config(user: KhojUser):
         server_chat_settings = ServerChatSettings.objects.first()
-        if server_chat_settings is None or server_chat_settings.chat_advanced is None:
-            return ConversationAdapters.get_default_conversation_config()
-        return server_chat_settings.chat_advanced
+        if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
+            return server_chat_settings.chat_advanced
+        return ConversationAdapters.get_default_conversation_config(user)
 
     @staticmethod
-    async def aget_advanced_conversation_config():
+    async def aget_advanced_conversation_config(user: KhojUser = None):
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related("chat_advanced", "chat_advanced__openai_config")
             .afirst()
         )
-        if server_chat_settings is None or server_chat_settings.chat_advanced is None:
-            return await ConversationAdapters.aget_default_conversation_config()
-        return server_chat_settings.chat_advanced
+        if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
+            return server_chat_settings.chat_advanced
+        return await ConversationAdapters.aget_default_conversation_config(user)
+
+    @staticmethod
+    async def aget_server_webscraper():
+        server_chat_settings = await ServerChatSettings.objects.filter().prefetch_related("web_scraper").afirst()
+        if server_chat_settings is not None and server_chat_settings.web_scraper is not None:
+            return server_chat_settings.web_scraper
+        return None
+
+    @staticmethod
+    async def aget_enabled_webscrapers() -> list[WebScraper]:
+        enabled_scrapers: list[WebScraper] = []
+        server_webscraper = await ConversationAdapters.aget_server_webscraper()
+        if server_webscraper:
+            # Only use the webscraper set in the server chat settings
+            enabled_scrapers = [server_webscraper]
+        if not enabled_scrapers:
+            # Use the enabled web scrapers, ordered by priority, until get web page content
+            enabled_scrapers = [scraper async for scraper in WebScraper.objects.all().order_by("priority").aiterator()]
+        if not enabled_scrapers:
+            # Use scrapers enabled via environment variables
+            if os.getenv("FIRECRAWL_API_KEY"):
+                api_url = os.getenv("FIRECRAWL_API_URL", "https://api.firecrawl.dev")
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.FIRECRAWL,
+                        name=WebScraper.WebScraperType.FIRECRAWL.capitalize(),
+                        api_key=os.getenv("FIRECRAWL_API_KEY"),
+                        api_url=api_url,
+                    )
+                )
+            if os.getenv("OLOSTEP_API_KEY"):
+                api_url = os.getenv("OLOSTEP_API_URL", "https://agent.olostep.com/olostep-p2p-incomingAPI")
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.OLOSTEP,
+                        name=WebScraper.WebScraperType.OLOSTEP.capitalize(),
+                        api_key=os.getenv("OLOSTEP_API_KEY"),
+                        api_url=api_url,
+                    )
+                )
+            # Jina is the default fallback scrapers to use as it does not require an API key
+            api_url = os.getenv("JINA_READER_API_URL", "https://r.jina.ai/")
+            enabled_scrapers.append(
+                WebScraper(
+                    type=WebScraper.WebScraperType.JINA,
+                    name=WebScraper.WebScraperType.JINA.capitalize(),
+                    api_key=os.getenv("JINA_API_KEY"),
+                    api_url=api_url,
+                )
+            )
+
+            # Only enable the direct web page scraper by default in self-hosted single user setups.
+            # Useful for reading webpages on your intranet.
+            if state.anonymous_mode or in_debug_mode():
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.DIRECT,
+                        name=WebScraper.WebScraperType.DIRECT.capitalize(),
+                        api_key=None,
+                        api_url=None,
+                    )
+                )
+
+        return enabled_scrapers
 
     @staticmethod
     def create_conversation_from_public_conversation(

khoj/database/admin.py

@@ -31,6 +31,7 @@ from khoj.database.models import (
     UserSearchModelConfig,
     UserVoiceModelConfig,
     VoiceModelOption,
+    WebScraper,
 )
 from khoj.utils.helpers import ImageIntentType
 
@@ -69,10 +70,11 @@ class KhojUserAdmin(UserAdmin):
         "id",
         "email",
         "username",
+        "phone_number",
         "is_active",
+        "uuid",
         "is_staff",
         "is_superuser",
-        "phone_number",
     )
     search_fields = ("email", "username", "phone_number", "uuid")
     filter_horizontal = ("groups", "user_permissions")
@@ -197,7 +199,22 @@ class ServerChatSettingsAdmin(admin.ModelAdmin):
     list_display = (
         "chat_default",
         "chat_advanced",
+        "web_scraper",
+    )
+
+
+@admin.register(WebScraper)
+class WebScraperAdmin(admin.ModelAdmin):
+    list_display = (
+        "priority",
+        "name",
+        "type",
+        "api_key",
+        "api_url",
+        "created_at",
     )
+    search_fields = ("name", "api_key", "api_url", "type")
+    ordering = ("priority",)
 
 
 @admin.register(Conversation)

khoj/database/migrations/0068_alter_agent_output_modes.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.0.8 on 2024-10-17 18:13
+
+import django.contrib.postgres.fields
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0067_alter_agent_style_icon"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="agent",
+            name="output_modes",
+            field=django.contrib.postgres.fields.ArrayField(
+                base_field=models.CharField(
+                    choices=[("text", "Text"), ("image", "Image"), ("automation", "Automation")], max_length=200
+                ),
+                default=list,
+                size=None,
+            ),
+        ),
+    ]

khoj/database/migrations/0069_webscraper_serverchatsettings_web_scraper.py

@@ -0,0 +1,89 @@
+# Generated by Django 5.0.8 on 2024-10-18 00:41
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0068_alter_agent_output_modes"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="WebScraper",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "name",
+                    models.CharField(
+                        blank=True,
+                        default=None,
+                        help_text="Friendly name. If not set, it will be set to the type of the scraper.",
+                        max_length=200,
+                        null=True,
+                        unique=True,
+                    ),
+                ),
+                (
+                    "type",
+                    models.CharField(
+                        choices=[
+                            ("Firecrawl", "Firecrawl"),
+                            ("Olostep", "Olostep"),
+                            ("Jina", "Jina"),
+                            ("Direct", "Direct"),
+                        ],
+                        default="Jina",
+                        max_length=20,
+                    ),
+                ),
+                (
+                    "api_key",
+                    models.CharField(
+                        blank=True,
+                        default=None,
+                        help_text="API key of the web scraper. Only set if scraper service requires an API key. Default is set from env var.",
+                        max_length=200,
+                        null=True,
+                    ),
+                ),
+                (
+                    "api_url",
+                    models.URLField(
+                        blank=True,
+                        default=None,
+                        help_text="API URL of the web scraper. Only set if scraper service on non-default URL.",
+                        null=True,
+                    ),
+                ),
+                (
+                    "priority",
+                    models.IntegerField(
+                        blank=True,
+                        default=None,
+                        help_text="Priority of the web scraper. Lower numbers run first.",
+                        null=True,
+                        unique=True,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="serverchatsettings",
+            name="web_scraper",
+            field=models.ForeignKey(
+                blank=True,
+                default=None,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="web_scraper",
+                to="database.webscraper",
+            ),
+        ),
+    ]