khoj 1.24.2.dev3__py3-none-any.whl → 1.25.1.dev34__py3-none-any.whl

khoj/configure.py

@@ -42,6 +42,7 @@ from khoj.database.adapters import (
 from khoj.database.models import ClientApplication, KhojUser, ProcessLock, Subscription
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.routers.api_content import configure_content, configure_search
+from khoj.routers.helpers import update_telemetry_state
 from khoj.routers.twilio import is_twilio_enabled
 from khoj.utils import constants, state
 from khoj.utils.config import SearchType
@@ -165,7 +166,15 @@ class UserAuthenticationBackend(AuthenticationBackend):
 
             create_if_not_exists = request.query_params.get("create_if_not_exists")
             if create_if_not_exists:
-                user = await aget_or_create_user_by_phone_number(phone_number)
+                user, is_new = await aget_or_create_user_by_phone_number(phone_number)
+                if user and is_new:
+                    update_telemetry_state(
+                        request=request,
+                        telemetry_type="api",
+                        api="create_user",
+                        metadata={"user_id": str(user.uuid)},
+                    )
+                    logger.log(logging.INFO, f"🥳 New User Created: {user.uuid}")
             else:
                 user = await aget_user_by_phone_number(phone_number)
 
@@ -244,7 +253,7 @@ def configure_server(
 
         state.SearchType = configure_search_types()
         state.search_models = configure_search(state.search_models, state.config.search_type)
-        setup_default_agent()
+        setup_default_agent(user)
 
         message = "📡 Telemetry disabled" if telemetry_disabled(state.config.app) else "📡 Telemetry enabled"
         logger.info(message)
@@ -256,8 +265,8 @@ def configure_server(
         raise e
 
 
-def setup_default_agent():
-    AgentAdapters.create_default_agent()
+def setup_default_agent(user: KhojUser):
+    AgentAdapters.create_default_agent(user)
 
 
 def initialize_content(regenerate: bool, search_type: Optional[SearchType] = None, user: KhojUser = None):

khoj/database/adapters/__init__.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import math
+import os
 import random
 import re
 import secrets
@@ -51,6 +52,7 @@ from khoj.database.models import (
     UserTextToImageModelConfig,
     UserVoiceModelConfig,
     VoiceModelOption,
+    WebScraper,
 )
 from khoj.processor.conversation import prompts
 from khoj.search_filter.date_filter import DateFilter
@@ -58,7 +60,12 @@ from khoj.search_filter.file_filter import FileFilter
 from khoj.search_filter.word_filter import WordFilter
 from khoj.utils import state
 from khoj.utils.config import OfflineChatProcessorModel
-from khoj.utils.helpers import generate_random_name, is_none_or_empty, timer
+from khoj.utils.helpers import (
+    generate_random_name,
+    in_debug_mode,
+    is_none_or_empty,
+    timer,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -112,13 +119,15 @@ async def get_or_create_user(token: dict) -> KhojUser:
     return user
 
 
-async def aget_or_create_user_by_phone_number(phone_number: str) -> KhojUser:
+async def aget_or_create_user_by_phone_number(phone_number: str) -> tuple[KhojUser, bool]:
+    is_new = False
     if is_none_or_empty(phone_number):
-        return None
+        return None, is_new
     user = await aget_user_by_phone_number(phone_number)
     if not user:
         user = await acreate_user_by_phone_number(phone_number)
-    return user
+        is_new = True
+    return user, is_new
 
 
 async def aset_user_phone_number(user: KhojUser, phone_number: str) -> KhojUser:
@@ -164,8 +173,10 @@ async def acreate_user_by_phone_number(phone_number: str) -> KhojUser:
     return user
 
 
-async def aget_or_create_user_by_email(email: str) -> KhojUser:
-    user, _ = await KhojUser.objects.filter(email=email).aupdate_or_create(defaults={"username": email, "email": email})
+async def aget_or_create_user_by_email(email: str) -> tuple[KhojUser, bool]:
+    user, is_new = await KhojUser.objects.filter(email=email).aupdate_or_create(
+        defaults={"username": email, "email": email}
+    )
     await user.asave()
 
     if user:
@@ -176,7 +187,7 @@ async def aget_or_create_user_by_email(email: str) -> KhojUser:
     if not user_subscription:
         await Subscription.objects.acreate(user=user, type="trial")
 
-    return user
+    return user, is_new
 
 
 async def aget_user_validated_by_email_verification_code(code: str) -> KhojUser:
@@ -247,9 +258,9 @@ def get_user_subscription(email: str) -> Optional[Subscription]:
 
 async def set_user_subscription(
     email: str, is_recurring=None, renewal_date=None, type="standard"
-) -> Optional[Subscription]:
+) -> tuple[Optional[Subscription], bool]:
     # Get or create the user object and their subscription
-    user = await aget_or_create_user_by_email(email)
+    user, is_new = await aget_or_create_user_by_email(email)
     user_subscription = await Subscription.objects.filter(user=user).afirst()
 
     # Update the user subscription state
@@ -261,7 +272,7 @@ async def set_user_subscription(
     elif renewal_date is not None:
         user_subscription.renewal_date = renewal_date
     await user_subscription.asave()
-    return user_subscription
+    return user_subscription, is_new
 
 
 def subscription_to_state(subscription: Subscription) -> str:
@@ -551,26 +562,78 @@ class ClientApplicationAdapters:
 
 class AgentAdapters:
     DEFAULT_AGENT_NAME = "Khoj"
-    DEFAULT_AGENT_AVATAR = "https://assets.khoj.dev/lamp-128.png"
     DEFAULT_AGENT_SLUG = "khoj"
 
+    @staticmethod
+    async def aget_readonly_agent_by_slug(agent_slug: str, user: KhojUser):
+        return (
+            await Agent.objects.filter(
+                (Q(slug__iexact=agent_slug.lower()))
+                & (
+                    Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
+                    | Q(privacy_level=Agent.PrivacyLevel.PROTECTED)
+                    | Q(creator=user)
+                )
+            )
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
+
+    @staticmethod
+    async def adelete_agent_by_slug(agent_slug: str, user: KhojUser):
+        agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
+
+        async for entry in Entry.objects.filter(agent=agent).aiterator():
+            await entry.adelete()
+
+        if agent:
+            await agent.adelete()
+            return True
+        return False
+
     @staticmethod
     async def aget_agent_by_slug(agent_slug: str, user: KhojUser):
-        return await Agent.objects.filter(
-            (Q(slug__iexact=agent_slug.lower())) & (Q(public=True) | Q(creator=user))
-        ).afirst()
+        return (
+            await Agent.objects.filter(
+                (Q(slug__iexact=agent_slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
+            )
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
+
+    @staticmethod
+    async def aget_agent_by_name(agent_name: str, user: KhojUser):
+        return (
+            await Agent.objects.filter(
+                (Q(name__iexact=agent_name.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
+            )
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+            .afirst()
+        )
 
     @staticmethod
     def get_agent_by_slug(slug: str, user: KhojUser = None):
         if user:
-            return Agent.objects.filter((Q(slug__iexact=slug.lower())) & (Q(public=True) | Q(creator=user))).first()
-        return Agent.objects.filter(slug__iexact=slug.lower(), public=True).first()
+            return Agent.objects.filter(
+                (Q(slug__iexact=slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
+            ).first()
+        return Agent.objects.filter(slug__iexact=slug.lower(), privacy_level=Agent.PrivacyLevel.PUBLIC).first()
 
     @staticmethod
     def get_all_accessible_agents(user: KhojUser = None):
+        public_query = Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
         if user:
-            return Agent.objects.filter(Q(public=True) | Q(creator=user)).distinct().order_by("created_at")
-        return Agent.objects.filter(public=True).order_by("created_at")
+            return (
+                Agent.objects.filter(public_query | Q(creator=user))
+                .distinct()
+                .order_by("created_at")
+                .prefetch_related("creator", "chat_model", "fileobject_set")
+            )
+        return (
+            Agent.objects.filter(public_query)
+            .order_by("created_at")
+            .prefetch_related("creator", "chat_model", "fileobject_set")
+        )
 
     @staticmethod
     async def aget_all_accessible_agents(user: KhojUser = None) -> List[Agent]:
@@ -590,8 +653,8 @@ class AgentAdapters:
         return Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).first()
 
     @staticmethod
-    def create_default_agent():
-        default_conversation_config = ConversationAdapters.get_default_conversation_config()
+    def create_default_agent(user: KhojUser):
+        default_conversation_config = ConversationAdapters.get_default_conversation_config(user)
         if default_conversation_config is None:
             logger.info("No default conversation config found, skipping default agent creation")
             return None
@@ -604,17 +667,19 @@ class AgentAdapters:
             agent.chat_model = default_conversation_config
             agent.slug = AgentAdapters.DEFAULT_AGENT_SLUG
             agent.name = AgentAdapters.DEFAULT_AGENT_NAME
+            agent.privacy_level = Agent.PrivacyLevel.PUBLIC
+            agent.managed_by_admin = True
+            agent.input_tools = []
+            agent.output_modes = []
             agent.save()
         else:
             # The default agent is public and managed by the admin. It's handled a little differently than other agents.
             agent = Agent.objects.create(
                 name=AgentAdapters.DEFAULT_AGENT_NAME,
-                public=True,
+                privacy_level=Agent.PrivacyLevel.PUBLIC,
                 managed_by_admin=True,
                 chat_model=default_conversation_config,
                 personality=default_personality,
-                tools=["*"],
-                avatar=AgentAdapters.DEFAULT_AGENT_AVATAR,
                 slug=AgentAdapters.DEFAULT_AGENT_SLUG,
             )
             Conversation.objects.filter(agent=None).update(agent=agent)
@@ -625,6 +690,70 @@ class AgentAdapters:
     async def aget_default_agent():
         return await Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).afirst()
 
+    @staticmethod
+    async def aupdate_agent(
+        user: KhojUser,
+        name: str,
+        personality: str,
+        privacy_level: str,
+        icon: str,
+        color: str,
+        chat_model: str,
+        files: List[str],
+        input_tools: List[str],
+        output_modes: List[str],
+        slug: Optional[str] = None,
+    ):
+        chat_model_option = await ChatModelOptions.objects.filter(chat_model=chat_model).afirst()
+
+        # Slug will be None for new agents, which will trigger a new agent creation with a generated, immutable slug
+        agent, created = await Agent.objects.filter(slug=slug, creator=user).aupdate_or_create(
+            defaults={
+                "name": name,
+                "creator": user,
+                "personality": personality,
+                "privacy_level": privacy_level,
+                "style_icon": icon,
+                "style_color": color,
+                "chat_model": chat_model_option,
+                "input_tools": input_tools,
+                "output_modes": output_modes,
+            }
+        )
+
+        # Delete all existing files and entries
+        await FileObject.objects.filter(agent=agent).adelete()
+        await Entry.objects.filter(agent=agent).adelete()
+
+        for file in files:
+            reference_file = await FileObject.objects.filter(file_name=file, user=agent.creator).afirst()
+            if reference_file:
+                await FileObject.objects.acreate(file_name=file, agent=agent, raw_text=reference_file.raw_text)
+
+                # Duplicate all entries associated with the file
+                entries: List[Entry] = []
+                async for entry in Entry.objects.filter(file_path=file, user=agent.creator).aiterator():
+                    entries.append(
+                        Entry(
+                            agent=agent,
+                            embeddings=entry.embeddings,
+                            raw=entry.raw,
+                            compiled=entry.compiled,
+                            heading=entry.heading,
+                            file_source=entry.file_source,
+                            file_type=entry.file_type,
+                            file_path=entry.file_path,
+                            file_name=entry.file_name,
+                            url=entry.url,
+                            hashed_value=entry.hashed_value,
+                        )
+                    )
+
+                # Bulk create entries
+                await Entry.objects.abulk_create(entries)
+
+        return agent
+
 
 class PublicConversationAdapters:
     @staticmethod
@@ -709,7 +838,7 @@ class ConversationAdapters:
         user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
     ):
         if agent_slug:
-            agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
+            agent = await AgentAdapters.aget_readonly_agent_by_slug(agent_slug, user)
             if agent is None:
                 raise HTTPException(status_code=400, detail="No such agent currently exists.")
             return await Conversation.objects.acreate(user=user, client=client_application, agent=agent, title=title)
@@ -721,7 +850,7 @@ class ConversationAdapters:
         user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
     ):
         if agent_slug:
-            agent = AgentAdapters.get_agent_by_slug(agent_slug, user)
+            agent = AgentAdapters.aget_readonly_agent_by_slug(agent_slug, user)
             if agent is None:
                 raise HTTPException(status_code=400, detail="No such agent currently exists.")
             return Conversation.objects.create(user=user, client=client_application, agent=agent, title=title)
@@ -816,21 +945,21 @@ class ConversationAdapters:
     def get_conversation_config(user: KhojUser):
         subscribed = is_user_subscribed(user)
         if not subscribed:
-            return ConversationAdapters.get_default_conversation_config()
+            return ConversationAdapters.get_default_conversation_config(user)
         config = UserConversationConfig.objects.filter(user=user).first()
         if config:
             return config.setting
-        return ConversationAdapters.get_advanced_conversation_config()
+        return ConversationAdapters.get_advanced_conversation_config(user)
 
     @staticmethod
     async def aget_conversation_config(user: KhojUser):
         subscribed = await ais_user_subscribed(user)
         if not subscribed:
-            return await ConversationAdapters.aget_default_conversation_config()
+            return await ConversationAdapters.aget_default_conversation_config(user)
         config = await UserConversationConfig.objects.filter(user=user).prefetch_related("setting").afirst()
         if config:
             return config.setting
-        return ConversationAdapters.aget_advanced_conversation_config()
+        return ConversationAdapters.aget_advanced_conversation_config(user)
 
     @staticmethod
     async def aget_voice_model_config(user: KhojUser) -> Optional[VoiceModelOption]:
@@ -851,40 +980,126 @@ class ConversationAdapters:
         return VoiceModelOption.objects.first()
 
     @staticmethod
-    def get_default_conversation_config():
+    def get_default_conversation_config(user: KhojUser = None):
+        """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
+        # Get the server chat settings
         server_chat_settings = ServerChatSettings.objects.first()
-        if server_chat_settings is None or server_chat_settings.chat_default is None:
-            return ChatModelOptions.objects.filter().first()
-        return server_chat_settings.chat_default
+        if server_chat_settings is not None and server_chat_settings.chat_default is not None:
+            return server_chat_settings.chat_default
+
+        # Get the user's chat settings, if the server chat settings are not set
+        user_chat_settings = UserConversationConfig.objects.filter(user=user).first() if user else None
+        if user_chat_settings is not None and user_chat_settings.setting is not None:
+            return user_chat_settings.setting
+
+        # Get the first chat model if even the user chat settings are not set
+        return ChatModelOptions.objects.filter().first()
 
     @staticmethod
-    async def aget_default_conversation_config():
+    async def aget_default_conversation_config(user: KhojUser = None):
+        """Get default conversation config. Prefer chat model by server admin > user > first created chat model"""
+        # Get the server chat settings
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related("chat_default", "chat_default__openai_config")
             .afirst()
         )
-        if server_chat_settings is None or server_chat_settings.chat_default is None:
-            return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
-        return server_chat_settings.chat_default
+        if server_chat_settings is not None and server_chat_settings.chat_default is not None:
+            return server_chat_settings.chat_default
+
+        # Get the user's chat settings, if the server chat settings are not set
+        user_chat_settings = (
+            (await UserConversationConfig.objects.filter(user=user).prefetch_related("setting__openai_config").afirst())
+            if user
+            else None
+        )
+        if user_chat_settings is not None and user_chat_settings.setting is not None:
+            return user_chat_settings.setting
+
+        # Get the first chat model if even the user chat settings are not set
+        return await ChatModelOptions.objects.filter().prefetch_related("openai_config").afirst()
 
     @staticmethod
-    def get_advanced_conversation_config():
+    def get_advanced_conversation_config(user: KhojUser):
         server_chat_settings = ServerChatSettings.objects.first()
-        if server_chat_settings is None or server_chat_settings.chat_advanced is None:
-            return ConversationAdapters.get_default_conversation_config()
-        return server_chat_settings.chat_advanced
+        if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
+            return server_chat_settings.chat_advanced
+        return ConversationAdapters.get_default_conversation_config(user)
 
     @staticmethod
-    async def aget_advanced_conversation_config():
+    async def aget_advanced_conversation_config(user: KhojUser = None):
         server_chat_settings: ServerChatSettings = (
             await ServerChatSettings.objects.filter()
             .prefetch_related("chat_advanced", "chat_advanced__openai_config")
             .afirst()
         )
-        if server_chat_settings is None or server_chat_settings.chat_advanced is None:
-            return await ConversationAdapters.aget_default_conversation_config()
-        return server_chat_settings.chat_advanced
+        if server_chat_settings is not None and server_chat_settings.chat_advanced is not None:
+            return server_chat_settings.chat_advanced
+        return await ConversationAdapters.aget_default_conversation_config(user)
+
+    @staticmethod
+    async def aget_server_webscraper():
+        server_chat_settings = await ServerChatSettings.objects.filter().prefetch_related("web_scraper").afirst()
+        if server_chat_settings is not None and server_chat_settings.web_scraper is not None:
+            return server_chat_settings.web_scraper
+        return None
+
+    @staticmethod
+    async def aget_enabled_webscrapers() -> list[WebScraper]:
+        enabled_scrapers: list[WebScraper] = []
+        server_webscraper = await ConversationAdapters.aget_server_webscraper()
+        if server_webscraper:
+            # Only use the webscraper set in the server chat settings
+            enabled_scrapers = [server_webscraper]
+        if not enabled_scrapers:
+            # Use the enabled web scrapers, ordered by priority, until get web page content
+            enabled_scrapers = [scraper async for scraper in WebScraper.objects.all().order_by("priority").aiterator()]
+        if not enabled_scrapers:
+            # Use scrapers enabled via environment variables
+            if os.getenv("FIRECRAWL_API_KEY"):
+                api_url = os.getenv("FIRECRAWL_API_URL", "https://api.firecrawl.dev")
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.FIRECRAWL,
+                        name=WebScraper.WebScraperType.FIRECRAWL.capitalize(),
+                        api_key=os.getenv("FIRECRAWL_API_KEY"),
+                        api_url=api_url,
+                    )
+                )
+            if os.getenv("OLOSTEP_API_KEY"):
+                api_url = os.getenv("OLOSTEP_API_URL", "https://agent.olostep.com/olostep-p2p-incomingAPI")
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.OLOSTEP,
+                        name=WebScraper.WebScraperType.OLOSTEP.capitalize(),
+                        api_key=os.getenv("OLOSTEP_API_KEY"),
+                        api_url=api_url,
+                    )
+                )
+            # Jina is the default fallback scrapers to use as it does not require an API key
+            api_url = os.getenv("JINA_READER_API_URL", "https://r.jina.ai/")
+            enabled_scrapers.append(
+                WebScraper(
+                    type=WebScraper.WebScraperType.JINA,
+                    name=WebScraper.WebScraperType.JINA.capitalize(),
+                    api_key=os.getenv("JINA_API_KEY"),
+                    api_url=api_url,
+                )
+            )
+
+            # Only enable the direct web page scraper by default in self-hosted single user setups.
+            # Useful for reading webpages on your intranet.
+            if state.anonymous_mode or in_debug_mode():
+                enabled_scrapers.append(
+                    WebScraper(
+                        type=WebScraper.WebScraperType.DIRECT,
+                        name=WebScraper.WebScraperType.DIRECT.capitalize(),
+                        api_key=None,
+                        api_url=None,
+                    )
+                )
+
+        return enabled_scrapers
 
     @staticmethod
     def create_conversation_from_public_conversation(
@@ -1113,8 +1328,8 @@ class FileObjectAdapters:
         return await FileObject.objects.acreate(user=user, file_name=file_name, raw_text=raw_text)
 
     @staticmethod
-    async def async_get_file_objects_by_name(user: KhojUser, file_name: str):
-        return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name=file_name))
+    async def async_get_file_objects_by_name(user: KhojUser, file_name: str, agent: Agent = None):
+        return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name=file_name, agent=agent))
 
     @staticmethod
     async def async_get_all_file_objects(user: KhojUser):
@@ -1196,10 +1411,18 @@ class EntryAdapters:
     def user_has_entries(user: KhojUser):
         return Entry.objects.filter(user=user).exists()
 
+    @staticmethod
+    def agent_has_entries(agent: Agent):
+        return Entry.objects.filter(agent=agent).exists()
+
     @staticmethod
     async def auser_has_entries(user: KhojUser):
         return await Entry.objects.filter(user=user).aexists()
 
+    @staticmethod
+    async def aagent_has_entries(agent: Agent):
+        return await Entry.objects.filter(agent=agent).aexists()
+
     @staticmethod
     async def adelete_entry_by_file(user: KhojUser, file_path: str):
         return await Entry.objects.filter(user=user, file_path=file_path).adelete()
@@ -1214,6 +1437,10 @@ class EntryAdapters:
 
         return deleted_count
 
+    @staticmethod
+    async def aget_agent_entry_filepaths(agent: Agent):
+        return await sync_to_async(list)(Entry.objects.filter(agent=agent).values_list("file_path", flat=True))
+
     @staticmethod
     def get_all_filenames_by_source(user: KhojUser, file_source: str):
         return (
@@ -1229,15 +1456,19 @@ class EntryAdapters:
         return total_size / 1024 / 1024
 
     @staticmethod
-    def apply_filters(user: KhojUser, query: str, file_type_filter: str = None):
+    def apply_filters(user: KhojUser, query: str, file_type_filter: str = None, agent: Agent = None):
         q_filter_terms = Q()
 
         word_filters = EntryAdapters.word_filter.get_filter_terms(query)
         file_filters = EntryAdapters.file_filter.get_filter_terms(query)
         date_filters = EntryAdapters.date_filter.get_query_date_range(query)
 
+        user_or_agent = Q(user=user)
+        if agent != None:
+            user_or_agent |= Q(agent=agent)
+
         if len(word_filters) == 0 and len(file_filters) == 0 and len(date_filters) == 0:
-            return Entry.objects.filter(user=user)
+            return Entry.objects.filter(user_or_agent)
 
         for term in word_filters:
             if term.startswith("+"):
@@ -1273,7 +1504,7 @@ class EntryAdapters:
                 formatted_max_date = date.fromtimestamp(max_date).strftime("%Y-%m-%d")
                 q_filter_terms &= Q(embeddings_dates__date__lte=formatted_max_date)
 
-        relevant_entries = Entry.objects.filter(user=user).filter(q_filter_terms)
+        relevant_entries = Entry.objects.filter(user_or_agent).filter(q_filter_terms)
         if file_type_filter:
             relevant_entries = relevant_entries.filter(file_type=file_type_filter)
         return relevant_entries
@@ -1286,9 +1517,15 @@ class EntryAdapters:
         file_type_filter: str = None,
         raw_query: str = None,
         max_distance: float = math.inf,
+        agent: Agent = None,
     ):
-        relevant_entries = EntryAdapters.apply_filters(user, raw_query, file_type_filter)
-        relevant_entries = relevant_entries.filter(user=user).annotate(
+        user_or_agent = Q(user=user)
+
+        if agent != None:
+            user_or_agent |= Q(agent=agent)
+
+        relevant_entries = EntryAdapters.apply_filters(user, raw_query, file_type_filter, agent)
+        relevant_entries = relevant_entries.filter(user_or_agent).annotate(
             distance=CosineDistance("embeddings", embeddings)
         )
         relevant_entries = relevant_entries.filter(distance__lte=max_distance)

khoj/database/admin.py

@@ -27,9 +27,11 @@ from khoj.database.models import (
     Subscription,
     TextToImageModelConfig,
     UserConversationConfig,
+    UserRequests,
     UserSearchModelConfig,
     UserVoiceModelConfig,
     VoiceModelOption,
+    WebScraper,
 )
 from khoj.utils.helpers import ImageIntentType
 
@@ -68,10 +70,11 @@ class KhojUserAdmin(UserAdmin):
         "id",
         "email",
         "username",
+        "phone_number",
         "is_active",
+        "uuid",
         "is_staff",
         "is_superuser",
-        "phone_number",
     )
     search_fields = ("email", "username", "phone_number", "uuid")
     filter_horizontal = ("groups", "user_permissions")
@@ -103,6 +106,7 @@ admin.site.register(NotionConfig)
 admin.site.register(UserVoiceModelConfig)
 admin.site.register(VoiceModelOption)
 admin.site.register(UserConversationConfig)
+admin.site.register(UserRequests)
 
 
 @admin.register(Agent)
@@ -195,7 +199,22 @@ class ServerChatSettingsAdmin(admin.ModelAdmin):
     list_display = (
         "chat_default",
         "chat_advanced",
+        "web_scraper",
+    )
+
+
+@admin.register(WebScraper)
+class WebScraperAdmin(admin.ModelAdmin):
+    list_display = (
+        "priority",
+        "name",
+        "type",
+        "api_key",
+        "api_url",
+        "created_at",
     )
+    search_fields = ("name", "api_key", "api_url", "type")
+    ordering = ("priority",)
 
 
 @admin.register(Conversation)