khoj 1.24.2.dev16__py3-none-any.whl → 1.25.1.dev34__py3-none-any.whl

khoj/routers/api_model.py

@@ -40,7 +40,7 @@ def get_user_chat_model(
     chat_model = ConversationAdapters.get_conversation_config(user)
 
     if chat_model is None:
-        chat_model = ConversationAdapters.get_default_conversation_config()
+        chat_model = ConversationAdapters.get_default_conversation_config(user)
 
     return Response(status_code=200, content=json.dumps({"id": chat_model.id, "chat_model": chat_model.chat_model}))
 

khoj/routers/auth.py

@@ -80,11 +80,19 @@ async def login_magic_link(request: Request, form: MagicLinkForm):
         request.session.pop("user", None)
 
     email = form.email
-    user = await aget_or_create_user_by_email(email)
+    user, is_new = await aget_or_create_user_by_email(email)
     unique_id = user.email_verification_code
 
     if user:
         await send_magic_link_email(email, unique_id, request.base_url)
+        if is_new:
+            update_telemetry_state(
+                request=request,
+                telemetry_type="api",
+                api="create_user",
+                metadata={"user_id": str(user.uuid)},
+            )
+            logger.log(logging.INFO, f"🥳 New User Created: {user.uuid}")
 
     return Response(status_code=200)
 

khoj/routers/helpers.py

@@ -39,6 +39,7 @@ from khoj.database.adapters import (
     AutomationAdapters,
     ConversationAdapters,
     EntryAdapters,
+    ais_user_subscribed,
     create_khoj_token,
     get_khoj_tokens,
     get_user_name,
@@ -119,20 +120,20 @@ def is_query_empty(query: str) -> bool:
     return is_none_or_empty(query.strip())
 
 
-def validate_conversation_config():
-    default_config = ConversationAdapters.get_default_conversation_config()
+def validate_conversation_config(user: KhojUser):
+    default_config = ConversationAdapters.get_default_conversation_config(user)
 
     if default_config is None:
-        raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.")
+        raise HTTPException(status_code=500, detail="Contact the server administrator to add a chat model.")
 
     if default_config.model_type == "openai" and not default_config.openai_config:
-        raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.")
+        raise HTTPException(status_code=500, detail="Contact the server administrator to add a chat model.")
 
 
 async def is_ready_to_chat(user: KhojUser):
-    user_conversation_config = (await ConversationAdapters.aget_user_conversation_config(user)) or (
-        await ConversationAdapters.aget_default_conversation_config()
-    )
+    user_conversation_config = await ConversationAdapters.aget_user_conversation_config(user)
+    if user_conversation_config == None:
+        user_conversation_config = await ConversationAdapters.aget_default_conversation_config()
 
     if user_conversation_config and user_conversation_config.model_type == ChatModelOptions.ModelType.OFFLINE:
         chat_model = user_conversation_config.chat_model
@@ -208,7 +209,7 @@ def get_next_url(request: Request) -> str:
 def construct_chat_history(conversation_history: dict, n: int = 4, agent_name="AI") -> str:
     chat_history = ""
     for chat in conversation_history.get("chat", [])[-n:]:
-        if chat["by"] == "khoj" and chat["intent"].get("type") in ["remember", "reminder"]:
+        if chat["by"] == "khoj" and chat["intent"].get("type") in ["remember", "reminder", "summarize"]:
             chat_history += f"User: {chat['intent']['query']}\n"
             chat_history += f"{agent_name}: {chat['message']}\n"
         elif chat["by"] == "khoj" and ("text-to-image" in chat["intent"].get("type")):
@@ -246,19 +247,19 @@ async def agenerate_chat_response(*args):
     return await loop.run_in_executor(executor, generate_chat_response, *args)
 
 
-async def acreate_title_from_query(query: str) -> str:
+async def acreate_title_from_query(query: str, user: KhojUser = None) -> str:
     """
     Create a title from the given query
     """
     title_generation_prompt = prompts.subject_generation.format(query=query)
 
     with timer("Chat actor: Generate title from query", logger):
-        response = await send_message_to_model_wrapper(title_generation_prompt)
+        response = await send_message_to_model_wrapper(title_generation_prompt, user=user)
 
     return response.strip()
 
 
-async def acheck_if_safe_prompt(system_prompt: str) -> Tuple[bool, str]:
+async def acheck_if_safe_prompt(system_prompt: str, user: KhojUser = None) -> Tuple[bool, str]:
     """
     Check if the system prompt is safe to use
     """
@@ -267,7 +268,7 @@ async def acheck_if_safe_prompt(system_prompt: str) -> Tuple[bool, str]:
     reason = ""
 
     with timer("Chat actor: Check if safe prompt", logger):
-        response = await send_message_to_model_wrapper(safe_prompt_check)
+        response = await send_message_to_model_wrapper(safe_prompt_check, user=user)
 
         response = response.strip()
         try:
@@ -288,7 +289,7 @@ async def aget_relevant_information_sources(
     query: str,
     conversation_history: dict,
     is_task: bool,
-    subscribed: bool,
+    user: KhojUser,
     uploaded_image_url: str = None,
     agent: Agent = None,
 ):
@@ -326,7 +327,7 @@ async def aget_relevant_information_sources(
         response = await send_message_to_model_wrapper(
             relevant_tools_prompt,
             response_type="json_object",
-            subscribed=subscribed,
+            user=user,
         )
 
     try:
@@ -348,15 +349,26 @@ async def aget_relevant_information_sources(
                 final_response.append(ConversationCommand(llm_suggested_tool))
 
         if is_none_or_empty(final_response):
-            final_response = [ConversationCommand.Default]
-        return final_response
-    except Exception as e:
+            if len(agent_tools) == 0:
+                final_response = [ConversationCommand.Default]
+            else:
+                final_response = [ConversationCommand.General]
+    except Exception:
         logger.error(f"Invalid response for determining relevant tools: {response}")
-        return [ConversationCommand.Default]
+        if len(agent_tools) == 0:
+            final_response = [ConversationCommand.Default]
+        else:
+            final_response = agent_tools
+    return final_response
 
 
 async def aget_relevant_output_modes(
-    query: str, conversation_history: dict, is_task: bool = False, uploaded_image_url: str = None, agent: Agent = None
+    query: str,
+    conversation_history: dict,
+    is_task: bool = False,
+    user: KhojUser = None,
+    uploaded_image_url: str = None,
+    agent: Agent = None,
 ):
     """
     Given a query, determine which of the available tools the agent should use in order to answer appropriately.
@@ -392,7 +404,7 @@ async def aget_relevant_output_modes(
     )
 
     with timer("Chat actor: Infer output mode for chat response", logger):
-        response = await send_message_to_model_wrapper(relevant_mode_prompt, response_type="json_object")
+        response = await send_message_to_model_wrapper(relevant_mode_prompt, response_type="json_object", user=user)
 
     try:
         response = response.strip()
@@ -447,7 +459,7 @@ async def infer_webpage_urls(
 
     with timer("Chat actor: Infer webpage urls to read", logger):
         response = await send_message_to_model_wrapper(
-            online_queries_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object"
+            online_queries_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object", user=user
         )
 
     # Validate that the response is a non-empty, JSON-serializable list of URLs
@@ -493,7 +505,7 @@ async def generate_online_subqueries(
 
     with timer("Chat actor: Generate online search subqueries", logger):
         response = await send_message_to_model_wrapper(
-            online_queries_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object"
+            online_queries_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object", user=user
         )
 
     # Validate that the response is a non-empty, JSON-serializable list
@@ -511,7 +523,9 @@ async def generate_online_subqueries(
         return [q]
 
 
-async def schedule_query(q: str, conversation_history: dict, uploaded_image_url: str = None) -> Tuple[str, ...]:
+async def schedule_query(
+    q: str, conversation_history: dict, user: KhojUser, uploaded_image_url: str = None
+) -> Tuple[str, ...]:
     """
     Schedule the date, time to run the query. Assume the server timezone is UTC.
     """
@@ -523,7 +537,7 @@ async def schedule_query(q: str, conversation_history: dict, uploaded_image_url:
     )
 
     raw_response = await send_message_to_model_wrapper(
-        crontime_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object"
+        crontime_prompt, uploaded_image_url=uploaded_image_url, response_type="json_object", user=user
     )
 
     # Validate that the response is a non-empty, JSON-serializable list
@@ -537,12 +551,14 @@ async def schedule_query(q: str, conversation_history: dict, uploaded_image_url:
         raise AssertionError(f"Invalid response for scheduling query: {raw_response}")
 
 
-async def extract_relevant_info(q: str, corpus: str, subscribed: bool, agent: Agent = None) -> Union[str, None]:
+async def extract_relevant_info(
+    qs: set[str], corpus: str, user: KhojUser = None, agent: Agent = None
+) -> Union[str, None]:
     """
     Extract relevant information for a given query from the target corpus
     """
 
-    if is_none_or_empty(corpus) or is_none_or_empty(q):
+    if is_none_or_empty(corpus) or is_none_or_empty(qs):
         return None
 
     personality_context = (
@@ -550,25 +566,26 @@ async def extract_relevant_info(q: str, corpus: str, subscribed: bool, agent: Ag
     )
 
     extract_relevant_information = prompts.extract_relevant_information.format(
-        query=q,
+        query=", ".join(qs),
         corpus=corpus.strip(),
         personality_context=personality_context,
     )
 
-    chat_model: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config()
-
-    with timer("Chat actor: Extract relevant information from data", logger):
-        response = await send_message_to_model_wrapper(
-            extract_relevant_information,
-            prompts.system_prompt_extract_relevant_information,
-            chat_model_option=chat_model,
-            subscribed=subscribed,
-        )
+    response = await send_message_to_model_wrapper(
+        extract_relevant_information,
+        prompts.system_prompt_extract_relevant_information,
+        user=user,
+    )
     return response.strip()
 
 
 async def extract_relevant_summary(
-    q: str, corpus: str, subscribed: bool = False, uploaded_image_url: str = None, agent: Agent = None
+    q: str,
+    corpus: str,
+    conversation_history: dict,
+    uploaded_image_url: str = None,
+    user: KhojUser = None,
+    agent: Agent = None,
 ) -> Union[str, None]:
     """
     Extract relevant information for a given query from the target corpus
@@ -581,20 +598,20 @@ async def extract_relevant_summary(
         prompts.personality_context.format(personality=agent.personality) if agent and agent.personality else ""
     )
 
+    chat_history = construct_chat_history(conversation_history)
+
     extract_relevant_information = prompts.extract_relevant_summary.format(
         query=q,
+        chat_history=chat_history,
         corpus=corpus.strip(),
         personality_context=personality_context,
     )
 
-    chat_model: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config()
-
     with timer("Chat actor: Extract relevant information from data", logger):
         response = await send_message_to_model_wrapper(
             extract_relevant_information,
             prompts.system_prompt_extract_relevant_summary,
-            chat_model_option=chat_model,
-            subscribed=subscribed,
+            user=user,
             uploaded_image_url=uploaded_image_url,
         )
     return response.strip()
@@ -607,8 +624,8 @@ async def generate_better_image_prompt(
     note_references: List[Dict[str, Any]],
     online_results: Optional[dict] = None,
     model_type: Optional[str] = None,
-    subscribed: bool = False,
     uploaded_image_url: Optional[str] = None,
+    user: KhojUser = None,
     agent: Agent = None,
 ) -> str:
     """
@@ -658,12 +675,8 @@ async def generate_better_image_prompt(
             personality_context=personality_context,
         )
 
-    chat_model: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config()
-
     with timer("Chat actor: Generate contextual image prompt", logger):
-        response = await send_message_to_model_wrapper(
-            image_prompt, chat_model_option=chat_model, subscribed=subscribed, uploaded_image_url=uploaded_image_url
-        )
+        response = await send_message_to_model_wrapper(image_prompt, uploaded_image_url=uploaded_image_url, user=user)
         response = response.strip()
         if response.startswith(('"', "'")) and response.endswith(('"', "'")):
             response = response[1:-1]
@@ -675,14 +688,10 @@ async def send_message_to_model_wrapper(
     message: str,
     system_message: str = "",
     response_type: str = "text",
-    chat_model_option: ChatModelOptions = None,
-    subscribed: bool = False,
+    user: KhojUser = None,
     uploaded_image_url: str = None,
 ):
-    conversation_config: ChatModelOptions = (
-        chat_model_option or await ConversationAdapters.aget_default_conversation_config()
-    )
-
+    conversation_config: ChatModelOptions = await ConversationAdapters.aget_default_conversation_config(user)
     vision_available = conversation_config.vision_enabled
     if not vision_available and uploaded_image_url:
         vision_enabled_config = await ConversationAdapters.aget_vision_enabled_config()
@@ -690,6 +699,7 @@ async def send_message_to_model_wrapper(
             conversation_config = vision_enabled_config
             vision_available = True
 
+    subscribed = await ais_user_subscribed(user)
     chat_model = conversation_config.chat_model
     max_tokens = (
         conversation_config.subscribed_max_prompt_size
@@ -788,8 +798,9 @@ def send_message_to_model_wrapper_sync(
     message: str,
     system_message: str = "",
     response_type: str = "text",
+    user: KhojUser = None,
 ):
-    conversation_config: ChatModelOptions = ConversationAdapters.get_default_conversation_config()
+    conversation_config: ChatModelOptions = ConversationAdapters.get_default_conversation_config(user)
 
     if conversation_config is None:
         raise HTTPException(status_code=500, detail="Contact the server administrator to set a default chat model.")
@@ -1168,7 +1179,7 @@ class CommonQueryParamsClass:
 CommonQueryParams = Annotated[CommonQueryParamsClass, Depends()]
 
 
-def should_notify(original_query: str, executed_query: str, ai_response: str) -> bool:
+def should_notify(original_query: str, executed_query: str, ai_response: str, user: KhojUser) -> bool:
     """
     Decide whether to notify the user of the AI response.
     Default to notifying the user for now.
@@ -1185,7 +1196,7 @@ def should_notify(original_query: str, executed_query: str, ai_response: str) ->
     with timer("Chat actor: Decide to notify user of automation response", logger):
         try:
             # TODO Replace with async call so we don't have to maintain a sync version
-            response = send_message_to_model_wrapper_sync(to_notify_or_not)
+            response = send_message_to_model_wrapper_sync(to_notify_or_not, user)
             should_notify_result = "no" not in response.lower()
             logger.info(f'Decided to {"not " if not should_notify_result else ""}notify user of automation response.')
             return should_notify_result
@@ -1277,7 +1288,9 @@ def scheduled_chat(
         ai_response = raw_response.text
 
     # Notify user if the AI response is satisfactory
-    if should_notify(original_query=scheduling_request, executed_query=cleaned_query, ai_response=ai_response):
+    if should_notify(
+        original_query=scheduling_request, executed_query=cleaned_query, ai_response=ai_response, user=user
+    ):
         if is_resend_enabled():
             send_task_email(user.get_short_name(), user.email, cleaned_query, ai_response, subject, is_image)
         else:
@@ -1287,7 +1300,7 @@ def scheduled_chat(
 async def create_automation(
     q: str, timezone: str, user: KhojUser, calling_url: URL, meta_log: dict = {}, conversation_id: str = None
 ):
-    crontime, query_to_run, subject = await schedule_query(q, meta_log)
+    crontime, query_to_run, subject = await schedule_query(q, meta_log, user)
     job = await schedule_automation(query_to_run, subject, crontime, timezone, q, user, calling_url, conversation_id)
     return job, crontime, query_to_run, subject
 
@@ -1481,9 +1494,9 @@ def get_user_config(user: KhojUser, request: Request, is_detailed: bool = False)
     current_notion_config = get_user_notion_config(user)
     notion_token = current_notion_config.token if current_notion_config else ""
 
-    selected_chat_model_config = (
-        ConversationAdapters.get_conversation_config(user) or ConversationAdapters.get_default_conversation_config()
-    )
+    selected_chat_model_config = ConversationAdapters.get_conversation_config(
+        user
+    ) or ConversationAdapters.get_default_conversation_config(user)
     chat_models = ConversationAdapters.get_conversation_processor_options().all()
     chat_model_options = list()
     for chat_model in chat_models:

khoj/routers/subscription.py

@@ -7,6 +7,7 @@ from fastapi import APIRouter, Request
 from starlette.authentication import requires
 
 from khoj.database import adapters
+from khoj.routers.helpers import update_telemetry_state
 from khoj.utils import state
 
 # Stripe integration for Khoj Cloud Subscription
@@ -48,6 +49,8 @@ async def subscribe(request: Request):
     customer_id = subscription["customer"]
     customer = stripe.Customer.retrieve(customer_id)
     customer_email = customer["email"]
+    user = None
+    is_new = False
 
     # Handle valid stripe webhook events
     success = True
@@ -55,7 +58,9 @@ async def subscribe(request: Request):
         # Mark the user as subscribed and update the next renewal date on payment
         subscription = stripe.Subscription.list(customer=customer_id).data[0]
         renewal_date = datetime.fromtimestamp(subscription["current_period_end"], tz=timezone.utc)
-        user = await adapters.set_user_subscription(customer_email, is_recurring=True, renewal_date=renewal_date)
+        user, is_new = await adapters.set_user_subscription(
+            customer_email, is_recurring=True, renewal_date=renewal_date
+        )
         success = user is not None
     elif event_type in {"customer.subscription.updated"}:
         user_subscription = await sync_to_async(adapters.get_user_subscription)(customer_email)
@@ -63,15 +68,24 @@ async def subscribe(request: Request):
         if user_subscription and user_subscription.renewal_date:
             # Mark user as unsubscribed or resubscribed
             is_recurring = not subscription["cancel_at_period_end"]
-            updated_user = await adapters.set_user_subscription(customer_email, is_recurring=is_recurring)
-            success = updated_user is not None
+            user, is_new = await adapters.set_user_subscription(customer_email, is_recurring=is_recurring)
+            success = user is not None
     elif event_type in {"customer.subscription.deleted"}:
         # Reset the user to trial state
-        user = await adapters.set_user_subscription(
+        user, is_new = await adapters.set_user_subscription(
             customer_email, is_recurring=False, renewal_date=False, type="trial"
         )
         success = user is not None
 
+    if user and is_new:
+        update_telemetry_state(
+            request=request,
+            telemetry_type="api",
+            api="create_user",
+            metadata={"user_id": str(user.user.uuid)},
+        )
+        logger.log(logging.INFO, f"🥳 New User Created: {user.user.uuid}")
+
     logger.info(f'Stripe subscription {event["type"]} for {customer_email}')
     return {"success": success}
 

khoj/search_type/text_search.py

@@ -3,6 +3,7 @@ import math
 from pathlib import Path
 from typing import List, Optional, Tuple, Type, Union
 
+import requests
 import torch
 from asgiref.sync import sync_to_async
 from sentence_transformers import util
@@ -231,8 +232,12 @@ def setup(
 
 def cross_encoder_score(query: str, hits: List[SearchResponse], search_model_name: str) -> List[SearchResponse]:
     """Score all retrieved entries using the cross-encoder"""
-    with timer("Cross-Encoder Predict Time", logger, state.device):
-        cross_scores = state.cross_encoder_model[search_model_name].predict(query, hits)
+    try:
+        with timer("Cross-Encoder Predict Time", logger, state.device):
+            cross_scores = state.cross_encoder_model[search_model_name].predict(query, hits)
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"Failed to rerank documents using the inference endpoint. Error: {e}.", exc_info=True)
+        cross_scores = [0.0] * len(hits)
 
     # Convert cross-encoder scores to distances and pass in hits for reranking
     for idx in range(len(cross_scores)):

khoj/utils/helpers.py

@@ -2,10 +2,12 @@ from __future__ import annotations  # to avoid quoting type hints
 
 import datetime
 import io
+import ipaddress
 import logging
 import os
 import platform
 import random
+import urllib.parse
 import uuid
 from collections import OrderedDict
 from enum import Enum
@@ -164,9 +166,9 @@ def get_class_by_name(name: str) -> object:
 class timer:
     """Context manager to log time taken for a block of code to run"""
 
-    def __init__(self, message: str, logger: logging.Logger, device: torch.device = None):
+    def __init__(self, message: str, logger: logging.Logger, device: torch.device = None, log_level=logging.DEBUG):
         self.message = message
-        self.logger = logger
+        self.logger = logger.debug if log_level == logging.DEBUG else logger.info
         self.device = device
 
     def __enter__(self):
@@ -176,9 +178,9 @@ class timer:
     def __exit__(self, *_):
         elapsed = perf_counter() - self.start
         if self.device is None:
-            self.logger.debug(f"{self.message}: {elapsed:.3f} seconds")
+            self.logger(f"{self.message}: {elapsed:.3f} seconds")
         else:
-            self.logger.debug(f"{self.message}: {elapsed:.3f} seconds on device: {self.device}")
+            self.logger(f"{self.message}: {elapsed:.3f} seconds on device: {self.device}")
 
 
 class LRU(OrderedDict):
@@ -321,7 +323,7 @@ command_descriptions = {
     ConversationCommand.Notes: "Only talk about information that is available in your knowledge base.",
     ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
     ConversationCommand.Online: "Search for information on the internet.",
-    ConversationCommand.Webpage: "Get information from webpage links provided by you.",
+    ConversationCommand.Webpage: "Get information from webpage suggested by you.",
     ConversationCommand.Image: "Generate images by describing your imagination in words.",
     ConversationCommand.Automation: "Automatically run your query at a specified time or interval.",
     ConversationCommand.Help: "Get help with how to use or setup Khoj from the documentation",
@@ -329,11 +331,11 @@ command_descriptions = {
 }
 
 command_descriptions_for_agent = {
-    ConversationCommand.General: "Respond without any outside information or personal knowledge.",
-    ConversationCommand.Notes: "Search through the knowledge base. Required if the agent expects context from the knowledge base.",
-    ConversationCommand.Online: "Search for the latest, up-to-date information from the internet.",
-    ConversationCommand.Webpage: "Scrape specific web pages for information.",
-    ConversationCommand.Summarize: "Retrieve an answer that depends on the entire document or a large text. Knowledge base must be a single document.",
+    ConversationCommand.General: "Agent can use the agents knowledge base and general knowledge.",
+    ConversationCommand.Notes: "Agent can search the users knowledge base for information.",
+    ConversationCommand.Online: "Agent can search the internet for information.",
+    ConversationCommand.Webpage: "Agent can read suggested web pages for information.",
+    ConversationCommand.Summarize: "Agent can read an entire document. Agents knowledge base must be a single document.",
 }
 
 tool_descriptions_for_llm = {
@@ -347,13 +349,14 @@ tool_descriptions_for_llm = {
 
 mode_descriptions_for_llm = {
     ConversationCommand.Image: "Use this if the user is requesting you to generate a picture based on their description.",
-    ConversationCommand.Automation: "Use this if the user is requesting a response at a scheduled date or time.",
+    ConversationCommand.Automation: "Use this if you are confident the user is requesting a response at a scheduled date, time and frequency",
     ConversationCommand.Text: "Use this if the other response modes don't seem to fit the query.",
 }
 
 mode_descriptions_for_agent = {
-    ConversationCommand.Image: "Allow the agent to generate images.",
-    ConversationCommand.Text: "Allow the agent to generate text.",
+    ConversationCommand.Image: "Agent can generate image in response.",
+    ConversationCommand.Automation: "Agent can schedule a task to run at a scheduled date, time and frequency in response.",
+    ConversationCommand.Text: "Agent can generate text in response.",
 }
 
 
@@ -435,6 +438,46 @@ def is_internet_connected():
         return False
 
 
+def is_internal_url(url: str) -> bool:
+    """
+    Check if a URL is likely to be internal/non-public.
+
+    Args:
+    url (str): The URL to check.
+
+    Returns:
+    bool: True if the URL is likely internal, False otherwise.
+    """
+    try:
+        parsed_url = urllib.parse.urlparse(url)
+        hostname = parsed_url.hostname
+
+        # Check for localhost
+        if hostname in ["localhost", "127.0.0.1", "::1"]:
+            return True
+
+        # Check for IP addresses in private ranges
+        try:
+            ip = ipaddress.ip_address(hostname)
+            return ip.is_private
+        except ValueError:
+            pass  # Not an IP address, continue with other checks
+
+        # Check for common internal TLDs
+        internal_tlds = [".local", ".internal", ".private", ".corp", ".home", ".lan"]
+        if any(hostname.endswith(tld) for tld in internal_tlds):
+            return True
+
+        # Check for URLs without a TLD
+        if "." not in hostname:
+            return True
+
+        return False
+    except Exception:
+        # If we can't parse the URL or something else goes wrong, assume it's not internal
+        return False
+
+
 def convert_image_to_webp(image_bytes):
     """Convert image bytes to webp format for faster loading"""
     image_io = io.BytesIO(image_bytes)

khoj/utils/initialization.py

@@ -129,9 +129,6 @@ def initialization(interactive: bool = True):
             if user_chat_model_name and ChatModelOptions.objects.filter(chat_model=user_chat_model_name).exists():
                 default_chat_model_name = user_chat_model_name
 
-            # Create a server chat settings object with the default chat model
-            default_chat_model = ChatModelOptions.objects.filter(chat_model=default_chat_model_name).first()
-            ServerChatSettings.objects.create(chat_default=default_chat_model)
             logger.info("🗣️ Chat model configuration complete")
 
         # Set up offline speech to text model