khoj 1.16.1.dev15__py3-none-any.whl

khoj/routers/api_chat.py

@@ -0,0 +1,1180 @@
+import json
+import logging
+import math
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+from urllib.parse import unquote
+
+from asgiref.sync import sync_to_async
+from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket
+from fastapi.requests import Request
+from fastapi.responses import Response, StreamingResponse
+from starlette.authentication import requires
+from starlette.websockets import WebSocketDisconnect
+from websockets import ConnectionClosedOK
+
+from khoj.app.settings import ALLOWED_HOSTS
+from khoj.database.adapters import (
+    ConversationAdapters,
+    DataStoreAdapters,
+    EntryAdapters,
+    FileObjectAdapters,
+    PublicConversationAdapters,
+    aget_user_name,
+)
+from khoj.database.models import KhojUser
+from khoj.processor.conversation.prompts import (
+    help_message,
+    no_entries_found,
+    no_notes_found,
+)
+from khoj.processor.conversation.utils import save_to_conversation_log
+from khoj.processor.speech.text_to_speech import generate_text_to_speech
+from khoj.processor.tools.online_search import read_webpages, search_online
+from khoj.routers.api import extract_references_and_questions
+from khoj.routers.helpers import (
+    ApiUserRateLimiter,
+    CommonQueryParams,
+    CommonQueryParamsClass,
+    ConversationCommandRateLimiter,
+    agenerate_chat_response,
+    aget_relevant_information_sources,
+    aget_relevant_output_modes,
+    construct_automation_created_message,
+    create_automation,
+    extract_relevant_summary,
+    get_conversation_command,
+    is_query_empty,
+    is_ready_to_chat,
+    text_to_image,
+    update_telemetry_state,
+    validate_conversation_config,
+)
+from khoj.utils import state
+from khoj.utils.helpers import (
+    AsyncIteratorWrapper,
+    ConversationCommand,
+    command_descriptions,
+    get_device,
+    is_none_or_empty,
+)
+from khoj.utils.rawconfig import FilterRequest, LocationData
+
+# Initialize Router
+logger = logging.getLogger(__name__)
+conversation_command_rate_limiter = ConversationCommandRateLimiter(
+    trial_rate_limit=2, subscribed_rate_limit=100, slug="command"
+)
+
+
+api_chat = APIRouter()
+
+from pydantic import BaseModel
+
+from khoj.routers.email import send_query_feedback
+
+
+@api_chat.get("/conversation/file-filters/{conversation_id}", response_class=Response)
+@requires(["authenticated"])
+def get_file_filter(request: Request, conversation_id: str) -> Response:
+    conversation = ConversationAdapters.get_conversation_by_user(
+        request.user.object, conversation_id=int(conversation_id)
+    )
+    if not conversation:
+        return Response(content=json.dumps({"status": "error", "message": "Conversation not found"}), status_code=404)
+
+    # get all files from "computer"
+    file_list = EntryAdapters.get_all_filenames_by_source(request.user.object, "computer")
+    file_filters = []
+    for file in conversation.file_filters:
+        if file in file_list:
+            file_filters.append(file)
+    return Response(content=json.dumps(file_filters), media_type="application/json", status_code=200)
+
+
+class FactCheckerStoreDataFormat(BaseModel):
+    factToVerify: str
+    response: str
+    references: Any
+    childReferences: List[Any]
+    runId: str
+    modelUsed: Dict[str, Any]
+
+
+class FactCheckerStoreData(BaseModel):
+    runId: str
+    storeData: FactCheckerStoreDataFormat
+
+
+@api_chat.post("/store/factchecker", response_class=Response)
+@requires(["authenticated"])
+async def store_factchecker(request: Request, common: CommonQueryParams, data: FactCheckerStoreData):
+    user = request.user.object
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="store_factchecker",
+        **common.__dict__,
+    )
+    fact_checker_key = f"factchecker_{data.runId}"
+    await DataStoreAdapters.astore_data(data.storeData.model_dump_json(), fact_checker_key, user, private=False)
+    return Response(content=json.dumps({"status": "ok"}), media_type="application/json", status_code=200)
+
+
+@api_chat.get("/store/factchecker", response_class=Response)
+async def get_factchecker(request: Request, common: CommonQueryParams, runId: str):
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="read_factchecker",
+        **common.__dict__,
+    )
+
+    fact_checker_key = f"factchecker_{runId}"
+
+    data = await DataStoreAdapters.aretrieve_public_data(fact_checker_key)
+    if data is None:
+        return Response(status_code=404)
+    return Response(content=json.dumps(data.value), media_type="application/json", status_code=200)
+
+
+@api_chat.post("/conversation/file-filters", response_class=Response)
+@requires(["authenticated"])
+def add_file_filter(request: Request, filter: FilterRequest):
+    try:
+        conversation = ConversationAdapters.get_conversation_by_user(
+            request.user.object, conversation_id=int(filter.conversation_id)
+        )
+        file_list = EntryAdapters.get_all_filenames_by_source(request.user.object, "computer")
+        if filter.filename in file_list and filter.filename not in conversation.file_filters:
+            conversation.file_filters.append(filter.filename)
+            conversation.save()
+        # remove files from conversation.file_filters that are not in file_list
+        conversation.file_filters = [file for file in conversation.file_filters if file in file_list]
+        conversation.save()
+        return Response(content=json.dumps(conversation.file_filters), media_type="application/json", status_code=200)
+    except Exception as e:
+        logger.error(f"Error adding file filter {filter.filename}: {e}", exc_info=True)
+        raise HTTPException(status_code=422, detail=str(e))
+
+
+@api_chat.delete("/conversation/file-filters", response_class=Response)
+@requires(["authenticated"])
+def remove_file_filter(request: Request, filter: FilterRequest) -> Response:
+    conversation = ConversationAdapters.get_conversation_by_user(
+        request.user.object, conversation_id=int(filter.conversation_id)
+    )
+    if filter.filename in conversation.file_filters:
+        conversation.file_filters.remove(filter.filename)
+    conversation.save()
+    # remove files from conversation.file_filters that are not in file_list
+    file_list = EntryAdapters.get_all_filenames_by_source(request.user.object, "computer")
+    conversation.file_filters = [file for file in conversation.file_filters if file in file_list]
+    conversation.save()
+    return Response(content=json.dumps(conversation.file_filters), media_type="application/json", status_code=200)
+
+
+class FeedbackData(BaseModel):
+    uquery: str
+    kquery: str
+    sentiment: str
+
+
+@api_chat.post("/feedback")
+@requires(["authenticated"])
+async def sendfeedback(request: Request, data: FeedbackData):
+    user: KhojUser = request.user.object
+    await send_query_feedback(data.uquery, data.kquery, data.sentiment, user.email)
+
+
+@api_chat.post("/speech")
+@requires(["authenticated", "premium"])
+async def text_to_speech(
+    request: Request,
+    common: CommonQueryParams,
+    text: str,
+    rate_limiter_per_minute=Depends(
+        ApiUserRateLimiter(requests=5, subscribed_requests=20, window=60, slug="chat_minute")
+    ),
+    rate_limiter_per_day=Depends(
+        ApiUserRateLimiter(requests=5, subscribed_requests=300, window=60 * 60 * 24, slug="chat_day")
+    ),
+) -> Response:
+    voice_model = await ConversationAdapters.aget_voice_model_config(request.user.object)
+
+    params = {"text_to_speak": text}
+
+    if voice_model:
+        params["voice_id"] = voice_model.model_id
+
+    speech_stream = generate_text_to_speech(**params)
+    return StreamingResponse(speech_stream.iter_content(chunk_size=1024), media_type="audio/mpeg")
+
+
+@api_chat.get("/starters", response_class=Response)
+@requires(["authenticated"])
+async def chat_starters(
+    request: Request,
+    common: CommonQueryParams,
+) -> Response:
+    user: KhojUser = request.user.object
+    starter_questions = await ConversationAdapters.aget_conversation_starters(user)
+    return Response(content=json.dumps(starter_questions), media_type="application/json", status_code=200)
+
+
+@api_chat.get("/history")
+@requires(["authenticated"])
+def chat_history(
+    request: Request,
+    common: CommonQueryParams,
+    conversation_id: Optional[int] = None,
+    n: Optional[int] = None,
+):
+    user = request.user.object
+    validate_conversation_config()
+
+    # Load Conversation History
+    conversation = ConversationAdapters.get_conversation_by_user(
+        user=user, client_application=request.user.client_app, conversation_id=conversation_id
+    )
+
+    if conversation is None:
+        return Response(
+            content=json.dumps({"status": "error", "message": f"Conversation: {conversation_id} not found"}),
+            status_code=404,
+        )
+
+    agent_metadata = None
+    if conversation.agent:
+        agent_metadata = {
+            "slug": conversation.agent.slug,
+            "name": conversation.agent.name,
+            "avatar": conversation.agent.avatar,
+            "isCreator": conversation.agent.creator == user,
+        }
+
+    meta_log = conversation.conversation_log
+    meta_log.update(
+        {
+            "conversation_id": conversation.id,
+            "slug": conversation.title if conversation.title else conversation.slug,
+            "agent": agent_metadata,
+        }
+    )
+
+    if n:
+        # Get latest N messages if N > 0
+        if n > 0 and meta_log.get("chat"):
+            meta_log["chat"] = meta_log["chat"][-n:]
+        # Else return all messages except latest N
+        elif n < 0 and meta_log.get("chat"):
+            meta_log["chat"] = meta_log["chat"][:n]
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="chat_history",
+        **common.__dict__,
+    )
+
+    return {"status": "ok", "response": meta_log}
+
+
+@api_chat.get("/share/history")
+def get_shared_chat(
+    request: Request,
+    common: CommonQueryParams,
+    public_conversation_slug: str,
+    n: Optional[int] = None,
+):
+    user = request.user.object if request.user.is_authenticated else None
+
+    # Load Conversation History
+    conversation = PublicConversationAdapters.get_public_conversation_by_slug(public_conversation_slug)
+
+    if conversation is None:
+        return Response(
+            content=json.dumps({"status": "error", "message": f"Conversation: {public_conversation_slug} not found"}),
+            status_code=404,
+        )
+
+    agent_metadata = None
+    if conversation.agent:
+        agent_metadata = {
+            "slug": conversation.agent.slug,
+            "name": conversation.agent.name,
+            "avatar": conversation.agent.avatar,
+            "isCreator": conversation.agent.creator == user,
+        }
+
+    meta_log = conversation.conversation_log
+    meta_log.update(
+        {
+            "conversation_id": conversation.id,
+            "slug": conversation.title if conversation.title else conversation.slug,
+            "agent": agent_metadata,
+        }
+    )
+
+    if n:
+        # Get latest N messages if N > 0
+        if n > 0 and meta_log.get("chat"):
+            meta_log["chat"] = meta_log["chat"][-n:]
+        # Else return all messages except latest N
+        elif n < 0 and meta_log.get("chat"):
+            meta_log["chat"] = meta_log["chat"][:n]
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="public_conversation_history",
+        **common.__dict__,
+    )
+
+    return {"status": "ok", "response": meta_log}
+
+
+@api_chat.delete("/history")
+@requires(["authenticated"])
+async def clear_chat_history(
+    request: Request,
+    common: CommonQueryParams,
+    conversation_id: Optional[int] = None,
+):
+    user = request.user.object
+
+    # Clear Conversation History
+    await ConversationAdapters.adelete_conversation_by_user(user, request.user.client_app, conversation_id)
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="clear_chat_history",
+        **common.__dict__,
+    )
+
+    return {"status": "ok", "message": "Conversation history cleared"}
+
+
+@api_chat.post("/share/fork")
+@requires(["authenticated"])
+def fork_public_conversation(
+    request: Request,
+    common: CommonQueryParams,
+    public_conversation_slug: str,
+):
+    user = request.user.object
+
+    # Load Conversation History
+    public_conversation = PublicConversationAdapters.get_public_conversation_by_slug(public_conversation_slug)
+
+    # Duplicate Public Conversation to User's Private Conversation
+    ConversationAdapters.create_conversation_from_public_conversation(
+        user, public_conversation, request.user.client_app
+    )
+
+    chat_metadata = {"forked_conversation": public_conversation.slug}
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="fork_public_conversation",
+        **common.__dict__,
+        metadata=chat_metadata,
+    )
+
+    redirect_uri = str(request.app.url_path_for("chat_page"))
+
+    return Response(status_code=200, content=json.dumps({"status": "ok", "next_url": redirect_uri}))
+
+
+@api_chat.post("/share")
+@requires(["authenticated"])
+def duplicate_chat_history_public_conversation(
+    request: Request,
+    common: CommonQueryParams,
+    conversation_id: int,
+):
+    user = request.user.object
+    domain = request.headers.get("host")
+    scheme = request.url.scheme
+
+    # Throw unauthorized exception if domain not in ALLOWED_HOSTS
+    host_domain = domain.split(":")[0]
+    if host_domain not in ALLOWED_HOSTS:
+        raise HTTPException(status_code=401, detail="Unauthorized domain")
+
+    # Duplicate Conversation History to Public Conversation
+    conversation = ConversationAdapters.get_conversation_by_user(user, request.user.client_app, conversation_id)
+    public_conversation = ConversationAdapters.make_public_conversation_copy(conversation)
+    public_conversation_url = PublicConversationAdapters.get_public_conversation_url(public_conversation)
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="post_chat_share",
+        **common.__dict__,
+    )
+
+    return Response(
+        status_code=200, content=json.dumps({"status": "ok", "url": f"{scheme}://{domain}{public_conversation_url}"})
+    )
+
+
+@api_chat.get("/sessions")
+@requires(["authenticated"])
+def chat_sessions(
+    request: Request,
+    common: CommonQueryParams,
+):
+    user = request.user.object
+
+    # Load Conversation Sessions
+    sessions = ConversationAdapters.get_conversation_sessions(user, request.user.client_app).values_list(
+        "id", "slug", "title"
+    )
+
+    session_values = [{"conversation_id": session[0], "slug": session[2] or session[1]} for session in sessions]
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="chat_sessions",
+        **common.__dict__,
+    )
+
+    return Response(content=json.dumps(session_values), media_type="application/json", status_code=200)
+
+
+@api_chat.post("/sessions")
+@requires(["authenticated"])
+async def create_chat_session(
+    request: Request,
+    common: CommonQueryParams,
+    agent_slug: Optional[str] = None,
+):
+    user = request.user.object
+
+    # Create new Conversation Session
+    conversation = await ConversationAdapters.acreate_conversation_session(user, request.user.client_app, agent_slug)
+
+    response = {"conversation_id": conversation.id}
+
+    conversation_metadata = {
+        "agent": agent_slug,
+    }
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="create_chat_sessions",
+        metadata=conversation_metadata,
+        **common.__dict__,
+    )
+
+    return Response(content=json.dumps(response), media_type="application/json", status_code=200)
+
+
+@api_chat.get("/options", response_class=Response)
+@requires(["authenticated"])
+async def chat_options(
+    request: Request,
+    common: CommonQueryParams,
+) -> Response:
+    cmd_options = {}
+    for cmd in ConversationCommand:
+        if cmd in command_descriptions:
+            cmd_options[cmd.value] = command_descriptions[cmd]
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="chat_options",
+        **common.__dict__,
+    )
+    return Response(content=json.dumps(cmd_options), media_type="application/json", status_code=200)
+
+
+@api_chat.patch("/title", response_class=Response)
+@requires(["authenticated"])
+async def set_conversation_title(
+    request: Request,
+    common: CommonQueryParams,
+    title: str,
+    conversation_id: Optional[int] = None,
+) -> Response:
+    user = request.user.object
+    title = title.strip()[:200]
+
+    # Set Conversation Title
+    conversation = await ConversationAdapters.aset_conversation_title(
+        user, request.user.client_app, conversation_id, title
+    )
+
+    success = True if conversation else False
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="set_conversation_title",
+        **common.__dict__,
+    )
+
+    return Response(
+        content=json.dumps({"status": "ok", "success": success}), media_type="application/json", status_code=200
+    )
+
+
+@api_chat.websocket("/ws")
+async def websocket_endpoint(
+    websocket: WebSocket,
+    conversation_id: int,
+    city: Optional[str] = None,
+    region: Optional[str] = None,
+    country: Optional[str] = None,
+    timezone: Optional[str] = None,
+):
+    connection_alive = True
+
+    async def send_status_update(message: str):
+        nonlocal connection_alive
+        if not connection_alive:
+            return
+
+        status_packet = {
+            "type": "status",
+            "message": message,
+            "content-type": "application/json",
+        }
+        try:
+            await websocket.send_text(json.dumps(status_packet))
+        except ConnectionClosedOK:
+            connection_alive = False
+            logger.info(f"User {user} disconnected web socket. Emitting rest of responses to clear thread")
+
+    async def send_complete_llm_response(llm_response: str):
+        nonlocal connection_alive
+        if not connection_alive:
+            return
+        try:
+            await websocket.send_text("start_llm_response")
+            await websocket.send_text(llm_response)
+            await websocket.send_text("end_llm_response")
+        except ConnectionClosedOK:
+            connection_alive = False
+            logger.info(f"User {user} disconnected web socket. Emitting rest of responses to clear thread")
+
+    async def send_message(message: str):
+        nonlocal connection_alive
+        if not connection_alive:
+            return
+        try:
+            await websocket.send_text(message)
+        except ConnectionClosedOK:
+            connection_alive = False
+            logger.info(f"User {user} disconnected web socket. Emitting rest of responses to clear thread")
+
+    async def send_rate_limit_message(message: str):
+        nonlocal connection_alive
+        if not connection_alive:
+            return
+
+        status_packet = {
+            "type": "rate_limit",
+            "message": message,
+            "content-type": "application/json",
+        }
+        try:
+            await websocket.send_text(json.dumps(status_packet))
+        except ConnectionClosedOK:
+            connection_alive = False
+            logger.info(f"User {user} disconnected web socket. Emitting rest of responses to clear thread")
+
+    user: KhojUser = websocket.user.object
+    conversation = await ConversationAdapters.aget_conversation_by_user(
+        user, client_application=websocket.user.client_app, conversation_id=conversation_id
+    )
+
+    hourly_limiter = ApiUserRateLimiter(requests=5, subscribed_requests=60, window=60, slug="chat_minute")
+
+    daily_limiter = ApiUserRateLimiter(requests=5, subscribed_requests=600, window=60 * 60 * 24, slug="chat_day")
+
+    await is_ready_to_chat(user)
+
+    user_name = await aget_user_name(user)
+
+    location = None
+
+    if city or region or country:
+        location = LocationData(city=city, region=region, country=country)
+
+    await websocket.accept()
+    while connection_alive:
+        try:
+            if conversation:
+                await sync_to_async(conversation.refresh_from_db)(fields=["conversation_log"])
+            q = await websocket.receive_text()
+
+            # Refresh these because the connection to the database might have been closed
+            await conversation.arefresh_from_db()
+
+        except WebSocketDisconnect:
+            logger.debug(f"User {user} disconnected web socket")
+            break
+
+        try:
+            await sync_to_async(hourly_limiter)(websocket)
+            await sync_to_async(daily_limiter)(websocket)
+        except HTTPException as e:
+            await send_rate_limit_message(e.detail)
+            break
+
+        if is_query_empty(q):
+            await send_message("start_llm_response")
+            await send_message(
+                "It seems like your query is incomplete. Could you please provide more details or specify what you need help with?"
+            )
+            await send_message("end_llm_response")
+            continue
+
+        user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        conversation_commands = [get_conversation_command(query=q, any_references=True)]
+
+        await send_status_update(f"**👀 Understanding Query**: {q}")
+
+        meta_log = conversation.conversation_log
+        is_automated_task = conversation_commands == [ConversationCommand.AutomatedTask]
+        used_slash_summarize = conversation_commands == [ConversationCommand.Summarize]
+
+        if conversation_commands == [ConversationCommand.Default] or is_automated_task:
+            conversation_commands = await aget_relevant_information_sources(q, meta_log, is_automated_task)
+            conversation_commands_str = ", ".join([cmd.value for cmd in conversation_commands])
+            await send_status_update(f"**🗃️ Chose Data Sources to Search:** {conversation_commands_str}")
+
+            mode = await aget_relevant_output_modes(q, meta_log, is_automated_task)
+            await send_status_update(f"**🧑🏾‍💻 Decided Response Mode:** {mode.value}")
+            if mode not in conversation_commands:
+                conversation_commands.append(mode)
+
+        for cmd in conversation_commands:
+            await conversation_command_rate_limiter.update_and_check_if_valid(websocket, cmd)
+            q = q.replace(f"/{cmd.value}", "").strip()
+
+        file_filters = conversation.file_filters if conversation else []
+        # Skip trying to summarize if
+        if (
+            # summarization intent was inferred
+            ConversationCommand.Summarize in conversation_commands
+            # and not triggered via slash command
+            and not used_slash_summarize
+            # but we can't actually summarize
+            and len(file_filters) != 1
+        ):
+            conversation_commands.remove(ConversationCommand.Summarize)
+        elif ConversationCommand.Summarize in conversation_commands:
+            response_log = ""
+            if len(file_filters) == 0:
+                response_log = "No files selected for summarization. Please add files using the section on the left."
+                await send_complete_llm_response(response_log)
+            elif len(file_filters) > 1:
+                response_log = "Only one file can be selected for summarization."
+                await send_complete_llm_response(response_log)
+            else:
+                try:
+                    file_object = await FileObjectAdapters.async_get_file_objects_by_name(user, file_filters[0])
+                    if len(file_object) == 0:
+                        response_log = "Sorry, we couldn't find the full text of this file. Please re-upload the document and try again."
+                        await send_complete_llm_response(response_log)
+                        continue
+                    contextual_data = " ".join([file.raw_text for file in file_object])
+                    if not q:
+                        q = "Create a general summary of the file"
+                    await send_status_update(f"**🧑🏾‍💻 Constructing Summary Using:** {file_object[0].file_name}")
+                    response = await extract_relevant_summary(q, contextual_data)
+                    response_log = str(response)
+                    await send_complete_llm_response(response_log)
+                except Exception as e:
+                    response_log = "Error summarizing file."
+                    logger.error(f"Error summarizing file for {user.email}: {e}", exc_info=True)
+                    await send_complete_llm_response(response_log)
+            await sync_to_async(save_to_conversation_log)(
+                q,
+                response_log,
+                user,
+                meta_log,
+                user_message_time,
+                intent_type="summarize",
+                client_application=websocket.user.client_app,
+                conversation_id=conversation_id,
+            )
+            update_telemetry_state(
+                request=websocket,
+                telemetry_type="api",
+                api="chat",
+                metadata={"conversation_command": conversation_commands[0].value},
+            )
+            continue
+
+        custom_filters = []
+        if conversation_commands == [ConversationCommand.Help]:
+            if not q:
+                conversation_config = await ConversationAdapters.aget_user_conversation_config(user)
+                if conversation_config == None:
+                    conversation_config = await ConversationAdapters.aget_default_conversation_config()
+                model_type = conversation_config.model_type
+                formatted_help = help_message.format(model=model_type, version=state.khoj_version, device=get_device())
+                await send_complete_llm_response(formatted_help)
+                continue
+            # Adding specification to search online specifically on khoj.dev pages.
+            custom_filters.append("site:khoj.dev")
+            conversation_commands.append(ConversationCommand.Online)
+
+        if ConversationCommand.Automation in conversation_commands:
+            try:
+                automation, crontime, query_to_run, subject = await create_automation(
+                    q, timezone, user, websocket.url, meta_log
+                )
+            except Exception as e:
+                logger.error(f"Error scheduling task {q} for {user.email}: {e}")
+                await send_complete_llm_response(
+                    f"Unable to create automation. Ensure the automation doesn't already exist."
+                )
+                continue
+
+            llm_response = construct_automation_created_message(automation, crontime, query_to_run, subject)
+            await sync_to_async(save_to_conversation_log)(
+                q,
+                llm_response,
+                user,
+                meta_log,
+                user_message_time,
+                intent_type="automation",
+                client_application=websocket.user.client_app,
+                conversation_id=conversation_id,
+                inferred_queries=[query_to_run],
+                automation_id=automation.id,
+            )
+            common = CommonQueryParamsClass(
+                client=websocket.user.client_app,
+                user_agent=websocket.headers.get("user-agent"),
+                host=websocket.headers.get("host"),
+            )
+            update_telemetry_state(
+                request=websocket,
+                telemetry_type="api",
+                api="chat",
+                **common.__dict__,
+            )
+            await send_complete_llm_response(llm_response)
+            continue
+
+        compiled_references, inferred_queries, defiltered_query = await extract_references_and_questions(
+            websocket, meta_log, q, 7, 0.18, conversation_id, conversation_commands, location, send_status_update
+        )
+
+        if compiled_references:
+            headings = "\n- " + "\n- ".join(set([c.get("compiled", c).split("\n")[0] for c in compiled_references]))
+            await send_status_update(f"**📜 Found Relevant Notes**: {headings}")
+
+        online_results: Dict = dict()
+
+        if conversation_commands == [ConversationCommand.Notes] and not await EntryAdapters.auser_has_entries(user):
+            await send_complete_llm_response(f"{no_entries_found.format()}")
+            continue
+
+        if ConversationCommand.Notes in conversation_commands and is_none_or_empty(compiled_references):
+            conversation_commands.remove(ConversationCommand.Notes)
+
+        if ConversationCommand.Online in conversation_commands:
+            try:
+                online_results = await search_online(
+                    defiltered_query, meta_log, location, send_status_update, custom_filters
+                )
+            except ValueError as e:
+                logger.warning(f"Error searching online: {e}. Attempting to respond without online results")
+                await send_complete_llm_response(
+                    f"Error searching online: {e}. Attempting to respond without online results"
+                )
+                continue
+
+        if ConversationCommand.Webpage in conversation_commands:
+            try:
+                direct_web_pages = await read_webpages(defiltered_query, meta_log, location, send_status_update)
+                webpages = []
+                for query in direct_web_pages:
+                    if online_results.get(query):
+                        online_results[query]["webpages"] = direct_web_pages[query]["webpages"]
+                    else:
+                        online_results[query] = {"webpages": direct_web_pages[query]["webpages"]}
+
+                    for webpage in direct_web_pages[query]["webpages"]:
+                        webpages.append(webpage["link"])
+
+                await send_status_update(f"**📚 Read web pages**: {webpages}")
+            except ValueError as e:
+                logger.warning(
+                    f"Error directly reading webpages: {e}. Attempting to respond without online results", exc_info=True
+                )
+
+        if ConversationCommand.Image in conversation_commands:
+            update_telemetry_state(
+                request=websocket,
+                telemetry_type="api",
+                api="chat",
+                metadata={"conversation_command": conversation_commands[0].value},
+            )
+            image, status_code, improved_image_prompt, intent_type = await text_to_image(
+                q,
+                user,
+                meta_log,
+                location_data=location,
+                references=compiled_references,
+                online_results=online_results,
+                send_status_func=send_status_update,
+            )
+            if image is None or status_code != 200:
+                content_obj = {
+                    "image": image,
+                    "intentType": intent_type,
+                    "detail": improved_image_prompt,
+                    "content-type": "application/json",
+                }
+                await send_complete_llm_response(json.dumps(content_obj))
+                continue
+
+            await sync_to_async(save_to_conversation_log)(
+                q,
+                image,
+                user,
+                meta_log,
+                user_message_time,
+                intent_type=intent_type,
+                inferred_queries=[improved_image_prompt],
+                client_application=websocket.user.client_app,
+                conversation_id=conversation_id,
+                compiled_references=compiled_references,
+                online_results=online_results,
+            )
+            content_obj = {"image": image, "intentType": intent_type, "inferredQueries": [improved_image_prompt], "context": compiled_references, "content-type": "application/json", "online_results": online_results}  # type: ignore
+
+            await send_complete_llm_response(json.dumps(content_obj))
+            continue
+
+        await send_status_update(f"**💭 Generating a well-informed response**")
+        llm_response, chat_metadata = await agenerate_chat_response(
+            defiltered_query,
+            meta_log,
+            conversation,
+            compiled_references,
+            online_results,
+            inferred_queries,
+            conversation_commands,
+            user,
+            websocket.user.client_app,
+            conversation_id,
+            location,
+            user_name,
+        )
+
+        chat_metadata["agent"] = conversation.agent.slug if conversation.agent else None
+
+        update_telemetry_state(
+            request=websocket,
+            telemetry_type="api",
+            api="chat",
+            metadata=chat_metadata,
+        )
+        iterator = AsyncIteratorWrapper(llm_response)
+
+        await send_message("start_llm_response")
+
+        async for item in iterator:
+            if item is None:
+                break
+            if connection_alive:
+                try:
+                    await send_message(f"{item}")
+                except ConnectionClosedOK:
+                    connection_alive = False
+                    logger.info(f"User {user} disconnected web socket. Emitting rest of responses to clear thread")
+
+        await send_message("end_llm_response")
+
+
+@api_chat.get("", response_class=Response)
+@requires(["authenticated"])
+async def chat(
+    request: Request,
+    common: CommonQueryParams,
+    q: str,
+    n: Optional[int] = 5,
+    d: Optional[float] = 0.22,
+    stream: Optional[bool] = False,
+    title: Optional[str] = None,
+    conversation_id: Optional[int] = None,
+    city: Optional[str] = None,
+    region: Optional[str] = None,
+    country: Optional[str] = None,
+    timezone: Optional[str] = None,
+    rate_limiter_per_minute=Depends(
+        ApiUserRateLimiter(requests=5, subscribed_requests=60, window=60, slug="chat_minute")
+    ),
+    rate_limiter_per_day=Depends(
+        ApiUserRateLimiter(requests=5, subscribed_requests=600, window=60 * 60 * 24, slug="chat_day")
+    ),
+) -> Response:
+    user: KhojUser = request.user.object
+    q = unquote(q)
+    if is_query_empty(q):
+        return Response(
+            content="It seems like your query is incomplete. Could you please provide more details or specify what you need help with?",
+            media_type="text/plain",
+            status_code=400,
+        )
+    user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    logger.info(f"Chat request by {user.username}: {q}")
+
+    await is_ready_to_chat(user)
+    conversation_commands = [get_conversation_command(query=q, any_references=True)]
+
+    _custom_filters = []
+    if conversation_commands == [ConversationCommand.Help]:
+        help_str = "/" + ConversationCommand.Help
+        if q.strip() == help_str:
+            conversation_config = await ConversationAdapters.aget_user_conversation_config(user)
+            if conversation_config == None:
+                conversation_config = await ConversationAdapters.aget_default_conversation_config()
+            model_type = conversation_config.model_type
+            formatted_help = help_message.format(model=model_type, version=state.khoj_version, device=get_device())
+            return StreamingResponse(iter([formatted_help]), media_type="text/event-stream", status_code=200)
+        # Adding specification to search online specifically on khoj.dev pages.
+        _custom_filters.append("site:khoj.dev")
+        conversation_commands.append(ConversationCommand.Online)
+
+    conversation = await ConversationAdapters.aget_conversation_by_user(
+        user, request.user.client_app, conversation_id, title
+    )
+    conversation_id = conversation.id if conversation else None
+
+    if not conversation:
+        return Response(
+            content=f"No conversation found with requested id, title", media_type="text/plain", status_code=400
+        )
+    else:
+        meta_log = conversation.conversation_log
+
+    if ConversationCommand.Summarize in conversation_commands:
+        file_filters = conversation.file_filters
+        llm_response = ""
+        if len(file_filters) == 0:
+            llm_response = "No files selected for summarization. Please add files using the section on the left."
+        elif len(file_filters) > 1:
+            llm_response = "Only one file can be selected for summarization."
+        else:
+            try:
+                file_object = await FileObjectAdapters.async_get_file_objects_by_name(user, file_filters[0])
+                if len(file_object) == 0:
+                    llm_response = "Sorry, we couldn't find the full text of this file. Please re-upload the document and try again."
+                    return StreamingResponse(content=llm_response, media_type="text/event-stream", status_code=200)
+                contextual_data = " ".join([file.raw_text for file in file_object])
+                summarizeStr = "/" + ConversationCommand.Summarize
+                if q.strip() == summarizeStr:
+                    q = "Create a general summary of the file"
+                response = await extract_relevant_summary(q, contextual_data)
+                llm_response = str(response)
+            except Exception as e:
+                logger.error(f"Error summarizing file for {user.email}: {e}")
+                llm_response = "Error summarizing file."
+        await sync_to_async(save_to_conversation_log)(
+            q,
+            llm_response,
+            user,
+            conversation.conversation_log,
+            user_message_time,
+            intent_type="summarize",
+            client_application=request.user.client_app,
+            conversation_id=conversation_id,
+        )
+        update_telemetry_state(
+            request=request,
+            telemetry_type="api",
+            api="chat",
+            metadata={"conversation_command": conversation_commands[0].value},
+            **common.__dict__,
+        )
+        return StreamingResponse(content=llm_response, media_type="text/event-stream", status_code=200)
+
+    is_automated_task = conversation_commands == [ConversationCommand.AutomatedTask]
+
+    if conversation_commands == [ConversationCommand.Default] or is_automated_task:
+        conversation_commands = await aget_relevant_information_sources(q, meta_log, is_automated_task)
+        mode = await aget_relevant_output_modes(q, meta_log, is_automated_task)
+        if mode not in conversation_commands:
+            conversation_commands.append(mode)
+
+    for cmd in conversation_commands:
+        await conversation_command_rate_limiter.update_and_check_if_valid(request, cmd)
+        q = q.replace(f"/{cmd.value}", "").strip()
+
+    location = None
+
+    if city or region or country:
+        location = LocationData(city=city, region=region, country=country)
+
+    user_name = await aget_user_name(user)
+
+    if ConversationCommand.Automation in conversation_commands:
+        try:
+            automation, crontime, query_to_run, subject = await create_automation(
+                q, timezone, user, request.url, meta_log
+            )
+        except Exception as e:
+            logger.error(f"Error creating automation {q} for {user.email}: {e}", exc_info=True)
+            return Response(
+                content=f"Unable to create automation. Ensure the automation doesn't already exist.",
+                media_type="text/plain",
+                status_code=500,
+            )
+
+        llm_response = construct_automation_created_message(automation, crontime, query_to_run, subject)
+        await sync_to_async(save_to_conversation_log)(
+            q,
+            llm_response,
+            user,
+            meta_log,
+            user_message_time,
+            intent_type="automation",
+            client_application=request.user.client_app,
+            conversation_id=conversation_id,
+            inferred_queries=[query_to_run],
+            automation_id=automation.id,
+        )
+
+        if stream:
+            return StreamingResponse(llm_response, media_type="text/event-stream", status_code=200)
+        else:
+            return Response(content=llm_response, media_type="text/plain", status_code=200)
+
+    compiled_references, inferred_queries, defiltered_query = await extract_references_and_questions(
+        request, meta_log, q, (n or 5), (d or math.inf), conversation_id, conversation_commands, location
+    )
+    online_results: Dict[str, Dict] = {}
+
+    if conversation_commands == [ConversationCommand.Notes] and not await EntryAdapters.auser_has_entries(user):
+        no_entries_found_format = no_entries_found.format()
+        if stream:
+            return StreamingResponse(iter([no_entries_found_format]), media_type="text/event-stream", status_code=200)
+        else:
+            response_obj = {"response": no_entries_found_format}
+            return Response(content=json.dumps(response_obj), media_type="text/plain", status_code=200)
+
+    if conversation_commands == [ConversationCommand.Notes] and is_none_or_empty(compiled_references):
+        no_notes_found_format = no_notes_found.format()
+        if stream:
+            return StreamingResponse(iter([no_notes_found_format]), media_type="text/event-stream", status_code=200)
+        else:
+            response_obj = {"response": no_notes_found_format}
+            return Response(content=json.dumps(response_obj), media_type="text/plain", status_code=200)
+
+    if ConversationCommand.Notes in conversation_commands and is_none_or_empty(compiled_references):
+        conversation_commands.remove(ConversationCommand.Notes)
+
+    if ConversationCommand.Online in conversation_commands:
+        try:
+            online_results = await search_online(defiltered_query, meta_log, location, custom_filters=_custom_filters)
+        except ValueError as e:
+            logger.warning(f"Error searching online: {e}. Attempting to respond without online results")
+
+    if ConversationCommand.Webpage in conversation_commands:
+        try:
+            online_results = await read_webpages(defiltered_query, meta_log, location)
+        except ValueError as e:
+            logger.warning(
+                f"Error directly reading webpages: {e}. Attempting to respond without online results", exc_info=True
+            )
+
+    if ConversationCommand.Image in conversation_commands:
+        update_telemetry_state(
+            request=request,
+            telemetry_type="api",
+            api="chat",
+            metadata={"conversation_command": conversation_commands[0].value},
+            **common.__dict__,
+        )
+        image, status_code, improved_image_prompt, intent_type = await text_to_image(
+            q, user, meta_log, location_data=location, references=compiled_references, online_results=online_results
+        )
+        if image is None:
+            content_obj = {"image": image, "intentType": intent_type, "detail": improved_image_prompt}
+            return Response(content=json.dumps(content_obj), media_type="application/json", status_code=status_code)
+
+        await sync_to_async(save_to_conversation_log)(
+            q,
+            image,
+            user,
+            meta_log,
+            user_message_time,
+            intent_type=intent_type,
+            inferred_queries=[improved_image_prompt],
+            client_application=request.user.client_app,
+            conversation_id=conversation.id,
+            compiled_references=compiled_references,
+            online_results=online_results,
+        )
+        content_obj = {"image": image, "intentType": intent_type, "inferredQueries": [improved_image_prompt], "context": compiled_references, "online_results": online_results}  # type: ignore
+        return Response(content=json.dumps(content_obj), media_type="application/json", status_code=status_code)
+
+    # Get the (streamed) chat response from the LLM of choice.
+    llm_response, chat_metadata = await agenerate_chat_response(
+        defiltered_query,
+        meta_log,
+        conversation,
+        compiled_references,
+        online_results,
+        inferred_queries,
+        conversation_commands,
+        user,
+        request.user.client_app,
+        conversation.id,
+        location,
+        user_name,
+    )
+
+    cmd_set = set([cmd.value for cmd in conversation_commands])
+    chat_metadata["conversation_command"] = cmd_set
+    chat_metadata["agent"] = conversation.agent.slug if conversation.agent else None
+
+    update_telemetry_state(
+        request=request,
+        telemetry_type="api",
+        api="chat",
+        metadata=chat_metadata,
+        **common.__dict__,
+    )
+
+    if llm_response is None:
+        return Response(content=llm_response, media_type="text/plain", status_code=500)
+
+    if stream:
+        return StreamingResponse(llm_response, media_type="text/event-stream", status_code=200)
+
+    iterator = AsyncIteratorWrapper(llm_response)
+
+    # Get the full response from the generator if the stream is not requested.
+    aggregated_gpt_response = ""
+    async for item in iterator:
+        if item is None:
+            break
+        aggregated_gpt_response += item
+
+    actual_response = aggregated_gpt_response.split("### compiled references:")[0]
+
+    response_obj = {
+        "response": actual_response,
+        "inferredQueries": inferred_queries,
+        "context": compiled_references,
+        "online_results": online_results,
+    }
+
+    return Response(content=json.dumps(response_obj), media_type="application/json", status_code=200)