khoj 1.17.1.dev229__py3-none-any.whl → 1.20.3__py3-none-any.whl

khoj/interface/web/assets/icons/automation.svg

@@ -1,37 +1,42 @@
 <svg
-   width="800px"
-   height="800px"
-   viewBox="0 0 24 24"
-   fill="none"
-   version="1.1"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:svg="http://www.w3.org/2000/svg">
-  <path
-     d="m 19.402765,19.007843 c 1.380497,-1.679442 2.307667,-4.013099 2.307667,-6.330999 C 21.710432,7.2551476 17.193958,2.86 11.622674,2.86 6.0513555,2.86 1.5349153,7.2551476 1.5349153,12.676844 c 0,5.421663 4.5164402,9.816844 10.0877587,9.816844 2.381867,0 4.570922,-0.803307 6.296712,-2.14673 z m -7.780091,1.925408 c -4.3394583,0 -8.6708434,-4.033489 -8.6708434,-8.256407 0,-4.2229187 4.3313851,-8.2564401 8.6708434,-8.2564401 4.339458,0 8.670809,4.2369112 8.670809,8.4598301 0,4.222918 -4.331351,8.053017 -8.670809,8.053017 z"
-     fill="#1c274c"
-     fill-rule="evenodd"
-     clip-rule="evenodd"
-     fill-opacity="1"
-     stroke-width="1.10519"
-     stroke-dasharray="none" />
-  <path
-     d="m 14.177351,11.200265 0.05184,2.153289"
-     stroke="#1c274c"
-     stroke-width="1.95702"
-     stroke-linecap="round" />
-  <path
-     d="m 9.271347,11.140946 0.051844,2.153289"
-     stroke="#1c274c"
-     stroke-width="1.95701"
-     stroke-linecap="round" />
-  <path
-     d="m 13.557051,1.4687179 c -1.779392,0.00605 -3.082184,0.01209 -3.6968064,0.018135"
-     stroke="#1c274c"
-     stroke-width="1.77333"
-     stroke-linecap="round" />
-  <path
-     d="M 20.342466,5.7144363 19.140447,6.8696139"
-     stroke="#1c274c"
-     stroke-width="1.95701"
-     stroke-linecap="round" />
+    xmlns="http://www.w3.org/2000/svg"
+    version="1.1"
+    viewBox="0 0 24 24"
+    width="800"
+    height="800"
+    fill="none"
+>
+    <path
+        d="m 20.503667,19.512812 c 1.498791,-1.721309 2.505409,-4.113142 2.505409,-6.488822 0,-5.556853 -4.903485,-10.0615654 -10.952165,-10.0615654 -6.0487172,0 -10.9521652,4.5047124 -10.9521652,10.0615654 0,5.556818 4.903448,10.061562 10.9521652,10.061562 2.585967,0 4.962599,-0.823331 6.836268,-2.200243 z m -8.446756,1.973405 c -4.7113005,0 -9.4138363,-4.134038 -9.4138363,-8.462227 0,-4.3281914 4.7025358,-8.4622628 9.4138363,-8.4622628 4.711301,0 9.4138,4.3425313 9.4138,8.6707228 0,4.328187 -4.702499,8.253767 -9.4138,8.253767 z"
+        fill="currentColor"
+        fill-rule="evenodd"
+        clip-rule="evenodd"
+        fill-opacity="1"
+        stroke-width="1.16584"
+        stroke-dasharray="none"
+    />
+    <path
+        d="m 14.703831,11.557853 0.0553,2.175599"
+        stroke="currentColor"
+        stroke-width="2.03174"
+        stroke-linecap="round"
+    />
+    <path
+        d="m 9.470316,11.497919 0.055305,2.175598"
+        stroke="currentColor"
+        stroke-width="2.03172"
+        stroke-linecap="round"
+    />
+    <path
+        d="m 13.921987,2.0293778 c -1.700446,0.00578 -2.945437,0.011553 -3.53279,0.01733"
+        stroke="currentColor"
+        stroke-width="1.69465"
+        stroke-linecap="round"
+    />
+    <path
+        d="m 21.395697,5.8423397 -1.28226,1.167146"
+        stroke="currentColor"
+        stroke-width="2.03172"
+        stroke-linecap="round"
+    />
 </svg>

khoj/interface/web/assets/icons/search.svg

@@ -1,25 +1,57 @@
-<svg xmlns="http://www.w3.org/2000/svg"
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
    width="800px"
    height="800px"
    viewBox="0 0 24 24"
    fill="none"
-   version="1.1">
+   version="1.1"
+   id="svg3"
+   sodipodi:docname="search.svg"
+   inkscape:version="1.3 (0e150ed, 2023-07-21)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs3" />
+  <sodipodi:namedview
+     id="namedview3"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:zoom="0.295"
+     inkscape:cx="398.30508"
+     inkscape:cy="400"
+     inkscape:window-width="1312"
+     inkscape:window-height="449"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg3" />
   <path
-     d="m 18.562765,17.147843 c 1.380497,-1.679442 2.307667,-4.013099 2.307667,-6.330999 C 20.870432,5.3951476 16.353958,1 10.782674,1 5.2113555,1 0.69491525,5.3951476 0.69491525,10.816844 c 0,5.421663 4.51644025,9.816844 10.08775875,9.816844 2.381867,0 4.570922,-0.803307 6.296712,-2.14673 0.508475,-0.508475 4.514633,4.192839 4.514633,4.192839 1.036377,1.008544 2.113087,-0.02559 1.07671,-1.034139 z m -7.780091,1.925408 c -4.3394583,0 -8.6708434,-4.033489 -8.6708434,-8.256407 0,-4.2229187 4.3313851,-8.2564401 8.6708434,-8.2564401 4.339458,0 8.670809,4.2369112 8.670809,8.4598301 0,4.222918 -4.331351,8.053017 -8.670809,8.053017 z"
+     d="m 18.682765,16.907843 c 1.380497,-1.679442 2.307667,-4.013099 2.307667,-6.330999 C 20.990432,5.1551476 16.473958,0.76 10.902674,0.76 5.3313555,0.76 0.81491525,5.1551476 0.81491525,10.576844 c 0,5.421663 4.51644025,9.816844 10.08775875,9.816844 2.381867,0 4.570922,-0.803307 6.296712,-2.14673 0.508475,-0.508475 4.616328,4.599619 4.616328,4.599619 1.036377,1.008544 2.113087,-0.02559 1.07671,-1.034139 z m -7.780091,1.925408 c -4.3394583,0 -8.6708434,-4.033489 -8.6708434,-8.256407 0,-4.2229187 4.3313851,-8.2564401 8.6708434,-8.2564401 4.339458,0 8.670809,4.2369112 8.670809,8.4598301 0,4.222918 -4.331351,8.053017 -8.670809,8.053017 z"
      fill="#1c274c"
      fill-rule="evenodd"
      clip-rule="evenodd"
      fill-opacity="1"
      stroke-width="1.10519"
-     stroke-dasharray="none" />
+     stroke-dasharray="none"
+     id="path1"
+     sodipodi:nodetypes="cssssccccsssss" />
   <path
-     d="m 13.337351,9.3402647 0.05184,2.1532893"
+     d="m 13.457351,9.1002647 0.05184,2.1532893"
      stroke="#1c274c"
      stroke-width="1.95702"
-     stroke-linecap="round" />
+     stroke-linecap="round"
+     id="path2" />
   <path
-     d="M 8.431347,9.2809457 8.483191,11.434235"
+     d="M 8.551347,9.0409457 8.603191,11.194235"
      stroke="#1c274c"
      stroke-width="1.95701"
-     stroke-linecap="round" />
+     stroke-linecap="round"
+     id="path3" />
 </svg>

khoj/interface/web/login.html

@@ -18,7 +18,7 @@
                 <div class="login-modal-title">Login to Khoj</div>
                 <!-- Sign in with Magic Link -->
                 <div class="khoj-magic-link">
-                    <input type="email" id="email" placeholder="Email" required>
+                    <input type="email" id="email" placeholder="Email" autofocus required>
                     <button id="magic-link-button">Send Magic Link</button>
                 </div>
                 <!-- Divider -->

khoj/routers/api_chat.py

@@ -41,6 +41,7 @@ from khoj.routers.helpers import (
     get_conversation_command,
     is_query_empty,
     is_ready_to_chat,
+    read_chat_stream,
     text_to_image,
     update_telemetry_state,
     validate_conversation_config,
@@ -570,8 +571,7 @@ async def chat(
                 logger.error(f"Failed to stream chat API response to {user} on {common.client}: {e}", exc_info=True)
                 return
             finally:
-                if stream:
-                    yield event_delimiter
+                yield event_delimiter
 
         async def send_llm_response(response: str):
             async for result in send_event(ChatEvent.START_LLM_RESPONSE, ""):
@@ -937,17 +937,6 @@ async def chat(
         return StreamingResponse(event_generator(q), media_type="text/plain")
     ## Non-Streaming Text Response
     else:
-        # Get the full response from the generator if the stream is not requested.
-        response_obj = {}
-        actual_response = ""
-        iterator = event_generator(q)
-        async for item in iterator:
-            try:
-                item_json = json.loads(item)
-                if "type" in item_json and item_json["type"] == ChatEvent.REFERENCES.value:
-                    response_obj = item_json["data"]
-            except:
-                actual_response += item
-        response_obj["response"] = actual_response
-
-        return Response(content=json.dumps(response_obj), media_type="application/json", status_code=200)
+        response_iterator = event_generator(q)
+        response_data = await read_chat_stream(response_iterator)
+        return Response(content=json.dumps(response_data), media_type="application/json", status_code=200)

khoj/routers/helpers.py

@@ -15,6 +15,7 @@ from random import random
 from typing import (
     Annotated,
     Any,
+    AsyncGenerator,
     Callable,
     Dict,
     Iterator,
@@ -1225,6 +1226,79 @@ class ChatEvent(Enum):
     STATUS = "status"
 
 
+class MessageProcessor:
+    def __init__(self):
+        self.references = {}
+        self.raw_response = ""
+
+    def convert_message_chunk_to_json(self, raw_chunk: str) -> Dict[str, Any]:
+        if raw_chunk.startswith("{") and raw_chunk.endswith("}"):
+            try:
+                json_chunk = json.loads(raw_chunk)
+                if "type" not in json_chunk:
+                    json_chunk = {"type": "message", "data": json_chunk}
+                return json_chunk
+            except json.JSONDecodeError:
+                return {"type": "message", "data": raw_chunk}
+        elif raw_chunk:
+            return {"type": "message", "data": raw_chunk}
+        return {"type": "", "data": ""}
+
+    def process_message_chunk(self, raw_chunk: str) -> None:
+        chunk = self.convert_message_chunk_to_json(raw_chunk)
+        if not chunk or not chunk["type"]:
+            return
+
+        chunk_type = ChatEvent(chunk["type"])
+        if chunk_type == ChatEvent.REFERENCES:
+            self.references = chunk["data"]
+        elif chunk_type == ChatEvent.MESSAGE:
+            chunk_data = chunk["data"]
+            if isinstance(chunk_data, dict):
+                self.raw_response = self.handle_json_response(chunk_data)
+            elif (
+                isinstance(chunk_data, str) and chunk_data.strip().startswith("{") and chunk_data.strip().endswith("}")
+            ):
+                try:
+                    json_data = json.loads(chunk_data.strip())
+                    self.raw_response = self.handle_json_response(json_data)
+                except json.JSONDecodeError:
+                    self.raw_response += chunk_data
+            else:
+                self.raw_response += chunk_data
+
+    def handle_json_response(self, json_data: Dict[str, str]) -> str | Dict[str, str]:
+        if "image" in json_data or "details" in json_data:
+            return json_data
+        if "response" in json_data:
+            return json_data["response"]
+        return json_data
+
+
+async def read_chat_stream(response_iterator: AsyncGenerator[str, None]) -> Dict[str, Any]:
+    processor = MessageProcessor()
+    event_delimiter = "␃🔚␗"
+    buffer = ""
+
+    async for chunk in response_iterator:
+        # Start buffering chunks until complete event is received
+        buffer += chunk
+
+        # Once the buffer contains a complete event
+        while event_delimiter in buffer:
+            # Extract the event from the buffer
+            event, buffer = buffer.split(event_delimiter, 1)
+            # Process the event
+            if event:
+                processor.process_message_chunk(event)
+
+    # Process any remaining data in the buffer
+    if buffer:
+        processor.process_message_chunk(buffer)
+
+    return {"response": processor.raw_response, "references": processor.references}
+
+
 def get_user_config(user: KhojUser, request: Request, is_detailed: bool = False):
     user_picture = request.session.get("user", {}).get("picture")
     is_active = has_required_scope(request, ["premium"])

khoj/routers/web_client.py

@@ -1,68 +1,54 @@
 # System Packages
 import json
 import os
-from typing import Optional
 
 from fastapi import APIRouter, Request
 from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
 from starlette.authentication import requires
 
-from khoj.database.adapters import (
-    AgentAdapters,
-    PublicConversationAdapters,
-    get_user_github_config,
-    get_user_notion_config,
-)
-from khoj.database.models import KhojUser
+from khoj.database.adapters import get_user_github_config
 from khoj.routers.helpers import get_next_url, get_user_config
-from khoj.utils import constants, state
-from khoj.utils.rawconfig import (
-    GithubContentConfig,
-    GithubRepoConfig,
-    NotionContentConfig,
-)
+from khoj.utils import constants
+from khoj.utils.rawconfig import GithubContentConfig, GithubRepoConfig
 
 # Initialize Router
 web_client = APIRouter()
-templates = Jinja2Templates([constants.web_directory, constants.next_js_directory])
+templates = Jinja2Templates([constants.web_directory, constants.next_js_directory, constants.pypi_static_directory])
 
 
 # Create Routes
 @web_client.get("/", response_class=FileResponse)
-@requires(["authenticated"], redirect="login_page")
 def index(request: Request):
-    user = request.user.object
-    user_config = get_user_config(user, request)
-
-    return templates.TemplateResponse("chat.html", context=user_config)
+    return templates.TemplateResponse("index.html", context={"request": request})
 
 
 @web_client.post("/", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def index_post(request: Request):
-    user = request.user.object
-    user_config = get_user_config(user, request)
-
-    return templates.TemplateResponse("chat.html", context=user_config)
+    return templates.TemplateResponse("index.html", context={"request": request})
 
 
 @web_client.get("/search", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def search_page(request: Request):
-    user = request.user.object
-    user_config = get_user_config(user, request)
-
-    return templates.TemplateResponse("search.html", context=user_config)
+    return templates.TemplateResponse(
+        "search/index.html",
+        context={
+            "request": request,
+        },
+    )
 
 
 @web_client.get("/chat", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def chat_page(request: Request):
-    user = request.user.object
-    user_config = get_user_config(user, request)
-
-    return templates.TemplateResponse("chat.html", context=user_config)
+    return templates.TemplateResponse(
+        "chat/index.html",
+        context={
+            "request": request,
+        },
+    )
 
 
 @web_client.get("/experimental", response_class=FileResponse)
@@ -113,38 +99,10 @@ def agents_page(request: Request):
     )
 
 
-@web_client.get("/agent/{agent_slug}", response_class=HTMLResponse)
-def agent_page(request: Request, agent_slug: str):
-    user: KhojUser = request.user.object if request.user.is_authenticated else None
-    user_config = get_user_config(user, request)
-    agent = AgentAdapters.get_agent_by_slug(agent_slug)
-
-    if agent == None:
-        user_config["has_documents"] = False
-        return templates.TemplateResponse("404.html", context=user_config)
-
-    user_config["agent"] = {
-        "slug": agent.slug,
-        "avatar": agent.avatar,
-        "name": agent.name,
-        "personality": agent.personality,
-        "public": agent.public,
-        "creator": agent.creator.username if agent.creator else None,
-        "managed_by_admin": agent.managed_by_admin,
-        "chat_model": agent.chat_model.chat_model,
-        "creator_not_self": agent.creator != user,
-    }
-
-    return templates.TemplateResponse("agent.html", context=user_config)
-
-
 @web_client.get("/settings", response_class=HTMLResponse)
 @requires(["authenticated"], redirect="login_page")
 def config_page(request: Request):
-    user: KhojUser = request.user.object
-    user_config = get_user_config(user, request, is_detailed=True)
-
-    return templates.TemplateResponse("settings.html", context=user_config)
+    return templates.TemplateResponse("settings/index.html", context={"request": request})
 
 
 @web_client.get("/settings/content/github", response_class=HTMLResponse)
@@ -177,82 +135,23 @@ def github_config_page(request: Request):
     return templates.TemplateResponse("content_source_github_input.html", context=user_config)
 
 
-@web_client.get("/settings/content/notion", response_class=HTMLResponse)
-@requires(["authenticated"], redirect="login_page")
-def notion_config_page(request: Request):
-    user = request.user.object
-    user_config = get_user_config(user, request)
-
-    current_notion_config = get_user_notion_config(user)
-    token = current_notion_config.token if current_notion_config else ""
-    current_config = NotionContentConfig(token=token)
-    current_config = json.loads(current_config.model_dump_json())
-
-    user_config["current_config"] = current_config
-    return templates.TemplateResponse("content_source_notion_input.html", context=user_config)
-
-
-@web_client.get("/settings/content/computer", response_class=HTMLResponse)
-@requires(["authenticated"], redirect="login_page")
-def computer_config_page(request: Request):
-    user = request.user.object if request.user.is_authenticated else None
-    user_config = get_user_config(user, request)
-
-    return templates.TemplateResponse("content_source_computer_input.html", context=user_config)
-
-
 @web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse)
 def view_public_conversation(request: Request):
-    public_conversation_slug = request.path_params.get("public_conversation_slug")
-    public_conversation = PublicConversationAdapters.get_public_conversation_by_slug(public_conversation_slug)
-    if not public_conversation:
-        return templates.TemplateResponse(
-            "404.html",
-            context={
-                "request": request,
-                "khoj_version": state.khoj_version,
-            },
-        )
-    user = request.user.object if request.user.is_authenticated else None
-    user_config = get_user_config(user, request)
-    user_config["public_conversation_slug"] = public_conversation_slug
-    user_config["google_client_id"] = os.environ.get("GOOGLE_CLIENT_ID")
-
-    all_agents = AgentAdapters.get_all_accessible_agents(request.user.object if request.user.is_authenticated else None)
-
-    # Filter out the current agent
-    all_agents = [agent for agent in all_agents if agent != public_conversation.agent]
-    agents_packet = []
-    for agent in all_agents:
-        agents_packet.append(
-            {
-                "slug": agent.slug,
-                "avatar": agent.avatar,
-                "name": agent.name,
-            }
-        )
-    user_config["agents"] = agents_packet
-
-    redirect_uri = str(request.app.url_path_for("auth"))
-    next_url = str(
-        request.app.url_path_for("view_public_conversation", public_conversation_slug=public_conversation_slug)
+    return templates.TemplateResponse(
+        "share/chat/index.html",
+        context={
+            "request": request,
+        },
     )
-    user_config["redirect_uri"] = f"{redirect_uri}?next={next_url}"
-
-    return templates.TemplateResponse("public_conversation.html", context=user_config)
 
 
 @web_client.get("/automations", response_class=HTMLResponse)
 def automations_config_page(
     request: Request,
-    subject: Optional[str] = None,
-    crontime: Optional[str] = None,
-    queryToRun: Optional[str] = None,
 ):
-    user = request.user.object if request.user.is_authenticated else None
-    user_config = get_user_config(user, request)
-    user_config["subject"] = subject if subject else ""
-    user_config["crontime"] = crontime if crontime else ""
-    user_config["queryToRun"] = queryToRun if queryToRun else ""
-
-    return templates.TemplateResponse("config_automation.html", context=user_config)
+    return templates.TemplateResponse(
+        "automations/index.html",
+        context={
+            "request": request,
+        },
+    )

khoj/utils/constants.py

@@ -3,6 +3,7 @@ from pathlib import Path
 app_root_directory = Path(__file__).parent.parent.parent
 web_directory = app_root_directory / "khoj/interface/web/"
 next_js_directory = app_root_directory / "khoj/interface/built/"
+pypi_static_directory = app_root_directory / "khoj/interface/compiled/"
 empty_escape_sequences = "\n|\r|\t| "
 app_env_filepath = "~/.khoj/env"
 telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"

{khoj-1.17.1.dev229.dist-info → khoj-1.20.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.17.1.dev229
+Version: 1.20.3
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev