khoj 1.16.1.dev25__py3-none-any.whl → 1.17.1.dev216__py3-none-any.whl

khoj/routers/web_client.py

@@ -1,30 +1,21 @@
 # System Packages
 import json
 import os
-from datetime import timedelta
 from typing import Optional
 
 from fastapi import APIRouter, Request
 from fastapi.responses import FileResponse, HTMLResponse, RedirectResponse
 from fastapi.templating import Jinja2Templates
-from starlette.authentication import has_required_scope, requires
+from starlette.authentication import requires
 
-from khoj.database import adapters
 from khoj.database.adapters import (
     AgentAdapters,
-    ConversationAdapters,
-    EntryAdapters,
     PublicConversationAdapters,
     get_user_github_config,
-    get_user_name,
     get_user_notion_config,
-    get_user_subscription_state,
 )
 from khoj.database.models import KhojUser
-from khoj.processor.speech.text_to_speech import is_eleven_labs_enabled
-from khoj.routers.helpers import get_next_url
-from khoj.routers.notion import get_notion_auth_url
-from khoj.routers.twilio import is_twilio_enabled
+from khoj.routers.helpers import get_next_url, get_user_config
 from khoj.utils import constants, state
 from khoj.utils.rawconfig import (
     GithubContentConfig,
@@ -42,80 +33,36 @@ templates = Jinja2Templates([constants.web_directory, constants.next_js_director
 @requires(["authenticated"], redirect="login_page")
 def index(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
+    user_config = get_user_config(user, request)
 
-    return templates.TemplateResponse(
-        "chat.html",
-        context={
-            "request": request,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    return templates.TemplateResponse("chat.html", context=user_config)
 
 
 @web_client.post("/", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def index_post(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
+    user_config = get_user_config(user, request)
 
-    return templates.TemplateResponse(
-        "chat.html",
-        context={
-            "request": request,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    return templates.TemplateResponse("chat.html", context=user_config)
 
 
 @web_client.get("/search", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def search_page(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
+    user_config = get_user_config(user, request)
 
-    return templates.TemplateResponse(
-        "search.html",
-        context={
-            "request": request,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    return templates.TemplateResponse("search.html", context=user_config)
 
 
 @web_client.get("/chat", response_class=FileResponse)
 @requires(["authenticated"], redirect="login_page")
 def chat_page(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
+    user_config = get_user_config(user, request)
 
-    return templates.TemplateResponse(
-        "chat.html",
-        context={
-            "request": request,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    return templates.TemplateResponse("chat.html", context=user_config)
 
 
 @web_client.get("/experimental", response_class=FileResponse)
@@ -169,25 +116,14 @@ def agents_page(request: Request):
 @web_client.get("/agent/{agent_slug}", response_class=HTMLResponse)
 def agent_page(request: Request, agent_slug: str):
     user: KhojUser = request.user.object if request.user.is_authenticated else None
-    user_picture = request.session.get("user", {}).get("picture") if user else None
-
+    user_config = get_user_config(user, request)
     agent = AgentAdapters.get_agent_by_slug(agent_slug)
-    has_documents = EntryAdapters.user_has_entries(user=user)
 
     if agent == None:
-        return templates.TemplateResponse(
-            "404.html",
-            context={
-                "request": request,
-                "khoj_version": state.khoj_version,
-                "username": user.username if user else None,
-                "has_documents": False,
-                "is_active": has_required_scope(request, ["premium"]),
-                "user_photo": user_picture,
-            },
-        )
+        user_config["has_documents"] = False
+        return templates.TemplateResponse("404.html", context=user_config)
 
-    agent_metadata = {
+    user_config["agent"] = {
         "slug": agent.slug,
         "avatar": agent.avatar,
         "name": agent.name,
@@ -199,115 +135,23 @@ def agent_page(request: Request, agent_slug: str):
         "creator_not_self": agent.creator != user,
     }
 
-    return templates.TemplateResponse(
-        "agent.html",
-        context={
-            "request": request,
-            "agent": agent_metadata,
-            "khoj_version": state.khoj_version,
-            "username": user.username if user else None,
-            "has_documents": has_documents,
-            "is_active": has_required_scope(request, ["premium"]),
-            "user_photo": user_picture,
-        },
-    )
+    return templates.TemplateResponse("agent.html", context=user_config)
 
 
-@web_client.get("/config", response_class=HTMLResponse)
+@web_client.get("/settings", response_class=HTMLResponse)
 @requires(["authenticated"], redirect="login_page")
 def config_page(request: Request):
     user: KhojUser = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
-
-    user_subscription_state = get_user_subscription_state(user.email)
-    user_subscription = adapters.get_user_subscription(user.email)
-    subscription_renewal_date = (
-        user_subscription.renewal_date.strftime("%d %b %Y")
-        if user_subscription and user_subscription.renewal_date
-        else (user_subscription.created_at + timedelta(days=7)).strftime("%d %b %Y")
-    )
-    given_name = get_user_name(user)
-
-    enabled_content_source = set(EntryAdapters.get_unique_file_sources(user))
-    successfully_configured = {
-        "computer": ("computer" in enabled_content_source),
-        "github": ("github" in enabled_content_source),
-        "notion": ("notion" in enabled_content_source),
-    }
-
-    selected_conversation_config = ConversationAdapters.get_conversation_config(user)
-    conversation_options = ConversationAdapters.get_conversation_processor_options().all()
-    all_conversation_options = list()
-    for conversation_option in conversation_options:
-        all_conversation_options.append({"chat_model": conversation_option.chat_model, "id": conversation_option.id})
-
-    search_model_options = adapters.get_or_create_search_models().all()
-    all_search_model_options = list()
-    for search_model_option in search_model_options:
-        all_search_model_options.append({"name": search_model_option.name, "id": search_model_option.id})
-
-    current_search_model_option = adapters.get_user_search_model_or_default(user)
-
-    selected_paint_model_config = ConversationAdapters.get_user_text_to_image_model_config(user)
-    paint_model_options = ConversationAdapters.get_text_to_image_model_options().all()
-    all_paint_model_options = list()
-    for paint_model in paint_model_options:
-        all_paint_model_options.append({"model_name": paint_model.model_name, "id": paint_model.id})
-
-    notion_oauth_url = get_notion_auth_url(user)
+    user_config = get_user_config(user, request, is_detailed=True)
 
-    eleven_labs_enabled = is_eleven_labs_enabled()
+    return templates.TemplateResponse("settings.html", context=user_config)
 
-    voice_models = ConversationAdapters.get_voice_model_options()
-    voice_model_options = list()
-    for voice_model in voice_models:
-        voice_model_options.append({"name": voice_model.name, "id": voice_model.model_id})
 
-    if len(voice_model_options) == 0:
-        eleven_labs_enabled = False
-
-    selected_voice_config = ConversationAdapters.get_voice_model_config(user)
-
-    return templates.TemplateResponse(
-        "config.html",
-        context={
-            "request": request,
-            "current_model_state": successfully_configured,
-            "anonymous_mode": state.anonymous_mode,
-            "username": user.username,
-            "given_name": given_name,
-            "search_model_options": all_search_model_options,
-            "selected_search_model_config": current_search_model_option.id,
-            "conversation_options": all_conversation_options,
-            "selected_conversation_config": selected_conversation_config.id if selected_conversation_config else None,
-            "paint_model_options": all_paint_model_options,
-            "selected_paint_model_config": selected_paint_model_config.id if selected_paint_model_config else None,
-            "user_photo": user_picture,
-            "billing_enabled": state.billing_enabled,
-            "subscription_state": user_subscription_state,
-            "subscription_renewal_date": subscription_renewal_date,
-            "khoj_cloud_subscription_url": os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL"),
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "is_twilio_enabled": is_twilio_enabled(),
-            "is_eleven_labs_enabled": eleven_labs_enabled,
-            "voice_model_options": voice_model_options,
-            "selected_voice_config": selected_voice_config.model_id if selected_voice_config else None,
-            "phone_number": user.phone_number,
-            "is_phone_number_verified": user.verified_phone_number,
-            "khoj_version": state.khoj_version,
-            "notion_oauth_url": notion_oauth_url,
-        },
-    )
-
-
-@web_client.get("/config/content-source/github", response_class=HTMLResponse)
+@web_client.get("/settings/content/github", response_class=HTMLResponse)
 @requires(["authenticated"], redirect="login_page")
 def github_config_page(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
+    user_config = get_user_config(user, request)
     current_github_config = get_user_github_config(user)
 
     if current_github_config:
@@ -329,66 +173,32 @@ def github_config_page(request: Request):
     else:
         current_config = {}  # type: ignore
 
-    return templates.TemplateResponse(
-        "content_source_github_input.html",
-        context={
-            "request": request,
-            "current_config": current_config,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    user_config["current_config"] = current_config
+    return templates.TemplateResponse("content_source_github_input.html", context=user_config)
 
 
-@web_client.get("/config/content-source/notion", response_class=HTMLResponse)
+@web_client.get("/settings/content/notion", response_class=HTMLResponse)
 @requires(["authenticated"], redirect="login_page")
 def notion_config_page(request: Request):
     user = request.user.object
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user)
-    current_notion_config = get_user_notion_config(user)
-
-    current_config = NotionContentConfig(
-        token=current_notion_config.token if current_notion_config else "",
-    )
+    user_config = get_user_config(user, request)
 
+    current_notion_config = get_user_notion_config(user)
+    token = current_notion_config.token if current_notion_config else ""
+    current_config = NotionContentConfig(token=token)
     current_config = json.loads(current_config.model_dump_json())
 
-    return templates.TemplateResponse(
-        "content_source_notion_input.html",
-        context={
-            "request": request,
-            "current_config": current_config,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    user_config["current_config"] = current_config
+    return templates.TemplateResponse("content_source_notion_input.html", context=user_config)
 
 
-@web_client.get("/config/content-source/computer", response_class=HTMLResponse)
+@web_client.get("/settings/content/computer", response_class=HTMLResponse)
 @requires(["authenticated"], redirect="login_page")
 def computer_config_page(request: Request):
     user = request.user.object if request.user.is_authenticated else None
-    user_picture = request.session.get("user", {}).get("picture") if user else None
-    has_documents = EntryAdapters.user_has_entries(user=user) if user else False
+    user_config = get_user_config(user, request)
 
-    return templates.TemplateResponse(
-        "content_source_computer_input.html",
-        context={
-            "request": request,
-            "username": user.username,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-        },
-    )
+    return templates.TemplateResponse("content_source_computer_input.html", context=user_config)
 
 
 @web_client.get("/share/chat/{public_conversation_slug}", response_class=HTMLResponse)
@@ -404,8 +214,9 @@ def view_public_conversation(request: Request):
             },
         )
     user = request.user.object if request.user.is_authenticated else None
-    user_picture = request.session.get("user", {}).get("picture") if user else None
-    has_documents = EntryAdapters.user_has_entries(user=user) if user else False
+    user_config = get_user_config(user, request)
+    user_config["public_conversation_slug"] = public_conversation_slug
+    user_config["google_client_id"] = os.environ.get("GOOGLE_CLIENT_ID")
 
     all_agents = AgentAdapters.get_all_accessible_agents(request.user.object if request.user.is_authenticated else None)
 
@@ -420,28 +231,15 @@ def view_public_conversation(request: Request):
                 "name": agent.name,
             }
         )
+    user_config["agents"] = agents_packet
 
-    google_client_id = os.environ.get("GOOGLE_CLIENT_ID")
     redirect_uri = str(request.app.url_path_for("auth"))
     next_url = str(
         request.app.url_path_for("view_public_conversation", public_conversation_slug=public_conversation_slug)
     )
+    user_config["redirect_uri"] = f"{redirect_uri}?next={next_url}"
 
-    return templates.TemplateResponse(
-        "public_conversation.html",
-        context={
-            "request": request,
-            "username": user.username if user else None,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-            "public_conversation_slug": public_conversation_slug,
-            "agents": agents_packet,
-            "google_client_id": google_client_id,
-            "redirect_uri": f"{redirect_uri}?next={next_url}",
-        },
-    )
+    return templates.TemplateResponse("public_conversation.html", context=user_config)
 
 
 @web_client.get("/automations", response_class=HTMLResponse)
@@ -452,20 +250,9 @@ def automations_config_page(
     queryToRun: Optional[str] = None,
 ):
     user = request.user.object if request.user.is_authenticated else None
-    user_picture = request.session.get("user", {}).get("picture")
-    has_documents = EntryAdapters.user_has_entries(user=user) if user else False
+    user_config = get_user_config(user, request)
+    user_config["subject"] = subject if subject else ""
+    user_config["crontime"] = crontime if crontime else ""
+    user_config["queryToRun"] = queryToRun if queryToRun else ""
 
-    return templates.TemplateResponse(
-        "config_automation.html",
-        context={
-            "request": request,
-            "username": user.username if user else None,
-            "user_photo": user_picture,
-            "is_active": has_required_scope(request, ["premium"]),
-            "has_documents": has_documents,
-            "khoj_version": state.khoj_version,
-            "subject": subject if subject else "",
-            "crontime": crontime if crontime else "",
-            "queryToRun": queryToRun if queryToRun else "",
-        },
-    )
+    return templates.TemplateResponse("config_automation.html", context=user_config)

khoj/search_type/text_search.py

@@ -199,17 +199,16 @@ def setup(
     text_to_entries: Type[TextToEntries],
     files: dict[str, str],
     regenerate: bool,
-    full_corpus: bool = True,
     user: KhojUser = None,
     config=None,
-) -> None:
+) -> Tuple[int, int]:
     if config:
         num_new_embeddings, num_deleted_embeddings = text_to_entries(config).process(
-            files=files, full_corpus=full_corpus, user=user, regenerate=regenerate
+            files=files, user=user, regenerate=regenerate
         )
     else:
         num_new_embeddings, num_deleted_embeddings = text_to_entries().process(
-            files=files, full_corpus=full_corpus, user=user, regenerate=regenerate
+            files=files, user=user, regenerate=regenerate
         )
 
     if files:
@@ -219,6 +218,8 @@ def setup(
             f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names[:10]} ..."
         )
 
+    return num_new_embeddings, num_deleted_embeddings
+
 
 def cross_encoder_score(query: str, hits: List[SearchResponse], search_model_name: str) -> List[SearchResponse]:
     """Score all retrieved entries using the cross-encoder"""

khoj/utils/fs_syncer.py

@@ -22,7 +22,7 @@ magika = Magika()
 
 
 def collect_files(search_type: Optional[SearchType] = SearchType.All, user=None) -> dict:
-    files = {}
+    files: dict[str, dict] = {"docx": {}, "image": {}}
 
     if search_type == SearchType.All or search_type == SearchType.Org:
         org_config = LocalOrgConfig.objects.filter(user=user).first()
@@ -36,6 +36,8 @@ def collect_files(search_type: Optional[SearchType] = SearchType.All, user=None)
     if search_type == SearchType.All or search_type == SearchType.Pdf:
         pdf_config = LocalPdfConfig.objects.filter(user=user).first()
         files["pdf"] = get_pdf_files(construct_config_from_db(pdf_config)) if pdf_config else {}
+    files["image"] = {}
+    files["docx"] = {}
     return files
 
 
@@ -122,7 +124,7 @@ def get_org_files(config: TextContentConfig):
         logger.debug("At least one of org-files or org-file-filter is required to be specified")
         return {}
 
-    "Get Org files to process"
+    # Get Org files to process
     absolute_org_files, filtered_org_files = set(), set()
     if org_files:
         absolute_org_files = {get_absolute_path(org_file) for org_file in org_files}

khoj/utils/rawconfig.py

@@ -27,11 +27,16 @@ class LocationData(BaseModel):
     country: Optional[str]
 
 
-class FilterRequest(BaseModel):
+class FileFilterRequest(BaseModel):
     filename: str
     conversation_id: str
 
 
+class FilesFilterRequest(BaseModel):
+    filenames: List[str]
+    conversation_id: str
+
+
 class TextConfigBase(ConfigBase):
     compressed_jsonl: Path
     embeddings_file: Path

{khoj-1.16.1.dev25.dist-info → khoj-1.17.1.dev216.dist-info}/METADATA

@@ -1,7 +1,7 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.16.1.dev25
-Summary: An AI copilot for your Second Brain
+Version: 1.17.1.dev216
+Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
 Project-URL: Code, https://github.com/khoj-ai/khoj