khoj 1.31.1.dev13__py3-none-any.whl → 1.32.0__py3-none-any.whl

khoj/processor/tools/online_search.py

@@ -102,8 +102,14 @@ async def search_online(
         async for event in send_status_func(f"**Searching the Internet for**: {subqueries_str}"):
             yield {ChatEvent.STATUS: event}
 
+    if SERPER_DEV_API_KEY:
+        search_func = search_with_serper
+    elif JINA_API_KEY:
+        search_func = search_with_jina
+    else:
+        search_func = search_with_searxng
+
     with timer(f"Internet searches for {subqueries} took", logger):
-        search_func = search_with_google if SERPER_DEV_API_KEY else search_with_jina
         search_tasks = [search_func(subquery, location) for subquery in subqueries]
         search_results = await asyncio.gather(*search_tasks)
         response_dict = {subquery: search_result for subquery, search_result in search_results}
@@ -148,7 +154,48 @@ async def search_online(
     yield response_dict
 
 
-async def search_with_google(query: str, location: LocationData) -> Tuple[str, Dict[str, List[Dict]]]:
+async def search_with_searxng(query: str, location: LocationData) -> Tuple[str, Dict[str, List[Dict]]]:
+    """Search using local SearXNG instance."""
+    # Use environment variable or default to localhost
+    searxng_url = os.getenv("KHOJ_SEARXNG_URL", "http://localhost:42113")
+    search_url = f"{searxng_url}/search"
+    country_code = location.country_code.lower() if location and location.country_code else "us"
+
+    params = {"q": query, "format": "html", "language": "en", "country": country_code, "categories": "general"}
+
+    async with aiohttp.ClientSession() as session:
+        try:
+            async with session.get(search_url, params=params) as response:
+                if response.status != 200:
+                    logger.error(f"SearXNG search failed to call {searxng_url}: {await response.text()}")
+                    return query, {}
+
+                html_content = await response.text()
+
+                soup = BeautifulSoup(html_content, "html.parser")
+                organic_results = []
+
+                for result in soup.find_all("article", class_="result"):
+                    title_elem = result.find("a", rel="noreferrer")
+                    if title_elem:
+                        title = title_elem.text.strip()
+                        link = title_elem["href"]
+
+                        description_elem = result.find("p", class_="content")
+                        description = description_elem.text.strip() if description_elem else None
+
+                        organic_results.append({"title": title, "link": link, "description": description})
+
+                extracted_search_result = {"organic": organic_results}
+
+                return query, extracted_search_result
+
+        except Exception as e:
+            logger.error(f"Error searching with SearXNG: {str(e)}")
+            return query, {}
+
+
+async def search_with_serper(query: str, location: LocationData) -> Tuple[str, Dict[str, List[Dict]]]:
     country_code = location.country_code.lower() if location and location.country_code else "us"
     payload = json.dumps({"q": query, "gl": country_code})
     headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}

khoj/routers/api_chat.py

@@ -724,7 +724,16 @@ async def chat(
                 yield result
             return
 
-        conversation_commands = [get_conversation_command(query=q, any_references=True)]
+        # Automated tasks are handled before to allow mixing them with other conversation commands
+        cmds_to_rate_limit = []
+        is_automated_task = False
+        if q.startswith("/automated_task"):
+            is_automated_task = True
+            q = q.replace("/automated_task", "").lstrip()
+            cmds_to_rate_limit += [ConversationCommand.AutomatedTask]
+
+        # Extract conversation command from query
+        conversation_commands = [get_conversation_command(query=q)]
 
         conversation = await ConversationAdapters.aget_conversation_by_user(
             user,
@@ -757,11 +766,8 @@ async def chat(
         location = None
         if city or region or country or country_code:
             location = LocationData(city=city, region=region, country=country, country_code=country_code)
-
         user_message_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
         meta_log = conversation.conversation_log
-        is_automated_task = conversation_commands == [ConversationCommand.AutomatedTask]
 
         researched_results = ""
         online_results: Dict = dict()
@@ -778,7 +784,7 @@ async def chat(
         generated_excalidraw_diagram: str = None
         program_execution_context: List[str] = []
 
-        if conversation_commands == [ConversationCommand.Default] or is_automated_task:
+        if conversation_commands == [ConversationCommand.Default]:
             chosen_io = await aget_data_sources_and_output_format(
                 q,
                 meta_log,
@@ -799,7 +805,8 @@ async def chat(
             async for result in send_event(ChatEvent.STATUS, f"**Selected Tools:** {conversation_commands_str}"):
                 yield result
 
-        for cmd in conversation_commands:
+        cmds_to_rate_limit += conversation_commands
+        for cmd in cmds_to_rate_limit:
             try:
                 await conversation_command_rate_limiter.update_and_check_if_valid(request, cmd)
                 q = q.replace(f"/{cmd.value}", "").strip()

khoj/routers/auth.py

@@ -4,7 +4,8 @@ import logging
 import os
 from typing import Optional
 
-from fastapi import APIRouter
+import requests
+from fastapi import APIRouter, Depends
 from pydantic import BaseModel, EmailStr
 from starlette.authentication import requires
 from starlette.config import Config
@@ -21,7 +22,11 @@ from khoj.database.adapters import (
     get_or_create_user,
 )
 from khoj.routers.email import send_magic_link_email, send_welcome_email
-from khoj.routers.helpers import get_next_url, update_telemetry_state
+from khoj.routers.helpers import (
+    EmailVerificationApiRateLimiter,
+    get_next_url,
+    update_telemetry_state,
+)
 from khoj.utils import state
 
 logger = logging.getLogger(__name__)
@@ -98,16 +103,28 @@ async def login_magic_link(request: Request, form: MagicLinkForm):
 
 
 @auth_router.get("/magic")
-async def sign_in_with_magic_link(request: Request, code: str):
-    user = await aget_user_validated_by_email_verification_code(code)
+async def sign_in_with_magic_link(
+    request: Request,
+    code: str,
+    email: str,
+    rate_limiter=Depends(
+        EmailVerificationApiRateLimiter(requests=10, window=60 * 60 * 24, slug="magic_link_verification")
+    ),
+):
+    user, code_is_expired = await aget_user_validated_by_email_verification_code(code, email)
+
     if user:
+        if code_is_expired:
+            request.session["user"] = {}
+            return Response(status_code=403)
+
         id_info = {
             "email": user.email,
         }
 
         request.session["user"] = dict(id_info)
         return RedirectResponse(url="/")
-    return RedirectResponse(request.app.url_path_for("login_page"))
+    return Response(status_code=401)
 
 
 @auth_router.post("/token")
@@ -140,11 +157,12 @@ async def delete_token(request: Request, token: str):
 
 
 @auth_router.post("/redirect")
-async def auth(request: Request):
+async def auth_post(request: Request):
+    # This is maintained for compatibility with the /login endpoint
     form = await request.form()
     next_url = get_next_url(request)
     for q in request.query_params:
-        if not q == "next":
+        if q != "next":
             next_url += f"&{q}={request.query_params[q]}"
 
     credential = form.get("credential")
@@ -183,7 +201,76 @@ async def auth(request: Request):
     return RedirectResponse(url=next_url, status_code=HTTP_302_FOUND)
 
 
+@auth_router.get("/redirect")
+async def auth(request: Request):
+    next_url = get_next_url(request)
+    for q in request.query_params:
+        if q in ["code", "state", "scope", "authuser", "prompt", "session_state", "access_type"]:
+            continue
+        if q != "next":
+            next_url += f"&{q}={request.query_params[q]}"
+
+    code = request.query_params.get("code")
+
+    # 1. Construct the full redirect URI including domain
+    base_url = str(request.base_url).rstrip("/")
+    redirect_uri = f"{base_url}{request.app.url_path_for('auth')}"
+
+    verified_data = requests.post(
+        "https://oauth2.googleapis.com/token",
+        headers={"Content-Type": "application/x-www-form-urlencoded"},
+        data={
+            "code": code,
+            "client_id": os.environ["GOOGLE_CLIENT_ID"],
+            "client_secret": os.environ["GOOGLE_CLIENT_SECRET"],
+            "redirect_uri": redirect_uri,
+            "grant_type": "authorization_code",
+        },
+    )
+
+    verified_data.raise_for_status()
+
+    credential = verified_data.json().get("id_token")
+
+    if not credential:
+        logger.error("Missing id_token in OAuth response")
+        return RedirectResponse(url="/login?error=invalid_token", status_code=HTTP_302_FOUND)
+
+    try:
+        idinfo = id_token.verify_oauth2_token(credential, google_requests.Request(), os.environ["GOOGLE_CLIENT_ID"])
+    except OAuthError as error:
+        return HTMLResponse(f"<h1>{error.error}</h1>")
+    khoj_user = await get_or_create_user(idinfo)
+
+    if khoj_user:
+        request.session["user"] = dict(idinfo)
+
+        if datetime.timedelta(minutes=3) > (datetime.datetime.now(datetime.timezone.utc) - khoj_user.date_joined):
+            asyncio.create_task(send_welcome_email(idinfo["name"], idinfo["email"]))
+            update_telemetry_state(
+                request=request,
+                telemetry_type="api",
+                api="create_user__google",
+                metadata={"server_id": str(khoj_user.uuid)},
+            )
+            logger.log(logging.INFO, f"🥳 New User Created: {khoj_user.uuid}")
+
+    return RedirectResponse(url=next_url, status_code=HTTP_302_FOUND)
+
+
 @auth_router.get("/logout")
 async def logout(request: Request):
     request.session.pop("user", None)
     return RedirectResponse(url="/")
+
+
+@auth_router.get("/oauth/metadata")
+async def oauth_metadata(request: Request):
+    redirect_uri = str(request.app.url_path_for("auth"))
+
+    return {
+        "google": {
+            "client_id": os.environ.get("GOOGLE_CLIENT_ID"),
+            "redirect_uri": f"{redirect_uri}",
+        }
+    }

khoj/routers/email.py

@@ -1,12 +1,8 @@
 import logging
 import os
 
-try:
-    import resend
-except ImportError:
-    pass
-
 import markdown_it
+import resend
 from django.conf import settings
 from jinja2 import Environment, FileSystemLoader
 
@@ -23,7 +19,7 @@ static_files = os.path.join(settings.BASE_DIR, "static")
 env = Environment(loader=FileSystemLoader(static_files))
 
 if not RESEND_API_KEY:
-    logger.warn("RESEND_API_KEY not set - email sending disabled")
+    logger.warning("RESEND_API_KEY not set - email sending disabled")
 else:
     resend.api_key = RESEND_API_KEY
 
@@ -33,7 +29,7 @@ def is_resend_enabled():
 
 
 async def send_magic_link_email(email, unique_id, host):
-    sign_in_link = f"{host}auth/magic?code={unique_id}"
+    sign_in_link = f"{host}auth/magic?code={unique_id}&email={email}"
 
     if not is_resend_enabled():
         logger.debug(f"Email sending disabled. Share this sign-in link with the user: {sign_in_link}")
@@ -41,13 +37,13 @@ async def send_magic_link_email(email, unique_id, host):
 
     template = env.get_template("magic_link.html")
 
-    html_content = template.render(link=f"{host}auth/magic?code={unique_id}")
+    html_content = template.render(link=f"{host}auth/magic?code={unique_id}", code=unique_id)
 
     resend.Emails.send(
         {
             "sender": os.environ.get("RESEND_EMAIL", "noreply@khoj.dev"),
             "to": email,
-            "subject": "Your Sign-In Link for Khoj 🚀",
+            "subject": f"Your login code to Khoj",
             "html": html_content,
         }
     )
@@ -64,7 +60,7 @@ async def send_welcome_email(name, email):
 
     resend.Emails.send(
         {
-            "sender": "team@khoj.dev",
+            "sender": os.environ.get("RESEND_EMAIL", "team@khoj.dev"),
             "to": email,
             "subject": f"{name}, four ways to use Khoj" if name else "Four ways to use Khoj",
             "html": html_content,
@@ -92,7 +88,7 @@ async def send_query_feedback(uquery, kquery, sentiment, user_email):
 
     logger.info(f"Sending feedback email for query {uquery}")
 
-    # rendering feedback email using feedback.html as template
+    # render feedback email using feedback.html as template
     template = env.get_template("feedback.html")
     html_content = template.render(
         uquery=uquery if not is_none_or_empty(uquery) else "N/A",
@@ -100,10 +96,10 @@ async def send_query_feedback(uquery, kquery, sentiment, user_email):
         sentiment=sentiment if not is_none_or_empty(sentiment) else "N/A",
         user_email=user_email if not is_none_or_empty(user_email) else "N/A",
     )
-    # send feedback from two fixed accounts
+    # send feedback to fixed account
     r = resend.Emails.send(
         {
-            "sender": "saba@khoj.dev",
+            "sender": os.environ.get("RESEND_EMAIL", "noreply@khoj.dev"),
             "to": "team@khoj.dev",
             "subject": f"User Feedback",
             "html": html_content,
@@ -130,7 +126,7 @@ def send_task_email(name, email, query, result, subject, is_image=False):
 
     r = resend.Emails.send(
         {
-            "sender": "Khoj <khoj@khoj.dev>",
+            "sender": f'Khoj <{os.environ.get("RESEND_EMAIL", "khoj@khoj.dev")}>',
             "to": email,
             "subject": f"✨ {subject}",
             "html": html_content,

khoj/routers/helpers.py

@@ -49,6 +49,7 @@ from khoj.database.adapters import (
     ais_user_subscribed,
     create_khoj_token,
     get_khoj_tokens,
+    get_user_by_email,
     get_user_name,
     get_user_notion_config,
     get_user_subscription_state,
@@ -230,7 +231,7 @@ def get_next_url(request: Request) -> str:
     return urljoin(str(request.base_url).rstrip("/"), next_path)
 
 
-def get_conversation_command(query: str, any_references: bool = False) -> ConversationCommand:
+def get_conversation_command(query: str) -> ConversationCommand:
     if query.startswith("/notes"):
         return ConversationCommand.Notes
     elif query.startswith("/help"):
@@ -253,9 +254,6 @@ def get_conversation_command(query: str, any_references: bool = False) -> Conver
         return ConversationCommand.Code
     elif query.startswith("/research"):
         return ConversationCommand.Research
-    # If no relevant notes found for the given query
-    elif not any_references:
-        return ConversationCommand.General
     else:
         return ConversationCommand.Default
 
@@ -407,42 +405,39 @@ async def aget_data_sources_and_output_format(
         response = clean_json(response)
         response = json.loads(response)
 
-        selected_sources = [q.strip() for q in response.get("source", []) if q.strip()]
-        selected_output = response.get("output", "text").strip()  # Default to text output
+        chosen_sources = [s.strip() for s in response.get("source", []) if s.strip()]
+        chosen_output = response.get("output", "text").strip()  # Default to text output
 
-        if not isinstance(selected_sources, list) or not selected_sources or len(selected_sources) == 0:
+        if is_none_or_empty(chosen_sources) or not isinstance(chosen_sources, list):
             raise ValueError(
-                f"Invalid response for determining relevant tools: {selected_sources}. Raw Response: {response}"
+                f"Invalid response for determining relevant tools: {chosen_sources}. Raw Response: {response}"
             )
 
-        result: Dict = {"sources": [], "output": None if not is_task else ConversationCommand.AutomatedTask}
-        for selected_source in selected_sources:
-            # Add a double check to verify it's in the agent list, because the LLM sometimes gets confused by the tool options.
-            if (
-                selected_source in source_options.keys()
-                and isinstance(result["sources"], list)
-                and (len(agent_sources) == 0 or selected_source in agent_sources)
-            ):
-                # Check whether the tool exists as a valid ConversationCommand
-                result["sources"].append(ConversationCommand(selected_source))
-
-        # Add a double check to verify it's in the agent list, because the LLM sometimes gets confused by the tool options.
-        if selected_output in output_options.keys() and (len(agent_outputs) == 0 or selected_output in agent_outputs):
-            # Check whether the tool exists as a valid ConversationCommand
-            result["output"] = ConversationCommand(selected_output)
-
-        if is_none_or_empty(result):
+        output_mode = ConversationCommand.Text
+        # Verify selected output mode is enabled for the agent, as the LLM can sometimes get confused by the tool options.
+        if chosen_output in output_options.keys() and (len(agent_outputs) == 0 or chosen_output in agent_outputs):
+            # Ensure that the chosen output mode exists as a valid ConversationCommand
+            output_mode = ConversationCommand(chosen_output)
+
+        data_sources = []
+        # Verify selected data sources are enabled for the agent, as the LLM can sometimes get confused by the tool options.
+        for chosen_source in chosen_sources:
+            # Ensure that the chosen data source exists as a valid ConversationCommand
+            if chosen_source in source_options.keys() and (len(agent_sources) == 0 or chosen_source in agent_sources):
+                data_sources.append(ConversationCommand(chosen_source))
+
+        # Fallback to default sources if the inferred data sources are unset or invalid
+        if is_none_or_empty(data_sources):
             if len(agent_sources) == 0:
-                result = {"sources": [ConversationCommand.Default], "output": ConversationCommand.Text}
+                data_sources = [ConversationCommand.Default]
             else:
-                result = {"sources": [ConversationCommand.General], "output": ConversationCommand.Text}
+                data_sources = [ConversationCommand.General]
     except Exception as e:
         logger.error(f"Invalid response for determining relevant tools: {response}. Error: {e}", exc_info=True)
-        sources = agent_sources if len(agent_sources) > 0 else [ConversationCommand.Default]
-        output = agent_outputs[0] if len(agent_outputs) > 0 else ConversationCommand.Text
-        result = {"sources": sources, "output": output}
+        data_sources = agent_sources if len(agent_sources) > 0 else [ConversationCommand.Default]
+        output_mode = agent_outputs[0] if len(agent_outputs) > 0 else ConversationCommand.Text
 
-    return result
+    return {"sources": data_sources, "output": output_mode}
 
 
 async def infer_webpage_urls(
@@ -1363,6 +1358,49 @@ class FeedbackData(BaseModel):
     sentiment: str
 
 
+class EmailVerificationApiRateLimiter:
+    def __init__(self, requests: int, window: int, slug: str):
+        self.requests = requests
+        self.window = window
+        self.slug = slug
+
+    def __call__(self, request: Request):
+        # Rate limiting disabled if billing is disabled
+        if state.billing_enabled is False:
+            return
+
+        # Extract the email query parameter
+        email = request.query_params.get("email")
+
+        if email:
+            logger.info(f"Email query parameter: {email}")
+
+        user: KhojUser = get_user_by_email(email)
+
+        if not user:
+            raise HTTPException(
+                status_code=404,
+                detail="User not found.",
+            )
+
+        # Remove requests outside of the time window
+        cutoff = datetime.now(tz=timezone.utc) - timedelta(seconds=self.window)
+        count_requests = UserRequests.objects.filter(user=user, created_at__gte=cutoff, slug=self.slug).count()
+
+        # Check if the user has exceeded the rate limit
+        if count_requests >= self.requests:
+            logger.info(
+                f"Rate limit: {count_requests}/{self.requests} requests not allowed in {self.window} seconds for email: {email}."
+            )
+            raise HTTPException(
+                status_code=429,
+                detail="Ran out of login attempts",
+            )
+
+        # Add the current request to the db
+        UserRequests.objects.create(user=user, slug=self.slug)
+
+
 class ApiUserRateLimiter:
     def __init__(self, requests: int, subscribed_requests: int, window: int, slug: str):
         self.requests = requests
@@ -1642,7 +1680,7 @@ def scheduled_chat(
             last_run_time = datetime.strptime(last_run_time, "%Y-%m-%d %I:%M %p %Z").replace(tzinfo=timezone.utc)
 
             # If the last run time was within the last 6 hours, don't run it again. This helps avoid multithreading issues and rate limits.
-            if (datetime.now(timezone.utc) - last_run_time).total_seconds() < 21600:
+            if (datetime.now(timezone.utc) - last_run_time).total_seconds() < 6 * 60 * 60:
                 logger.info(f"Skipping scheduled chat {job_id} as the next run time is in the future.")
                 return
 

khoj/routers/web_client.py

@@ -57,7 +57,7 @@ def login_page(request: Request):
     if request.user.is_authenticated:
         return RedirectResponse(url=next_url)
     google_client_id = os.environ.get("GOOGLE_CLIENT_ID")
-    redirect_uri = str(request.app.url_path_for("auth"))
+    redirect_uri = str(request.app.url_path_for("auth_post"))
     return templates.TemplateResponse(
         "login.html",
         context={

{khoj-1.31.1.dev13.dist-info → khoj-1.32.0.dist-info}/METADATA

@@ -1,12 +1,13 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: khoj
-Version: 1.31.1.dev13
+Version: 1.32.0
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
 Project-URL: Code, https://github.com/khoj-ai/khoj
 Author: Debanjum Singh Solanky, Saba Imran
-License: AGPL-3.0-or-later
+License-Expression: AGPL-3.0-or-later
+License-File: LICENSE
 Keywords: AI,NLP,images,markdown,org-mode,pdf,productivity,search,semantic-search
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Intended Audience :: Information Technology
@@ -63,6 +64,7 @@ Requires-Dist: pytz~=2024.1
 Requires-Dist: pyyaml~=6.0
 Requires-Dist: rapidocr-onnxruntime==1.3.24
 Requires-Dist: requests>=2.26.0
+Requires-Dist: resend==1.0.1
 Requires-Dist: rich>=13.3.1
 Requires-Dist: schedule==1.1.0
 Requires-Dist: sentence-transformers==3.0.1
@@ -90,14 +92,12 @@ Requires-Dist: pytest-asyncio==0.21.1; extra == 'dev'
 Requires-Dist: pytest-django==4.5.2; extra == 'dev'
 Requires-Dist: pytest-xdist[psutil]; extra == 'dev'
 Requires-Dist: pytest>=7.1.2; extra == 'dev'
-Requires-Dist: resend==1.0.1; extra == 'dev'
 Requires-Dist: stripe==7.3.0; extra == 'dev'
 Requires-Dist: twilio==8.11; extra == 'dev'
 Provides-Extra: prod
 Requires-Dist: boto3>=1.34.57; extra == 'prod'
 Requires-Dist: google-auth==2.23.3; extra == 'prod'
 Requires-Dist: gunicorn==22.0.0; extra == 'prod'
-Requires-Dist: resend==1.0.1; extra == 'prod'
 Requires-Dist: stripe==7.3.0; extra == 'prod'
 Requires-Dist: twilio==8.11; extra == 'prod'
 Description-Content-Type: text/markdown