khoj 1.24.2.dev3__py3-none-any.whl → 1.25.1.dev34__py3-none-any.whl

khoj/routers/subscription.py

@@ -7,6 +7,7 @@ from fastapi import APIRouter, Request
 from starlette.authentication import requires
 
 from khoj.database import adapters
+from khoj.routers.helpers import update_telemetry_state
 from khoj.utils import state
 
 # Stripe integration for Khoj Cloud Subscription
@@ -48,6 +49,8 @@ async def subscribe(request: Request):
     customer_id = subscription["customer"]
     customer = stripe.Customer.retrieve(customer_id)
     customer_email = customer["email"]
+    user = None
+    is_new = False
 
     # Handle valid stripe webhook events
     success = True
@@ -55,7 +58,9 @@ async def subscribe(request: Request):
         # Mark the user as subscribed and update the next renewal date on payment
         subscription = stripe.Subscription.list(customer=customer_id).data[0]
         renewal_date = datetime.fromtimestamp(subscription["current_period_end"], tz=timezone.utc)
-        user = await adapters.set_user_subscription(customer_email, is_recurring=True, renewal_date=renewal_date)
+        user, is_new = await adapters.set_user_subscription(
+            customer_email, is_recurring=True, renewal_date=renewal_date
+        )
         success = user is not None
     elif event_type in {"customer.subscription.updated"}:
         user_subscription = await sync_to_async(adapters.get_user_subscription)(customer_email)
@@ -63,15 +68,24 @@ async def subscribe(request: Request):
         if user_subscription and user_subscription.renewal_date:
             # Mark user as unsubscribed or resubscribed
             is_recurring = not subscription["cancel_at_period_end"]
-            updated_user = await adapters.set_user_subscription(customer_email, is_recurring=is_recurring)
-            success = updated_user is not None
+            user, is_new = await adapters.set_user_subscription(customer_email, is_recurring=is_recurring)
+            success = user is not None
     elif event_type in {"customer.subscription.deleted"}:
         # Reset the user to trial state
-        user = await adapters.set_user_subscription(
+        user, is_new = await adapters.set_user_subscription(
             customer_email, is_recurring=False, renewal_date=False, type="trial"
         )
         success = user is not None
 
+    if user and is_new:
+        update_telemetry_state(
+            request=request,
+            telemetry_type="api",
+            api="create_user",
+            metadata={"user_id": str(user.user.uuid)},
+        )
+        logger.log(logging.INFO, f"🥳 New User Created: {user.user.uuid}")
+
     logger.info(f'Stripe subscription {event["type"]} for {customer_email}')
     return {"success": success}
 

khoj/search_type/text_search.py

@@ -1,13 +1,15 @@
 import logging
 import math
 from pathlib import Path
-from typing import List, Tuple, Type, Union
+from typing import List, Optional, Tuple, Type, Union
 
+import requests
 import torch
 from asgiref.sync import sync_to_async
 from sentence_transformers import util
 
 from khoj.database.adapters import EntryAdapters, get_user_search_model_or_default
+from khoj.database.models import Agent
 from khoj.database.models import Entry as DbEntry
 from khoj.database.models import KhojUser
 from khoj.processor.content.text_to_entries import TextToEntries
@@ -101,6 +103,7 @@ async def query(
     type: SearchType = SearchType.All,
     question_embedding: Union[torch.Tensor, None] = None,
     max_distance: float = None,
+    agent: Optional[Agent] = None,
 ) -> Tuple[List[dict], List[Entry]]:
     "Search for entries that answer the query"
 
@@ -129,6 +132,7 @@ async def query(
             file_type_filter=file_type,
             raw_query=raw_query,
             max_distance=max_distance,
+            agent=agent,
         ).all()
         hits = await sync_to_async(list)(hits)  # type: ignore[call-arg]
 
@@ -228,8 +232,12 @@ def setup(
 
 def cross_encoder_score(query: str, hits: List[SearchResponse], search_model_name: str) -> List[SearchResponse]:
     """Score all retrieved entries using the cross-encoder"""
-    with timer("Cross-Encoder Predict Time", logger, state.device):
-        cross_scores = state.cross_encoder_model[search_model_name].predict(query, hits)
+    try:
+        with timer("Cross-Encoder Predict Time", logger, state.device):
+            cross_scores = state.cross_encoder_model[search_model_name].predict(query, hits)
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"Failed to rerank documents using the inference endpoint. Error: {e}.", exc_info=True)
+        cross_scores = [0.0] * len(hits)
 
     # Convert cross-encoder scores to distances and pass in hits for reranking
     for idx in range(len(cross_scores)):

khoj/utils/helpers.py

@@ -2,10 +2,12 @@ from __future__ import annotations  # to avoid quoting type hints
 
 import datetime
 import io
+import ipaddress
 import logging
 import os
 import platform
 import random
+import urllib.parse
 import uuid
 from collections import OrderedDict
 from enum import Enum
@@ -164,9 +166,9 @@ def get_class_by_name(name: str) -> object:
 class timer:
     """Context manager to log time taken for a block of code to run"""
 
-    def __init__(self, message: str, logger: logging.Logger, device: torch.device = None):
+    def __init__(self, message: str, logger: logging.Logger, device: torch.device = None, log_level=logging.DEBUG):
         self.message = message
-        self.logger = logger
+        self.logger = logger.debug if log_level == logging.DEBUG else logger.info
         self.device = device
 
     def __enter__(self):
@@ -176,9 +178,9 @@ class timer:
     def __exit__(self, *_):
         elapsed = perf_counter() - self.start
         if self.device is None:
-            self.logger.debug(f"{self.message}: {elapsed:.3f} seconds")
+            self.logger(f"{self.message}: {elapsed:.3f} seconds")
         else:
-            self.logger.debug(f"{self.message}: {elapsed:.3f} seconds on device: {self.device}")
+            self.logger(f"{self.message}: {elapsed:.3f} seconds on device: {self.device}")
 
 
 class LRU(OrderedDict):
@@ -321,11 +323,19 @@ command_descriptions = {
     ConversationCommand.Notes: "Only talk about information that is available in your knowledge base.",
     ConversationCommand.Default: "The default command when no command specified. It intelligently auto-switches between general and notes mode.",
     ConversationCommand.Online: "Search for information on the internet.",
-    ConversationCommand.Webpage: "Get information from webpage links provided by you.",
+    ConversationCommand.Webpage: "Get information from webpage suggested by you.",
     ConversationCommand.Image: "Generate images by describing your imagination in words.",
     ConversationCommand.Automation: "Automatically run your query at a specified time or interval.",
     ConversationCommand.Help: "Get help with how to use or setup Khoj from the documentation",
-    ConversationCommand.Summarize: "Create an appropriate summary using provided documents.",
+    ConversationCommand.Summarize: "Get help with a question pertaining to an entire document.",
+}
+
+command_descriptions_for_agent = {
+    ConversationCommand.General: "Agent can use the agents knowledge base and general knowledge.",
+    ConversationCommand.Notes: "Agent can search the users knowledge base for information.",
+    ConversationCommand.Online: "Agent can search the internet for information.",
+    ConversationCommand.Webpage: "Agent can read suggested web pages for information.",
+    ConversationCommand.Summarize: "Agent can read an entire document. Agents knowledge base must be a single document.",
 }
 
 tool_descriptions_for_llm = {
@@ -334,15 +344,21 @@ tool_descriptions_for_llm = {
     ConversationCommand.Notes: "To search the user's personal knowledge base. Especially helpful if the question expects context from the user's notes or documents.",
     ConversationCommand.Online: "To search for the latest, up-to-date information from the internet. Note: **Questions about Khoj should always use this data source**",
     ConversationCommand.Webpage: "To use if the user has directly provided the webpage urls or you are certain of the webpage urls to read.",
-    ConversationCommand.Summarize: "To create a summary of the document provided by the user.",
+    ConversationCommand.Summarize: "To retrieve an answer that depends on the entire document or a large text.",
 }
 
 mode_descriptions_for_llm = {
     ConversationCommand.Image: "Use this if the user is requesting you to generate a picture based on their description.",
-    ConversationCommand.Automation: "Use this if the user is requesting a response at a scheduled date or time.",
+    ConversationCommand.Automation: "Use this if you are confident the user is requesting a response at a scheduled date, time and frequency",
     ConversationCommand.Text: "Use this if the other response modes don't seem to fit the query.",
 }
 
+mode_descriptions_for_agent = {
+    ConversationCommand.Image: "Agent can generate image in response.",
+    ConversationCommand.Automation: "Agent can schedule a task to run at a scheduled date, time and frequency in response.",
+    ConversationCommand.Text: "Agent can generate text in response.",
+}
+
 
 class ImageIntentType(Enum):
     """
@@ -422,6 +438,46 @@ def is_internet_connected():
         return False
 
 
+def is_internal_url(url: str) -> bool:
+    """
+    Check if a URL is likely to be internal/non-public.
+
+    Args:
+    url (str): The URL to check.
+
+    Returns:
+    bool: True if the URL is likely internal, False otherwise.
+    """
+    try:
+        parsed_url = urllib.parse.urlparse(url)
+        hostname = parsed_url.hostname
+
+        # Check for localhost
+        if hostname in ["localhost", "127.0.0.1", "::1"]:
+            return True
+
+        # Check for IP addresses in private ranges
+        try:
+            ip = ipaddress.ip_address(hostname)
+            return ip.is_private
+        except ValueError:
+            pass  # Not an IP address, continue with other checks
+
+        # Check for common internal TLDs
+        internal_tlds = [".local", ".internal", ".private", ".corp", ".home", ".lan"]
+        if any(hostname.endswith(tld) for tld in internal_tlds):
+            return True
+
+        # Check for URLs without a TLD
+        if "." not in hostname:
+            return True
+
+        return False
+    except Exception:
+        # If we can't parse the URL or something else goes wrong, assume it's not internal
+        return False
+
+
 def convert_image_to_webp(image_bytes):
     """Convert image bytes to webp format for faster loading"""
     image_io = io.BytesIO(image_bytes)

khoj/utils/initialization.py

@@ -129,9 +129,6 @@ def initialization(interactive: bool = True):
             if user_chat_model_name and ChatModelOptions.objects.filter(chat_model=user_chat_model_name).exists():
                 default_chat_model_name = user_chat_model_name
 
-            # Create a server chat settings object with the default chat model
-            default_chat_model = ChatModelOptions.objects.filter(chat_model=default_chat_model_name).first()
-            ServerChatSettings.objects.create(chat_default=default_chat_model)
             logger.info("🗣️ Chat model configuration complete")
 
         # Set up offline speech to text model

{khoj-1.24.2.dev3.dist-info → khoj-1.25.1.dev34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: khoj
-Version: 1.24.2.dev3
+Version: 1.25.1.dev34
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
@@ -32,7 +32,7 @@ Requires-Dist: dateparser>=1.1.1
 Requires-Dist: defusedxml==0.7.1
 Requires-Dist: django-apscheduler==0.6.2
 Requires-Dist: django-phonenumber-field==7.3.0
-Requires-Dist: django==5.0.8
+Requires-Dist: django==5.0.9
 Requires-Dist: docx2txt==0.8
 Requires-Dist: einops==0.8.0
 Requires-Dist: fastapi>=0.110.0
@@ -103,14 +103,14 @@ Description-Content-Type: text/markdown
 <div align="center">
 
 [![test](https://github.com/khoj-ai/khoj/actions/workflows/test.yml/badge.svg)](https://github.com/khoj-ai/khoj/actions/workflows/test.yml)
-[![dockerize](https://github.com/khoj-ai/khoj/actions/workflows/dockerize.yml/badge.svg)](https://github.com/khoj-ai/khoj/pkgs/container/khoj)
+[![docker](https://github.com/khoj-ai/khoj/actions/workflows/dockerize.yml/badge.svg)](https://github.com/khoj-ai/khoj/pkgs/container/khoj)
 [![pypi](https://github.com/khoj-ai/khoj/actions/workflows/pypi.yml/badge.svg)](https://pypi.org/project/khoj/)
-![Discord](https://img.shields.io/discord/1112065956647284756?style=plastic&label=discord)
+[![discord](https://img.shields.io/discord/1112065956647284756?style=plastic&label=discord)](https://discord.gg/BDgyabRM6e)
 
 </div>
 
 <div align="center">
-<b>The open-source, personal AI for your digital brain</b>
+<b>Your AI second brain</b>
 </div>
 
 <br />
@@ -119,11 +119,13 @@ Description-Content-Type: text/markdown
 
 [📑 Docs](https://docs.khoj.dev)
 <span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
-[🏮 App](https://khoj.dev)
+[🌐 Web](https://khoj.dev)
+<span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
+[🔥 App](https://app.khoj.dev)
 <span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
 [💬 Discord](https://discord.gg/BDgyabRM6e)
 <span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
-[📚 Blog](https://blog.khoj.dev)
+[✍🏽 Blog](https://blog.khoj.dev)
 
 </div>
 
@@ -131,14 +133,17 @@ Description-Content-Type: text/markdown
 
 ***
 
-[Khoj](https://khoj.dev) is a personal, open-source AI application for you to extend your capabilities.
-- Share your documents to extend your digital brain.
-- Access the internet, getting fresh information.
-- You can share pdf, markdown, org-mode, notion files and github repositories.
-- Fast, accurate semantic search on top of your docs.
-- Create images, talk out loud, play your messages.
-- Available Desktop, Emacs, Obsidian, Web and Whatsapp.
+[Khoj](https://khoj.dev) is a personal AI app to extend your capabilities. It smoothly scales up from an on-device personal AI to a cloud-scale enterprise AI.
+
+- Chat with any local or online LLM (e.g llama3, qwen, gemma, mistral, gpt, claude, gemini).
+- Get answers from the internet and your docs (including image, pdf, markdown, org-mode, word, notion files).
+- Access it from your Browser, Obsidian, Emacs, Desktop, Phone or Whatsapp.
+- Create agents with custom knowledge, persona, chat model and tools to take on any role.
+- Automate away repetitive research. Get personal newsletters and smart notifications delivered to your inbox.
+- Find relevant docs quickly and easily using our advanced semantic search.
+- Generate images, talk out loud, play your messages.
 - Khoj is open-source, self-hostable. Always.
+- Run it privately on [your computer](https://docs.khoj.dev/get-started/setup) or try it on our [cloud app](https://app.khoj.dev).
 
 ***
 
@@ -169,10 +174,3 @@ Made with [contrib.rocks](https://contrib.rocks).
 ### Interested in Contributing?
 
 We are always looking for contributors to help us build new features, improve the project documentation, or fix bugs. If you're interested, please see our [Contributing Guidelines](https://docs.khoj.dev/contributing/development) and check out our [Contributors Project Board](https://github.com/orgs/khoj-ai/projects/4).
-
-## [Sponsors](https://github.com/sponsors/khoj-ai)
-Shout out to our brilliant sponsors! 🌈
-
-<a href="http://github.com/beekeeb">
-  <img src="https://raw.githubusercontent.com/beekeeb/piantor/main/docs/beekeeb.png" width=250/>
-</a>