khoj 1.16.1.dev15__py3-none-any.whl

khoj/main.py

@@ -0,0 +1,241 @@
+""" Main module for Khoj Assistant
+   isort:skip_file
+"""
+
+from contextlib import redirect_stdout
+import logging
+import io
+import os
+import atexit
+import sys
+import locale
+
+from rich.logging import RichHandler
+import threading
+import warnings
+from importlib.metadata import version
+
+from khoj.utils.helpers import in_debug_mode, is_env_var_true
+
+# Ignore non-actionable warnings
+warnings.filterwarnings("ignore", message=r"snapshot_download.py has been made private", category=FutureWarning)
+warnings.filterwarnings("ignore", message=r"legacy way to download files from the HF hub,", category=FutureWarning)
+
+
+import uvicorn
+import django
+from apscheduler.schedulers.background import BackgroundScheduler
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+import schedule
+
+from django.core.asgi import get_asgi_application
+from django.core.management import call_command
+
+# Initialize Django
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
+django.setup()
+
+# Setup Logger
+rich_handler = RichHandler(rich_tracebacks=True)
+rich_handler.setFormatter(fmt=logging.Formatter(fmt="%(name)s: %(message)s", datefmt="[%H:%M:%S.%f]"))
+logging.basicConfig(handlers=[rich_handler])
+
+logging.getLogger("uvicorn.error").setLevel(logging.INFO)
+
+logger = logging.getLogger("khoj")
+
+# Initialize Django Database
+db_migrate_output = io.StringIO()
+with redirect_stdout(db_migrate_output):
+    call_command("migrate", "--noinput")
+
+# Initialize Django Static Files
+collectstatic_output = io.StringIO()
+with redirect_stdout(collectstatic_output):
+    call_command("collectstatic", "--noinput")
+
+# Initialize the Application Server
+if in_debug_mode():
+    app = FastAPI(debug=True)
+else:
+    app = FastAPI(docs_url=None)  # Disable Swagger UI in production
+
+# Get Django Application
+django_app = get_asgi_application()
+
+# Add CORS middleware
+KHOJ_DOMAIN = os.getenv("KHOJ_DOMAIN", "app.khoj.dev")
+scheme = "https" if not is_env_var_true("KHOJ_NO_HTTPS") else "http"
+custom_origins = [f"{scheme}://{KHOJ_DOMAIN.strip()}", f"{scheme}://{KHOJ_DOMAIN.strip()}:*"]
+default_origins = [
+    "app://obsidian.md",  # To allow access from Obsidian desktop app
+    "capacitor://localhost",  # To allow access from Obsidian iOS app using Capacitor.JS
+    "http://localhost",  # To allow access from Obsidian Android app
+    "http://localhost:*",  # To allow access from localhost
+    "http://127.0.0.1:*",  # To allow access from localhost
+    "app://khoj.dev",  # To allow access from Khoj desktop app
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=default_origins + custom_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Set Locale
+locale.setlocale(locale.LC_ALL, "")
+
+# We import these packages after setting up Django so that Django features are accessible to the app.
+from khoj.configure import configure_routes, initialize_server, configure_middleware
+from khoj.utils import state
+from khoj.utils.cli import cli
+from khoj.utils.initialization import initialization
+from khoj.database.adapters import ProcessLockAdapters
+from khoj.database.models import ProcessLock
+
+from django.db.utils import IntegrityError
+
+SCHEDULE_LEADER_NAME = ProcessLock.Operation.SCHEDULE_LEADER
+
+
+def shutdown_scheduler():
+    logger.info("🌑 Shutting down Khoj")
+
+    if state.schedule_leader_process_lock:
+        logger.info("🔓 Schedule Leader released")
+        ProcessLockAdapters.remove_process_lock(state.schedule_leader_process_lock)
+
+    state.scheduler.shutdown()
+
+
+def run(should_start_server=True):
+    # Turn Tokenizers Parallelism Off. App does not support it.
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+    # Load config from CLI
+    state.cli_args = sys.argv[1:]
+    args = cli(state.cli_args)
+    set_state(args)
+
+    # Set Logging Level
+    if args.verbose == 0:
+        logger.setLevel(logging.INFO)
+    elif args.verbose >= 1:
+        logger.setLevel(logging.DEBUG)
+
+    logger.info(f"🚒 Initializing Khoj v{state.khoj_version}")
+    logger.info(f"📦 Initializing DB:\n{db_migrate_output.getvalue().strip()}")
+    logger.debug(f"🌍 Initializing Web Client:\n{collectstatic_output.getvalue().strip()}")
+
+    initialization()
+
+    # Create app directory, if it doesn't exist
+    state.config_file.parent.mkdir(parents=True, exist_ok=True)
+
+    # Set Log File
+    fh = logging.FileHandler(state.config_file.parent / "khoj.log", encoding="utf-8")
+    fh.setLevel(logging.DEBUG)
+    logger.addHandler(fh)
+
+    logger.info("🌘 Starting Khoj")
+
+    # Setup task scheduler
+    poll_task_scheduler()
+
+    # Setup Background Scheduler
+    from django_apscheduler.jobstores import DjangoJobStore
+
+    state.scheduler = BackgroundScheduler(
+        {
+            "apscheduler.timezone": "UTC",
+            "apscheduler.job_defaults.misfire_grace_time": "60",  # Useful to run scheduled jobs even when worker delayed because it was busy or down
+            "apscheduler.job_defaults.coalesce": "true",  # Combine multiple jobs into one if they are scheduled at the same time
+        }
+    )
+    state.scheduler.add_jobstore(DjangoJobStore(), "default")
+
+    # We use this mechanism to only elect one schedule leader in a distributed environment. This one will be responsible for actually executing the scheduled tasks. The others will still be capable of adding and removing tasks, but they will not execute them. This is to decrease the overall burden on the database and the system.
+    try:
+        schedule_leader_process_lock = ProcessLockAdapters.get_process_lock(SCHEDULE_LEADER_NAME)
+        if schedule_leader_process_lock:
+            logger.info("🔒 Schedule Leader is already running")
+            state.scheduler.start(paused=True)
+        else:
+            logger.info("🔒 Schedule Leader elected")
+            created_process_lock = ProcessLockAdapters.set_process_lock(
+                SCHEDULE_LEADER_NAME, max_duration_in_seconds=43200
+            )
+            state.scheduler.start()
+            state.schedule_leader_process_lock = created_process_lock
+    except IntegrityError:
+        logger.info("🔒 Schedule Leader running elsewhere")
+        state.scheduler.start(paused=True)
+    finally:
+        logger.info("Started Background Scheduler")
+
+    # Start Server
+    configure_routes(app)
+
+    #  Mount Django and Static Files
+    app.mount("/server", django_app, name="server")
+    static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
+    if not os.path.exists(static_dir):
+        os.mkdir(static_dir)
+    app.mount(f"/static", StaticFiles(directory=static_dir), name=static_dir)
+
+    # Configure Middleware
+    configure_middleware(app)
+
+    initialize_server(args.config)
+
+    # If the server is started through gunicorn (external to the script), don't start the server
+    if should_start_server:
+        start_server(app, host=args.host, port=args.port, socket=args.socket)
+        # Teardown
+        shutdown_scheduler()
+
+
+def set_state(args):
+    state.config_file = args.config_file
+    state.config = args.config
+    state.verbose = args.verbose
+    state.host = args.host
+    state.port = args.port
+    state.anonymous_mode = args.anonymous_mode
+    state.khoj_version = version("khoj")
+    state.chat_on_gpu = args.chat_on_gpu
+
+
+def start_server(app, host=None, port=None, socket=None):
+    logger.info("🌖 Khoj is ready to use")
+    if socket:
+        uvicorn.run(app, proxy_headers=True, uds=socket, log_level="debug", use_colors=True, log_config=None)
+    else:
+        uvicorn.run(
+            app,
+            host=host,
+            port=port,
+            log_level="debug" if state.verbose > 1 else "info",
+            use_colors=True,
+            log_config=None,
+            timeout_keep_alive=60,
+        )
+    logger.info("🌒 Stopping Khoj")
+
+
+def poll_task_scheduler():
+    timer_thread = threading.Timer(60.0, poll_task_scheduler)
+    timer_thread.daemon = True
+    timer_thread.start()
+    schedule.run_pending()
+
+
+if __name__ == "__main__":
+    run()
+else:
+    run(should_start_server=False)
+    atexit.register(shutdown_scheduler)

khoj/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "khoj.app.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == "__main__":
+    main()

khoj/migrations/migrate_offline_chat_default_model.py

@@ -0,0 +1,69 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
+    ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+    ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_default_model(args):
+    schema_version = "0.12.4"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+    if "offline-chat" not in raw_config["processor"]["conversation"]:
+        return args
+    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.12.4"):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
+        )
+        raw_config["version"] = schema_version
+
+        # Update offline chat model to mistral in GGUF format to use latest GPT4All
+        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
+        if offline_chat_model.endswith(".bin"):
+            raw_config["processor"]["conversation"]["offline-chat"]["chat-model"] = "mistral-7b-instruct-v0.1.Q4_0.gguf"
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_chat_default_model_2.py

@@ -0,0 +1,71 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+    ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: NousResearch/Hermes-2-Pro-Mistral-7B-GGUF
+    ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_default_model(args):
+    schema_version = "1.7.0"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+    if "offline-chat" not in raw_config["processor"]["conversation"]:
+        return args
+    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse(schema_version):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
+        )
+        raw_config["version"] = schema_version
+
+        # Update offline chat model to use Nous Research's Hermes-2-Pro GGUF in path format suitable for llama-cpp
+        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
+        if offline_chat_model == "mistral-7b-instruct-v0.1.Q4_0.gguf":
+            raw_config["processor"]["conversation"]["offline-chat"][
+                "chat-model"
+            ] = "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF"
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_chat_schema.py

@@ -0,0 +1,83 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    enable-offline-chat: false
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    openai:
+        ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
+    tokenizer: null
+    max_prompt_size: null
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    openai:
+        ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_schema(args):
+    schema_version = "0.12.3"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.12.3"):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to make (offline) chat more configuration"
+        )
+        raw_config["version"] = schema_version
+
+        # Create max-prompt-size field in conversation processor schema
+        raw_config["processor"]["conversation"]["max-prompt-size"] = None
+        raw_config["processor"]["conversation"]["tokenizer"] = None
+
+        # Create offline chat schema based on existing enable_offline_chat field in khoj config schema
+        offline_chat_model = (
+            raw_config["processor"]["conversation"]
+            .get("offline-chat", {})
+            .get("chat-model", "llama-2-7b-chat.ggmlv3.q4_0.bin")
+        )
+        raw_config["processor"]["conversation"]["offline-chat"] = {
+            "enable-offline-chat": raw_config["processor"]["conversation"].get("enable-offline-chat", False),
+            "chat-model": offline_chat_model,
+        }
+
+        # Delete old enable-offline-chat field from conversation processor schema
+        if "enable-offline-chat" in raw_config["processor"]["conversation"]:
+            del raw_config["processor"]["conversation"]["enable-offline-chat"]
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_model.py

@@ -0,0 +1,29 @@
+import logging
+import os
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_model(args):
+    schema_version = "0.10.1"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.10.1"):
+        logger.info(
+            f"Migrating offline model used for version {previous_version} to latest version for {args.version_no}"
+        )
+        raw_config["version"] = schema_version
+
+        # If the user has downloaded the offline model, remove it from the cache.
+        offline_model_path = os.path.expanduser("~/.cache/gpt4all/llama-2-7b-chat.ggmlv3.q4_K_S.bin")
+        if os.path.exists(offline_model_path):
+            os.remove(offline_model_path)
+
+        save_config_to_file(raw_config, args.config_file)
+
+    return args

khoj/migrations/migrate_processor_config_openai.py

@@ -0,0 +1,67 @@
+"""
+Current format of khoj.yml
+---
+app:
+  should-log-telemetry: true
+content-type:
+    ...
+processor:
+  conversation:
+    chat-model: gpt-3.5-turbo
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    model: text-davinci-003
+    openai-api-key: sk-secret-key
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+  should-log-telemetry: true
+content-type:
+    ...
+processor:
+  conversation:
+    openai:
+        chat-model: gpt-3.5-turbo
+        openai-api-key: sk-secret-key
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    enable-offline-chat: false
+search-type:
+    ...
+"""
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+
+def migrate_processor_conversation_schema(args):
+    schema_version = "0.10.0"
+    raw_config = load_config_from_file(args.config_file)
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+
+    current_openai_api_key = raw_config["processor"]["conversation"].get("openai-api-key", None)
+    current_chat_model = raw_config["processor"]["conversation"].get("chat-model", None)
+    if current_openai_api_key is None and current_chat_model is None:
+        return args
+
+    raw_config["version"] = schema_version
+
+    # Add enable_offline_chat to khoj config schema
+    if "enable-offline-chat" not in raw_config["processor"]["conversation"]:
+        raw_config["processor"]["conversation"]["enable-offline-chat"] = False
+
+    # Update conversation processor schema
+    conversation_logfile = raw_config["processor"]["conversation"].get("conversation-logfile", None)
+    raw_config["processor"]["conversation"] = {
+        "openai": {"chat-model": current_chat_model, "api-key": current_openai_api_key},
+        "conversation-logfile": conversation_logfile,
+        "enable-offline-chat": False,
+    }
+
+    save_config_to_file(raw_config, args.config_file)
+    return args