khoj 1.33.3.dev32__py3-none-any.whl

khoj/migrations/migrate_offline_chat_default_model.py

@@ -0,0 +1,69 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
+    ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+    ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_default_model(args):
+    schema_version = "0.12.4"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+    if "offline-chat" not in raw_config["processor"]["conversation"]:
+        return args
+    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.12.4"):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
+        )
+        raw_config["version"] = schema_version
+
+        # Update offline chat model to mistral in GGUF format to use latest GPT4All
+        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
+        if offline_chat_model.endswith(".bin"):
+            raw_config["processor"]["conversation"]["offline-chat"]["chat-model"] = "mistral-7b-instruct-v0.1.Q4_0.gguf"
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_chat_default_model_2.py

@@ -0,0 +1,71 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+    ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: NousResearch/Hermes-2-Pro-Mistral-7B-GGUF
+    ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_default_model(args):
+    schema_version = "1.7.0"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+    if "offline-chat" not in raw_config["processor"]["conversation"]:
+        return args
+    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse(schema_version):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
+        )
+        raw_config["version"] = schema_version
+
+        # Update offline chat model to use Nous Research's Hermes-2-Pro GGUF in path format suitable for llama-cpp
+        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
+        if offline_chat_model == "mistral-7b-instruct-v0.1.Q4_0.gguf":
+            raw_config["processor"]["conversation"]["offline-chat"][
+                "chat-model"
+            ] = "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF"
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_chat_schema.py

@@ -0,0 +1,83 @@
+"""
+Current format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    enable-offline-chat: false
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    openai:
+        ...
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+    ...
+content-type:
+    ...
+processor:
+  conversation:
+    offline-chat:
+        enable-offline-chat: false
+        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
+    tokenizer: null
+    max_prompt_size: null
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    openai:
+        ...
+search-type:
+    ...
+"""
+import logging
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_chat_schema(args):
+    schema_version = "0.12.3"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.12.3"):
+        logger.info(
+            f"Upgrading config schema to {schema_version} from {previous_version} to make (offline) chat more configuration"
+        )
+        raw_config["version"] = schema_version
+
+        # Create max-prompt-size field in conversation processor schema
+        raw_config["processor"]["conversation"]["max-prompt-size"] = None
+        raw_config["processor"]["conversation"]["tokenizer"] = None
+
+        # Create offline chat schema based on existing enable_offline_chat field in khoj config schema
+        offline_chat_model = (
+            raw_config["processor"]["conversation"]
+            .get("offline-chat", {})
+            .get("chat-model", "llama-2-7b-chat.ggmlv3.q4_0.bin")
+        )
+        raw_config["processor"]["conversation"]["offline-chat"] = {
+            "enable-offline-chat": raw_config["processor"]["conversation"].get("enable-offline-chat", False),
+            "chat-model": offline_chat_model,
+        }
+
+        # Delete old enable-offline-chat field from conversation processor schema
+        if "enable-offline-chat" in raw_config["processor"]["conversation"]:
+            del raw_config["processor"]["conversation"]["enable-offline-chat"]
+
+        save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_offline_model.py

@@ -0,0 +1,29 @@
+import logging
+import os
+
+from packaging import version
+
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_offline_model(args):
+    schema_version = "0.10.1"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if previous_version is None or version.parse(previous_version) < version.parse("0.10.1"):
+        logger.info(
+            f"Migrating offline model used for version {previous_version} to latest version for {args.version_no}"
+        )
+        raw_config["version"] = schema_version
+
+        # If the user has downloaded the offline model, remove it from the cache.
+        offline_model_path = os.path.expanduser("~/.cache/gpt4all/llama-2-7b-chat.ggmlv3.q4_K_S.bin")
+        if os.path.exists(offline_model_path):
+            os.remove(offline_model_path)
+
+        save_config_to_file(raw_config, args.config_file)
+
+    return args

khoj/migrations/migrate_processor_config_openai.py

@@ -0,0 +1,67 @@
+"""
+Current format of khoj.yml
+---
+app:
+  should-log-telemetry: true
+content-type:
+    ...
+processor:
+  conversation:
+    chat-model: gpt-3.5-turbo
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    model: text-davinci-003
+    openai-api-key: sk-secret-key
+search-type:
+    ...
+
+New format of khoj.yml
+---
+app:
+  should-log-telemetry: true
+content-type:
+    ...
+processor:
+  conversation:
+    openai:
+        chat-model: gpt-3.5-turbo
+        openai-api-key: sk-secret-key
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    enable-offline-chat: false
+search-type:
+    ...
+"""
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+
+def migrate_processor_conversation_schema(args):
+    schema_version = "0.10.0"
+    raw_config = load_config_from_file(args.config_file)
+
+    if "processor" not in raw_config:
+        return args
+    if raw_config["processor"] is None:
+        return args
+    if "conversation" not in raw_config["processor"]:
+        return args
+
+    current_openai_api_key = raw_config["processor"]["conversation"].get("openai-api-key", None)
+    current_chat_model = raw_config["processor"]["conversation"].get("chat-model", None)
+    if current_openai_api_key is None and current_chat_model is None:
+        return args
+
+    raw_config["version"] = schema_version
+
+    # Add enable_offline_chat to khoj config schema
+    if "enable-offline-chat" not in raw_config["processor"]["conversation"]:
+        raw_config["processor"]["conversation"]["enable-offline-chat"] = False
+
+    # Update conversation processor schema
+    conversation_logfile = raw_config["processor"]["conversation"].get("conversation-logfile", None)
+    raw_config["processor"]["conversation"] = {
+        "openai": {"chat-model": current_chat_model, "api-key": current_openai_api_key},
+        "conversation-logfile": conversation_logfile,
+        "enable-offline-chat": False,
+    }
+
+    save_config_to_file(raw_config, args.config_file)
+    return args

khoj/migrations/migrate_server_pg.py

@@ -0,0 +1,132 @@
+"""
+The application config currently looks like this:
+app:
+  should-log-telemetry: true
+content-type:
+    ...
+processor:
+  conversation:
+    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
+    max-prompt-size: null
+    offline-chat:
+      chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+      enable-offline-chat: false
+    openai:
+      api-key: sk-blah
+      chat-model: gpt-3.5-turbo
+    tokenizer: null
+search-type:
+  asymmetric:
+    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
+    encoder: sentence-transformers/multi-qa-MiniLM-L6-cos-v1
+    encoder-type: null
+    model-directory: /Users/si/.khoj/search/asymmetric
+  image:
+    encoder: sentence-transformers/clip-ViT-B-32
+    encoder-type: null
+    model-directory: /Users/si/.khoj/search/image
+  symmetric:
+    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
+    encoder: sentence-transformers/all-MiniLM-L6-v2
+    encoder-type: null
+    model-directory: ~/.khoj/search/symmetric
+version: 0.14.0
+
+
+The new version will looks like this:
+app:
+  should-log-telemetry: true
+processor:
+  conversation:
+    offline-chat:
+      enabled: false
+    openai:
+      api-key: sk-blah
+    chat-model-options:
+      - chat-model: gpt-3.5-turbo
+        tokenizer: null
+        type: openai
+      - chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
+        tokenizer: null
+        type: offline
+search-type:
+  asymmetric:
+    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
+    encoder: sentence-transformers/multi-qa-MiniLM-L6-cos-v1
+version: 0.15.0
+"""
+
+import logging
+
+from packaging import version
+
+from khoj.database.models import AiModelApi, ChatModel, SearchModelConfig
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+logger = logging.getLogger(__name__)
+
+
+def migrate_server_pg(args):
+    schema_version = "0.15.0"
+    raw_config = load_config_from_file(args.config_file)
+    previous_version = raw_config.get("version")
+
+    if previous_version is None or version.parse(previous_version) < version.parse(schema_version):
+        logger.info(
+            f"Migrating configuration used for version {previous_version} to latest version for server with postgres in {args.version_no}"
+        )
+        raw_config["version"] = schema_version
+
+        if raw_config is None:
+            return args
+
+        if "search-type" in raw_config and raw_config["search-type"]:
+            if "asymmetric" in raw_config["search-type"]:
+                # Delete all existing search models
+                SearchModelConfig.objects.filter(model_type=SearchModelConfig.ModelType.TEXT).delete()
+                # Create new search model from existing Khoj YAML config
+                asymmetric_search = raw_config["search-type"]["asymmetric"]
+                SearchModelConfig.objects.create(
+                    name="default",
+                    model_type=SearchModelConfig.ModelType.TEXT,
+                    bi_encoder=asymmetric_search.get("encoder"),
+                    cross_encoder=asymmetric_search.get("cross-encoder"),
+                )
+
+        if "processor" in raw_config and raw_config["processor"] and "conversation" in raw_config["processor"]:
+            processor_conversation = raw_config["processor"]["conversation"]
+
+            if "offline-chat" in raw_config["processor"]["conversation"]:
+                offline_chat = raw_config["processor"]["conversation"]["offline-chat"]
+                ChatModel.objects.create(
+                    name=offline_chat.get("chat-model"),
+                    tokenizer=processor_conversation.get("tokenizer"),
+                    max_prompt_size=processor_conversation.get("max-prompt-size"),
+                    model_type=ChatModel.ModelType.OFFLINE,
+                )
+
+            if (
+                "openai" in raw_config["processor"]["conversation"]
+                and raw_config["processor"]["conversation"]["openai"]
+            ):
+                openai = raw_config["processor"]["conversation"]["openai"]
+
+                if openai.get("api-key") is None:
+                    logger.error("OpenAI API Key is not set. Will not be migrating OpenAI config.")
+                else:
+                    if openai.get("chat-model") is None:
+                        openai["chat-model"] = "gpt-3.5-turbo"
+
+                    openai_model_api = AiModelApi.objects.create(api_key=openai.get("api-key"), name="default")
+
+                    ChatModel.objects.create(
+                        name=openai.get("chat-model"),
+                        tokenizer=processor_conversation.get("tokenizer"),
+                        max_prompt_size=processor_conversation.get("max-prompt-size"),
+                        model_type=ChatModel.ModelType.OPENAI,
+                        ai_model_api=openai_model_api,
+                    )
+
+        save_config_to_file(raw_config, args.config_file)
+
+    return args

khoj/migrations/migrate_version.py

@@ -0,0 +1,17 @@
+from khoj.utils.yaml import load_config_from_file, save_config_to_file
+
+
+def migrate_config_to_version(args):
+    schema_version = "0.9.0"
+    raw_config = load_config_from_file(args.config_file)
+
+    # Add version to khoj config schema
+    if "version" not in raw_config:
+        raw_config["version"] = schema_version
+        save_config_to_file(raw_config, args.config_file)
+
+        # regenerate khoj index on first start of this version
+        # this should refresh index and apply index corruption fixes from #325
+        args.regenerate = True
+
+    return args

khoj/processor/content/docx/docx_to_entries.py

@@ -0,0 +1,111 @@
+import logging
+import tempfile
+from typing import Dict, List, Tuple
+
+from langchain_community.document_loaders import Docx2txtLoader
+
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
+from khoj.processor.content.text_to_entries import TextToEntries
+from khoj.utils.helpers import timer
+from khoj.utils.rawconfig import Entry
+
+logger = logging.getLogger(__name__)
+
+
+class DocxToEntries(TextToEntries):
+    def __init__(self):
+        super().__init__()
+
+    # Define Functions
+    def process(self, files: dict[str, str], user: KhojUser, regenerate: bool = False) -> Tuple[int, int]:
+        # Extract required fields from config
+        deletion_file_names = set([file for file in files if files[file] == b""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
+
+        # Extract Entries from specified Docx files
+        with timer("Extract entries from specified DOCX files", logger):
+            file_to_text_map, current_entries = DocxToEntries.extract_docx_entries(files)
+
+        # Split entries by max tokens supported by model
+        with timer("Split entries by max token size supported by model", logger):
+            current_entries = self.split_entries_by_max_tokens(current_entries, max_tokens=256)
+
+        # Identify, mark and merge any new entries with previous entries
+        with timer("Identify new or updated entries", logger):
+            num_new_embeddings, num_deleted_embeddings = self.update_embeddings(
+                user,
+                current_entries,
+                DbEntry.EntryType.DOCX,
+                DbEntry.EntrySource.COMPUTER,
+                "compiled",
+                logger,
+                deletion_file_names,
+                regenerate=regenerate,
+                file_to_text_map=file_to_text_map,
+            )
+
+        return num_new_embeddings, num_deleted_embeddings
+
+    @staticmethod
+    def extract_docx_entries(docx_files) -> Tuple[Dict, List[Entry]]:
+        """Extract entries from specified DOCX files"""
+
+        entries: List[str] = []
+        entry_to_location_map: List[Tuple[str, str]] = []
+        file_to_text_map = dict()
+        for docx_file in docx_files:
+            try:
+                docx_texts = DocxToEntries.extract_text(docx_files[docx_file])
+                entry_to_location_map += zip(docx_texts, [docx_file] * len(docx_texts))
+                entries.extend(docx_texts)
+                file_to_text_map[docx_file] = docx_texts
+            except Exception as e:
+                logger.warning(f"Unable to extract entries from file: {docx_file}")
+                logger.warning(e, exc_info=True)
+        return file_to_text_map, DocxToEntries.convert_docx_entries_to_maps(entries, dict(entry_to_location_map))
+
+    @staticmethod
+    def convert_docx_entries_to_maps(parsed_entries: List[str], entry_to_file_map) -> List[Entry]:
+        """Convert each DOCX entry into a dictionary"""
+        entries = []
+        for parsed_entry in parsed_entries:
+            entry_filename = entry_to_file_map[parsed_entry]
+            # Append base filename to compiled entry for context to model
+            heading = f"{entry_filename}\n"
+            compiled_entry = f"{heading}{parsed_entry}"
+            entries.append(
+                Entry(
+                    compiled=compiled_entry,
+                    raw=parsed_entry,
+                    heading=heading,
+                    file=f"{entry_filename}",
+                )
+            )
+
+        logger.debug(f"Converted {len(parsed_entries)} DOCX entries to dictionaries")
+
+        return entries
+
+    @staticmethod
+    def extract_text(docx_file):
+        """Extract text from specified DOCX file"""
+        try:
+            docx_entry_by_pages = []
+            # Create temp file with .docx extension that gets auto-deleted
+            with tempfile.NamedTemporaryFile(suffix=".docx", delete=True) as tmp:
+                tmp.write(docx_file)
+                tmp.flush()  # Ensure all data is written
+
+                # Load the content using Docx2txtLoader
+                loader = Docx2txtLoader(tmp.name)
+                docx_entries_per_file = loader.load()
+
+                # Convert the loaded entries into the desired format
+                docx_entry_by_pages = [page.page_content for page in docx_entries_per_file]
+        except Exception as e:
+            logger.warning(f"Unable to extract text from file: {docx_file}")
+            logger.warning(e, exc_info=True)
+
+        return docx_entry_by_pages