khoj 2.0.0b12.dev5__py3-none-any.whl → 2.0.0b13.dev19__py3-none-any.whl

khoj/utils/initialization.py

@@ -16,7 +16,6 @@ from khoj.processor.conversation.utils import model_to_prompt_size, model_to_tok
 from khoj.utils.constants import (
     default_anthropic_chat_models,
     default_gemini_chat_models,
-    default_offline_chat_models,
     default_openai_chat_models,
 )
 
@@ -61,9 +60,7 @@ def initialization(interactive: bool = True):
                 ]
                 default_chat_models = known_available_models + other_available_models
             except Exception as e:
-                logger.warning(
-                    f"⚠️ Failed to fetch {provider} chat models. Fallback to default models. Error: {str(e)}"
-                )
+                logger.warning(f"⚠️ Failed to fetch {provider} chat models. Fallback to default models. Error: {str(e)}")
 
         # Set up OpenAI's online chat models
         openai_configured, openai_provider = _setup_chat_model_provider(
@@ -72,7 +69,6 @@ def initialization(interactive: bool = True):
             default_api_key=openai_api_key,
             api_base_url=openai_base_url,
             vision_enabled=True,
-            is_offline=False,
             interactive=interactive,
             provider_name=provider,
         )
@@ -118,7 +114,6 @@ def initialization(interactive: bool = True):
             default_gemini_chat_models,
             default_api_key=os.getenv("GEMINI_API_KEY"),
             vision_enabled=True,
-            is_offline=False,
             interactive=interactive,
             provider_name="Google Gemini",
         )
@@ -145,40 +140,11 @@ def initialization(interactive: bool = True):
             default_anthropic_chat_models,
             default_api_key=os.getenv("ANTHROPIC_API_KEY"),
             vision_enabled=True,
-            is_offline=False,
-            interactive=interactive,
-        )
-
-        # Set up offline chat models
-        _setup_chat_model_provider(
-            ChatModel.ModelType.OFFLINE,
-            default_offline_chat_models,
-            default_api_key=None,
-            vision_enabled=False,
-            is_offline=True,
             interactive=interactive,
         )
 
         logger.info("🗣️ Chat model configuration complete")
 
-        # Set up offline speech to text model
-        use_offline_speech2text_model = "n" if not interactive else input("Use offline speech to text model? (y/n): ")
-        if use_offline_speech2text_model == "y":
-            logger.info("🗣️ Setting up offline speech to text model")
-            # Delete any existing speech to text model options. There can only be one.
-            SpeechToTextModelOptions.objects.all().delete()
-
-            default_offline_speech2text_model = "base"
-            offline_speech2text_model = input(
-                f"Enter the Whisper model to use Offline (default: {default_offline_speech2text_model}): "
-            )
-            offline_speech2text_model = offline_speech2text_model or default_offline_speech2text_model
-            SpeechToTextModelOptions.objects.create(
-                model_name=offline_speech2text_model, model_type=SpeechToTextModelOptions.ModelType.OFFLINE
-            )
-
-            logger.info(f"🗣️  Offline speech to text model configured to {offline_speech2text_model}")
-
     def _setup_chat_model_provider(
         model_type: ChatModel.ModelType,
         default_chat_models: list,
@@ -186,7 +152,6 @@ def initialization(interactive: bool = True):
         interactive: bool,
         api_base_url: str = None,
         vision_enabled: bool = False,
-        is_offline: bool = False,
         provider_name: str = None,
     ) -> Tuple[bool, AiModelApi]:
         supported_vision_models = (
@@ -195,11 +160,6 @@ def initialization(interactive: bool = True):
         provider_name = provider_name or model_type.name.capitalize()
 
         default_use_model = default_api_key is not None
-        # If not in interactive mode & in the offline setting, it's most likely that we're running in a containerized environment.
-        # This usually means there's not enough RAM to load offline models directly within the application.
-        # In such cases, we default to not using the model -- it's recommended to use another service like Ollama to host the model locally in that case.
-        if is_offline:
-            default_use_model = False
 
         use_model_provider = (
             default_use_model if not interactive else input(f"Add {provider_name} chat models? (y/n): ") == "y"
@@ -211,13 +171,12 @@ def initialization(interactive: bool = True):
         logger.info(f"️💬 Setting up your {provider_name} chat configuration")
 
         ai_model_api = None
-        if not is_offline:
-            if interactive:
-                user_api_key = input(f"Enter your {provider_name} API key (default: {default_api_key}): ")
-                api_key = user_api_key if user_api_key != "" else default_api_key
-            else:
-                api_key = default_api_key
-            ai_model_api = AiModelApi.objects.create(api_key=api_key, name=provider_name, api_base_url=api_base_url)
+        if interactive:
+            user_api_key = input(f"Enter your {provider_name} API key (default: {default_api_key}): ")
+            api_key = user_api_key if user_api_key != "" else default_api_key
+        else:
+            api_key = default_api_key
+        ai_model_api = AiModelApi.objects.create(api_key=api_key, name=provider_name, api_base_url=api_base_url)
 
         if interactive:
             user_chat_models = input(

khoj/utils/models.py

@@ -8,12 +8,10 @@ from tqdm import trange
 
 class BaseEncoder(ABC):
     @abstractmethod
-    def __init__(self, model_name: str, device: torch.device = None, **kwargs):
-        ...
+    def __init__(self, model_name: str, device: torch.device = None, **kwargs): ...
 
     @abstractmethod
-    def encode(self, entries: List[str], device: torch.device = None, **kwargs) -> torch.Tensor:
-        ...
+    def encode(self, entries: List[str], device: torch.device = None, **kwargs) -> torch.Tensor: ...
 
 
 class OpenAI(BaseEncoder):

khoj/utils/rawconfig.py

@@ -1,8 +1,7 @@
 # System Packages
 import json
 import uuid
-from pathlib import Path
-from typing import Dict, List, Optional
+from typing import List, Optional
 
 from pydantic import BaseModel
 
@@ -48,17 +47,6 @@ class FilesFilterRequest(BaseModel):
     conversation_id: str
 
 
-class TextConfigBase(ConfigBase):
-    compressed_jsonl: Path
-    embeddings_file: Path
-
-
-class TextContentConfig(ConfigBase):
-    input_files: Optional[List[Path]] = None
-    input_filter: Optional[List[str]] = None
-    index_heading_entries: Optional[bool] = False
-
-
 class GithubRepoConfig(ConfigBase):
     name: str
     owner: str
@@ -74,62 +62,6 @@ class NotionContentConfig(ConfigBase):
     token: str
 
 
-class ContentConfig(ConfigBase):
-    org: Optional[TextContentConfig] = None
-    markdown: Optional[TextContentConfig] = None
-    pdf: Optional[TextContentConfig] = None
-    plaintext: Optional[TextContentConfig] = None
-    github: Optional[GithubContentConfig] = None
-    notion: Optional[NotionContentConfig] = None
-    image: Optional[TextContentConfig] = None
-    docx: Optional[TextContentConfig] = None
-
-
-class ImageSearchConfig(ConfigBase):
-    encoder: str
-    encoder_type: Optional[str] = None
-    model_directory: Optional[Path] = None
-
-    class Config:
-        protected_namespaces = ()
-
-
-class SearchConfig(ConfigBase):
-    image: Optional[ImageSearchConfig] = None
-
-
-class OpenAIProcessorConfig(ConfigBase):
-    api_key: str
-    chat_model: Optional[str] = "gpt-4o-mini"
-
-
-class OfflineChatProcessorConfig(ConfigBase):
-    chat_model: Optional[str] = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
-
-
-class ConversationProcessorConfig(ConfigBase):
-    openai: Optional[OpenAIProcessorConfig] = None
-    offline_chat: Optional[OfflineChatProcessorConfig] = None
-    max_prompt_size: Optional[int] = None
-    tokenizer: Optional[str] = None
-
-
-class ProcessorConfig(ConfigBase):
-    conversation: Optional[ConversationProcessorConfig] = None
-
-
-class AppConfig(ConfigBase):
-    should_log_telemetry: bool = True
-
-
-class FullConfig(ConfigBase):
-    content_type: Optional[ContentConfig] = None
-    search_type: Optional[SearchConfig] = None
-    processor: Optional[ProcessorConfig] = None
-    app: Optional[AppConfig] = AppConfig()
-    version: Optional[str] = None
-
-
 class SearchResponse(ConfigBase):
     entry: str
     score: float

khoj/utils/state.py

@@ -2,7 +2,7 @@ import os
 import threading
 from collections import defaultdict
 from pathlib import Path
-from typing import Any, Dict, List
+from typing import Dict, List
 
 from apscheduler.schedulers.background import BackgroundScheduler
 from openai import OpenAI
@@ -12,19 +12,14 @@ from whisper import Whisper
 from khoj.database.models import ProcessLock
 from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
 from khoj.utils import config as utils_config
-from khoj.utils.config import OfflineChatProcessorModel, SearchModels
 from khoj.utils.helpers import LRU, get_device, is_env_var_true
-from khoj.utils.rawconfig import FullConfig
 
 # Application Global State
-config = FullConfig()
-search_models = SearchModels()
 embeddings_model: Dict[str, EmbeddingsModel] = None
 cross_encoder_model: Dict[str, CrossEncoderModel] = None
 openai_client: OpenAI = None
-offline_chat_processor_config: OfflineChatProcessorModel = None
 whisper_model: Whisper = None
-config_file: Path = None
+log_file: Path = None
 verbose: int = 0
 host: str = None
 port: int = None
@@ -39,7 +34,6 @@ telemetry: List[Dict[str, str]] = []
 telemetry_disabled: bool = is_env_var_true("KHOJ_TELEMETRY_DISABLE")
 khoj_version: str = None
 device = get_device()
-chat_on_gpu: bool = True
 anonymous_mode: bool = False
 pretrained_tokenizers: Dict[str, PreTrainedTokenizer | PreTrainedTokenizerFast] = dict()
 billing_enabled: bool = (

khoj/utils/yaml.py

@@ -1,47 +1,8 @@
-from pathlib import Path
-
 import yaml
 
-from khoj.utils import state
-from khoj.utils.rawconfig import FullConfig
-
 # Do not emit tags when dumping to YAML
 yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None  # type: ignore[assignment]
 
 
-def save_config_to_file_updated_state():
-    with open(state.config_file, "w") as outfile:
-        yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile)
-        outfile.close()
-    return state.config
-
-
-def save_config_to_file(yaml_config: dict, yaml_config_file: Path):
-    "Write config to YML file"
-    # Create output directory, if it doesn't exist
-    yaml_config_file.parent.mkdir(parents=True, exist_ok=True)
-
-    with open(yaml_config_file, "w", encoding="utf-8") as config_file:
-        yaml.safe_dump(yaml_config, config_file, allow_unicode=True)
-
-
-def load_config_from_file(yaml_config_file: Path) -> dict:
-    "Read config from YML file"
-    config_from_file = None
-    with open(yaml_config_file, "r", encoding="utf-8") as config_file:
-        config_from_file = yaml.safe_load(config_file)
-    return config_from_file
-
-
-def parse_config_from_string(yaml_config: dict) -> FullConfig:
-    "Parse and validate config in YML string"
-    return FullConfig.model_validate(yaml_config)
-
-
-def parse_config_from_file(yaml_config_file):
-    "Parse and validate config in YML file"
-    return parse_config_from_string(load_config_from_file(yaml_config_file))
-
-
 def yaml_dump(data):
     return yaml.dump(data, allow_unicode=True, sort_keys=False, default_flow_style=False)

{khoj-2.0.0b12.dev5.dist-info → khoj-2.0.0b13.dev19.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: khoj
-Version: 2.0.0b12.dev5
+Version: 2.0.0b13.dev19
 Summary: Your Second Brain
 Project-URL: Homepage, https://khoj.dev
 Project-URL: Documentation, https://docs.khoj.dev
@@ -27,6 +27,7 @@ Requires-Dist: anyio~=4.8.0
 Requires-Dist: apscheduler~=3.10.0
 Requires-Dist: authlib==1.2.1
 Requires-Dist: beautifulsoup4~=4.12.3
+Requires-Dist: click<8.2.2
 Requires-Dist: cron-descriptor==1.4.3
 Requires-Dist: dateparser>=1.1.1
 Requires-Dist: defusedxml==0.7.1
@@ -47,7 +48,6 @@ Requires-Dist: itsdangerous==2.1.2
 Requires-Dist: jinja2==3.1.6
 Requires-Dist: langchain-community==0.3.3
 Requires-Dist: langchain-text-splitters==0.3.1
-Requires-Dist: llama-cpp-python==0.2.88
 Requires-Dist: lxml==4.9.3
 Requires-Dist: magika~=0.5.1
 Requires-Dist: markdown-it-py~=3.0.0
@@ -79,7 +79,6 @@ Requires-Dist: tzdata==2023.3
 Requires-Dist: uvicorn==0.30.6
 Requires-Dist: websockets==13.0
 Provides-Extra: dev
-Requires-Dist: black>=23.1.0; extra == 'dev'
 Requires-Dist: boto3>=1.34.57; extra == 'dev'
 Requires-Dist: datasets; extra == 'dev'
 Requires-Dist: factory-boy>=3.2.1; extra == 'dev'
@@ -94,6 +93,7 @@ Requires-Dist: pytest-asyncio==0.21.1; extra == 'dev'
 Requires-Dist: pytest-django==4.5.2; extra == 'dev'
 Requires-Dist: pytest-xdist[psutil]; extra == 'dev'
 Requires-Dist: pytest>=7.1.2; extra == 'dev'
+Requires-Dist: ruff>=0.12.0; extra == 'dev'
 Requires-Dist: stripe==7.3.0; extra == 'dev'
 Requires-Dist: twilio==8.11; extra == 'dev'
 Provides-Extra: local