khoj 1.33.3.dev32__py3-none-any.whl

khoj/search_filter/base_filter.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import List
+
+
+class BaseFilter(ABC):
+    @abstractmethod
+    def get_filter_terms(self, query: str) -> List[str]:
+        ...
+
+    def can_filter(self, raw_query: str) -> bool:
+        return len(self.get_filter_terms(raw_query)) > 0
+
+    @abstractmethod
+    def defilter(self, query: str) -> str:
+        ...

khoj/search_filter/date_filter.py

@@ -0,0 +1,215 @@
+import calendar
+import logging
+import re
+from collections import defaultdict
+from datetime import datetime, timedelta
+from math import inf
+from typing import List, Tuple
+
+import dateparser as dtparse
+from dateutil.relativedelta import relativedelta
+
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU, merge_dicts, timer
+
+logger = logging.getLogger(__name__)
+
+
+class DateFilter(BaseFilter):
+    # Date Range Filter Regexes
+    # Example filter queries:
+    # - dt>="yesterday" dt<"tomorrow"
+    # - dt>="last week"
+    # - dt:"2 years ago"
+    date_regex = r"dt([:><=]{1,2})[\"'‘’](.*?)[\"'‘’]"
+
+    def __init__(self, entry_key="compiled"):
+        self.entry_key = entry_key
+        self.date_to_entry_ids = defaultdict(set)
+        self.cache = LRU()
+        self.dtparser_regexes = self.compile_date_regexes()
+        self.dtparser_ordinal_suffixes = re.compile(r"(st|nd|rd|th)")
+        self.dtparser_settings = {
+            "PREFER_DAY_OF_MONTH": "first",
+            "DATE_ORDER": "YMD",  # Prefer YMD and DMY over MDY when parsing ambiguous dates
+        }
+
+    def compile_date_regexes(self):
+        months = calendar.month_name[1:]
+        abbr_months = calendar.month_abbr[1:]
+        # Extract natural dates from content like 1st April 1984, 31 April 84, Apr 4th 1984, 13 Apr 84
+        dBY_regex = re.compile(r"\b\d{1,2}(?:st|nd|rd|th)? (?:" + "|".join(months) + r") \d{4}\b", re.IGNORECASE)
+        dBy_regex = re.compile(r"\b\d{1,2}(?:st|nd|rd|th)? (?:" + "|".join(months) + r") \d{2}\b", re.IGNORECASE)
+        BdY_regex = re.compile(r"\b(?:" + "|".join(months) + r") \d{1,2}(?:st|nd|rd|th)? \d{4}\b", re.IGNORECASE)
+        Bdy_regex = re.compile(r"\b(?:" + "|".join(months) + r") \d{1,2}(?:st|nd|rd|th)? \d{2}\b", re.IGNORECASE)
+        dbY_regex = re.compile(r"\b\d{1,2}(?:st|nd|rd|th)? (?:" + "|".join(abbr_months) + r") \d{4}\b", re.IGNORECASE)
+        dby_regex = re.compile(r"\b\d{1,2}(?:st|nd|rd|th)? (?:" + "|".join(abbr_months) + r") \d{2}\b", re.IGNORECASE)
+        bdY_regex = re.compile(r"\b(?:" + "|".join(abbr_months) + r") \d{1,2}(?:st|nd|rd|th)? \d{4}\b", re.IGNORECASE)
+        bdy_regex = re.compile(r"\b(?:" + "|".join(abbr_months) + r") \d{1,2}(?:st|nd|rd|th)? \d{2}\b", re.IGNORECASE)
+        # Extract natural of form Month, Year like January 2021, Jan 2021, Jan 21
+        BY_regex = re.compile(r"\b(?:" + "|".join(months) + r") \d{4}\b", re.IGNORECASE)
+        By_regex = re.compile(r"\b(?:" + "|".join(months) + r") \d{2}\b", re.IGNORECASE)
+        bY_regex = re.compile(r"\b(?:" + "|".join(abbr_months) + r") \d{4}\b", re.IGNORECASE)
+        by_regex = re.compile(r"\b(?:" + "|".join(abbr_months) + r") \d{2}\b", re.IGNORECASE)
+        # Extract structured dates from content like 1984-04-01, 1984/04/01, 01-04-1984, 01/04/1984, 01.04.1984, 01-04-84, 01/04/84
+        Ymd_date_regex = re.compile(r"\b\d{4}[-\/]\d{2}[-\/]\d{2}\b", re.IGNORECASE)
+        dmY_date_regex = re.compile(r"\b\d{2}[-\/]\d{2}[-\/]\d{4}\b", re.IGNORECASE)
+        dmy_date_regex = re.compile(r"\b\d{2}[-\/]\d{2}[-\/]\d{2}\b", re.IGNORECASE)
+        dmY_dot_date_regex = re.compile(r"\b\d{2}[\.]\d{2}[\.]\d{4}\b", re.IGNORECASE)
+
+        # Combine date formatter and date identifier regex pairs
+        dtparser_regexes: List[Tuple[str, re.Pattern[str]]] = [
+            # Structured dates
+            ("%Y-%m-%d", Ymd_date_regex),
+            ("%Y/%m/%d", Ymd_date_regex),
+            ("%d-%m-%Y", dmY_date_regex),
+            ("%d/%m/%Y", dmY_date_regex),
+            ("%d.%m.%Y", dmY_dot_date_regex),
+            ("%d-%m-%y", dmy_date_regex),
+            ("%d/%m/%y", dmy_date_regex),
+            # Natural dates
+            ("%d %B %Y", dBY_regex),
+            ("%d %B %y", dBy_regex),
+            ("%B %d %Y", BdY_regex),
+            ("%B %d %y", Bdy_regex),
+            ("%d %b %Y", dbY_regex),
+            ("%d %b %y", dby_regex),
+            ("%b %d %Y", bdY_regex),
+            ("%b %d %y", bdy_regex),
+            # Partial natural dates
+            ("%B %Y", BY_regex),
+            ("%B %y", By_regex),
+            ("%b %Y", bY_regex),
+            ("%b %y", by_regex),
+        ]
+        return dtparser_regexes
+
+    def extract_dates(self, content):
+        "Extract natural and structured dates from content"
+        valid_dates = set()
+        for date_format, date_regex in self.dtparser_regexes:
+            matched_dates = date_regex.findall(content)
+            for date_str in matched_dates:
+                # Remove ordinal suffixes to parse date
+                date_str = self.dtparser_ordinal_suffixes.sub("", date_str)
+                try:
+                    valid_dates.add(datetime.strptime(date_str, date_format))
+                except ValueError:
+                    continue
+
+        return list(valid_dates)
+
+    def get_filter_terms(self, query: str) -> List[str]:
+        "Get all filter terms in query"
+        return [f"dt{item[0]}'{item[1]}'" for item in re.findall(self.date_regex, query)]
+
+    def get_query_date_range(self, query) -> List:
+        with timer("Extract date range to filter from query", logger):
+            query_daterange = self.extract_date_range(query)
+
+        return query_daterange
+
+    def defilter(self, query):
+        # remove date range filter from query
+        query = re.sub(rf"\s+{self.date_regex}", " ", query)
+        query = re.sub(r"\s{2,}", " ", query).strip()  # remove multiple spaces
+        return query
+
+    def extract_date_range(self, query):
+        # find date range filter in query
+        date_range_matches = re.findall(self.date_regex, query)
+
+        if len(date_range_matches) == 0:
+            return []
+
+        # extract, parse natural dates ranges from date range filter passed in query
+        # e.g. today maps to (start_of_day, start_of_tomorrow)
+        date_ranges_from_filter = []
+        for cmp, date_str in date_range_matches:
+            if self.parse(date_str):
+                dt_start, dt_end = self.parse(date_str)
+                date_ranges_from_filter += [[cmp, (dt_start.timestamp(), dt_end.timestamp())]]
+
+        # Combine dates with their comparators to form date range intervals
+        # For e.g.
+        #   >=yesterday maps to [start_of_yesterday, inf)
+        #   <tomorrow maps to [0, start_of_tomorrow)
+        # ---
+        effective_date_range: List = [0, inf]
+        date_range_considering_comparator = []
+        for cmp, (dtrange_start, dtrange_end) in date_ranges_from_filter:
+            if cmp == ">":
+                date_range_considering_comparator += [[dtrange_end, inf]]
+            elif cmp == ">=":
+                date_range_considering_comparator += [[dtrange_start, inf]]
+            elif cmp == "<":
+                date_range_considering_comparator += [[0, dtrange_start]]
+            elif cmp == "<=":
+                date_range_considering_comparator += [[0, dtrange_end]]
+            elif cmp == "=" or cmp == ":" or cmp == "==":
+                date_range_considering_comparator += [[dtrange_start, dtrange_end]]
+
+        # Combine above intervals (via AND/intersect)
+        # In the above example, this gives us [start_of_yesterday, start_of_tomorrow)
+        # This is the effective date range to filter entries by
+        # ---
+        for date_range in date_range_considering_comparator:
+            effective_date_range = [
+                max(effective_date_range[0], date_range[0]),
+                min(effective_date_range[1], date_range[1]),
+            ]
+
+        if effective_date_range == [0, inf] or effective_date_range[0] > effective_date_range[1]:
+            return []
+        else:
+            # If the first element is 0, replace it with None
+
+            if effective_date_range[0] == 0:
+                effective_date_range[0] = None
+
+            # If the second element is inf, replace it with None
+            if effective_date_range[1] == inf:
+                effective_date_range[1] = None
+
+            return effective_date_range
+
+    def parse(self, date_str, relative_base=None):
+        "Parse date string passed in date filter of query to datetime object"
+        # clean date string to handle future date parsing by date parser
+        future_strings = ["later", "from now", "from today"]
+        prefer_dates_from = {True: "future", False: "past"}[any([True for fstr in future_strings if fstr in date_str])]
+        dtquery_settings = {"RELATIVE_BASE": relative_base or datetime.now(), "PREFER_DATES_FROM": prefer_dates_from}
+        dtparser_settings = merge_dicts(dtquery_settings, self.dtparser_settings)
+
+        # parse date passed in query date filter
+        clean_date_str = re.sub("|".join(future_strings), "", date_str)
+        try:
+            parsed_date = dtparse.parse(clean_date_str, settings=dtparser_settings)
+        except Exception as e:
+            logger.error(f"Failed to parse date string: {date_str} with error: {e}")
+            return None
+
+        if parsed_date is None:
+            return None
+
+        return self.date_to_daterange(parsed_date, date_str)
+
+    def date_to_daterange(self, parsed_date, date_str):
+        "Convert parsed date to date ranges at natural granularity (day, week, month or year)"
+
+        start_of_day = parsed_date.replace(hour=0, minute=0, second=0, microsecond=0)
+
+        if "year" in date_str:
+            return (datetime(parsed_date.year, 1, 1, 0, 0, 0), datetime(parsed_date.year + 1, 1, 1, 0, 0, 0))
+        if "month" in date_str:
+            start_of_month = datetime(parsed_date.year, parsed_date.month, 1, 0, 0, 0)
+            next_month = start_of_month + relativedelta(months=1)
+            return (start_of_month, next_month)
+        if "week" in date_str:
+            # if week in date string, dateparser parses it to next week start
+            # so today = end of this week
+            start_of_week = start_of_day - timedelta(days=7)
+            return (start_of_week, start_of_day)
+        else:
+            next_day = start_of_day + relativedelta(days=1)
+        return (start_of_day, next_day)

khoj/search_filter/file_filter.py

@@ -0,0 +1,32 @@
+import logging
+import re
+from collections import defaultdict
+from typing import List
+
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU
+
+logger = logging.getLogger(__name__)
+
+
+class FileFilter(BaseFilter):
+    file_filter_regex = r'(?<!-)file:"(.+?)" ?'
+    excluded_file_filter_regex = r'-file:"(.+?)" ?'
+
+    def __init__(self, entry_key="file"):
+        self.entry_key = entry_key
+        self.file_to_entry_map = defaultdict(set)
+        self.cache = LRU()
+
+    def get_filter_terms(self, query: str) -> List[str]:
+        "Get all filter terms in query"
+        required_files = [f"{required_file}" for required_file in re.findall(self.file_filter_regex, query)]
+        excluded_files = [f"-{excluded_file}" for excluded_file in re.findall(self.excluded_file_filter_regex, query)]
+        return required_files + excluded_files
+
+    def convert_to_regex(self, file_filter: str) -> str:
+        "Convert file filter to regex"
+        return file_filter.replace(".", r"\.").replace("*", r".*")
+
+    def defilter(self, query: str) -> str:
+        return re.sub(self.file_filter_regex, "", query).strip()

khoj/search_filter/word_filter.py

@@ -0,0 +1,29 @@
+import logging
+import re
+from collections import defaultdict
+from typing import List
+
+from khoj.search_filter.base_filter import BaseFilter
+from khoj.utils.helpers import LRU
+
+logger = logging.getLogger(__name__)
+
+
+class WordFilter(BaseFilter):
+    # Filter Regex
+    required_regex = r'\+"([a-zA-Z0-9_-]+)" ?'
+    blocked_regex = r'\-"([a-zA-Z0-9_-]+)" ?'
+
+    def __init__(self, entry_key="raw"):
+        self.entry_key = entry_key
+        self.word_to_entry_index = defaultdict(set)
+        self.cache = LRU()
+
+    def get_filter_terms(self, query: str) -> List[str]:
+        "Get all filter terms in query"
+        required_terms = [f"+{required_term}" for required_term in re.findall(self.required_regex, query)]
+        blocked_terms = [f"-{blocked_term}" for blocked_term in re.findall(self.blocked_regex, query)]
+        return required_terms + blocked_terms
+
+    def defilter(self, query: str) -> str:
+        return re.sub(self.blocked_regex, "", re.sub(self.required_regex, "", query)).strip()

khoj/search_type/text_search.py

@@ -0,0 +1,255 @@
+import logging
+import math
+from pathlib import Path
+from typing import List, Optional, Tuple, Type, Union
+
+import requests
+import torch
+from asgiref.sync import sync_to_async
+from sentence_transformers import util
+
+from khoj.database.adapters import EntryAdapters, get_default_search_model
+from khoj.database.models import Agent
+from khoj.database.models import Entry as DbEntry
+from khoj.database.models import KhojUser
+from khoj.processor.content.text_to_entries import TextToEntries
+from khoj.utils import state
+from khoj.utils.helpers import get_absolute_path, timer
+from khoj.utils.jsonl import load_jsonl
+from khoj.utils.models import BaseEncoder
+from khoj.utils.rawconfig import Entry, SearchResponse
+from khoj.utils.state import SearchType
+
+logger = logging.getLogger(__name__)
+
+search_type_to_embeddings_type = {
+    SearchType.Org.value: DbEntry.EntryType.ORG,
+    SearchType.Markdown.value: DbEntry.EntryType.MARKDOWN,
+    SearchType.Plaintext.value: DbEntry.EntryType.PLAINTEXT,
+    SearchType.Pdf.value: DbEntry.EntryType.PDF,
+    SearchType.Github.value: DbEntry.EntryType.GITHUB,
+    SearchType.Notion.value: DbEntry.EntryType.NOTION,
+    SearchType.All.value: None,
+}
+
+
+def extract_entries(jsonl_file) -> List[Entry]:
+    "Load entries from compressed jsonl"
+    return list(map(Entry.from_dict, load_jsonl(jsonl_file)))
+
+
+def compute_embeddings(
+    entries_with_ids: List[Tuple[int, Entry]],
+    bi_encoder: BaseEncoder,
+    embeddings_file: Path,
+    regenerate=False,
+    normalize=True,
+):
+    "Compute (and Save) Embeddings or Load Pre-Computed Embeddings"
+    new_embeddings = torch.tensor([], device=state.device)
+    existing_embeddings = torch.tensor([], device=state.device)
+    create_index_msg = ""
+    # Load pre-computed embeddings from file if exists and update them if required
+    if embeddings_file.exists() and not regenerate:
+        corpus_embeddings: torch.Tensor = torch.load(get_absolute_path(embeddings_file), map_location=state.device)
+        logger.debug(f"Loaded {len(corpus_embeddings)} text embeddings from {embeddings_file}")
+    else:
+        corpus_embeddings = torch.tensor([], device=state.device)
+        create_index_msg = " Creating index from scratch."
+
+    # Encode any new entries in the corpus and update corpus embeddings
+    new_entries = [entry.compiled for id, entry in entries_with_ids if id == -1]
+    if new_entries:
+        logger.info(f"📩 Indexing {len(new_entries)} text entries.{create_index_msg}")
+        new_embeddings = bi_encoder.encode(
+            new_entries, convert_to_tensor=True, device=state.device, show_progress_bar=True
+        )
+
+    # Extract existing embeddings from previous corpus embeddings
+    existing_entry_ids = [id for id, _ in entries_with_ids if id != -1]
+    if existing_entry_ids:
+        existing_embeddings = torch.index_select(
+            corpus_embeddings, 0, torch.tensor(existing_entry_ids, device=state.device)
+        )
+
+    # Set corpus embeddings to merger of existing and new embeddings
+    corpus_embeddings = torch.cat([existing_embeddings, new_embeddings], dim=0)
+    if normalize:
+        # Normalize embeddings for faster lookup via dot product when querying
+        corpus_embeddings = util.normalize_embeddings(corpus_embeddings)
+
+    # Save regenerated or updated embeddings to file
+    torch.save(corpus_embeddings, embeddings_file)
+    logger.info(f"📩 Saved computed text embeddings to {embeddings_file}")
+
+    return corpus_embeddings
+
+
+def load_embeddings(
+    embeddings_file: Path,
+):
+    "Load pre-computed embeddings from file if exists and update them if required"
+    if embeddings_file.exists():
+        corpus_embeddings: torch.Tensor = torch.load(get_absolute_path(embeddings_file), map_location=state.device)
+        logger.debug(f"Loaded {len(corpus_embeddings)} text embeddings from {embeddings_file}")
+        return util.normalize_embeddings(corpus_embeddings)
+
+    return None
+
+
+async def query(
+    raw_query: str,
+    user: KhojUser,
+    type: SearchType = SearchType.All,
+    question_embedding: Union[torch.Tensor, None] = None,
+    max_distance: float = None,
+    agent: Optional[Agent] = None,
+) -> Tuple[List[dict], List[Entry]]:
+    "Search for entries that answer the query"
+
+    file_type = search_type_to_embeddings_type[type.value]
+
+    query = raw_query
+    search_model = await sync_to_async(get_default_search_model)()
+    if not max_distance:
+        if search_model.bi_encoder_confidence_threshold:
+            max_distance = search_model.bi_encoder_confidence_threshold
+        else:
+            max_distance = math.inf
+
+    # Encode the query using the bi-encoder
+    if question_embedding is None:
+        with timer("Query Encode Time", logger, state.device):
+            question_embedding = state.embeddings_model[search_model.name].embed_query(query)
+
+    # Find relevant entries for the query
+    top_k = 10
+    with timer("Search Time", logger, state.device):
+        hits = EntryAdapters.search_with_embeddings(
+            raw_query=raw_query,
+            embeddings=question_embedding,
+            max_results=top_k,
+            file_type_filter=file_type,
+            max_distance=max_distance,
+            user=user,
+            agent=agent,
+        ).all()
+        hits = await sync_to_async(list)(hits)  # type: ignore[call-arg]
+
+    return hits
+
+
+def collate_results(hits, dedupe=True):
+    hit_ids = set()
+    hit_hashes = set()
+    for hit in hits:
+        if dedupe and (hit.hashed_value in hit_hashes or hit.corpus_id in hit_ids):
+            continue
+
+        else:
+            hit_hashes.add(hit.hashed_value)
+            hit_ids.add(hit.corpus_id)
+            yield SearchResponse.model_validate(
+                {
+                    "entry": hit.raw,
+                    "score": hit.distance,
+                    "corpus_id": str(hit.corpus_id),
+                    "additional": {
+                        "source": hit.file_source,
+                        "file": hit.file_path,
+                        "compiled": hit.compiled,
+                        "heading": hit.heading,
+                    },
+                }
+            )
+
+
+def deduplicated_search_responses(hits: List[SearchResponse]):
+    hit_ids = set()
+    for hit in hits:
+        if hit.corpus_id in hit_ids:
+            continue
+
+        else:
+            hit_ids.add(hit.corpus_id)
+            yield SearchResponse.model_validate(
+                {
+                    "entry": hit.entry,
+                    "score": hit.score,
+                    "corpus_id": hit.corpus_id,
+                    "additional": {
+                        "source": hit.additional["source"],
+                        "file": hit.additional["file"],
+                        "compiled": hit.additional["compiled"],
+                        "heading": hit.additional["heading"],
+                    },
+                }
+            )
+
+
+def rerank_and_sort_results(hits, query, rank_results, search_model_name):
+    # Rerank results if explicitly requested, if can use inference server
+    # AND if we have more than one result
+    rank_results = (rank_results or state.cross_encoder_model[search_model_name].inference_server_enabled()) and len(
+        list(hits)
+    ) > 1
+
+    # Score all retrieved entries using the cross-encoder
+    if rank_results:
+        hits = cross_encoder_score(query, hits, search_model_name)
+
+    # Sort results by cross-encoder score followed by bi-encoder score
+    hits = sort_results(rank_results=rank_results, hits=hits)
+
+    return hits
+
+
+def setup(
+    text_to_entries: Type[TextToEntries],
+    files: dict[str, str],
+    regenerate: bool,
+    user: KhojUser,
+    config=None,
+) -> Tuple[int, int]:
+    if config:
+        num_new_embeddings, num_deleted_embeddings = text_to_entries(config).process(
+            files=files, user=user, regenerate=regenerate
+        )
+    else:
+        num_new_embeddings, num_deleted_embeddings = text_to_entries().process(
+            files=files, user=user, regenerate=regenerate
+        )
+
+    if files:
+        file_names = [file_name for file_name in files]
+
+        logger.info(
+            f"Deleted {num_deleted_embeddings} entries. Created {num_new_embeddings} new entries for user {user} from files {file_names[:10]} ..."
+        )
+
+    return num_new_embeddings, num_deleted_embeddings
+
+
+def cross_encoder_score(query: str, hits: List[SearchResponse], search_model_name: str) -> List[SearchResponse]:
+    """Score all retrieved entries using the cross-encoder"""
+    try:
+        with timer("Cross-Encoder Predict Time", logger, state.device):
+            cross_scores = state.cross_encoder_model[search_model_name].predict(query, hits)
+    except requests.exceptions.HTTPError as e:
+        logger.error(f"Failed to rerank documents using the inference endpoint. Error: {e}.", exc_info=True)
+        cross_scores = [0.0] * len(hits)
+
+    # Convert cross-encoder scores to distances and pass in hits for reranking
+    for idx in range(len(cross_scores)):
+        hits[idx]["cross_score"] = 1 - cross_scores[idx]
+
+    return hits
+
+
+def sort_results(rank_results: bool, hits: List[dict]) -> List[dict]:
+    """Order results by cross-encoder score followed by bi-encoder score"""
+    with timer("Rank Time", logger, state.device):
+        hits.sort(key=lambda x: x["score"])  # sort by bi-encoder score
+        if rank_results:
+            hits.sort(key=lambda x: x["cross_score"])  # sort by cross-encoder score
+    return hits

khoj/utils/cli.py

@@ -0,0 +1,101 @@
+import argparse
+import logging
+import os
+import pathlib
+from importlib.metadata import version
+
+logger = logging.getLogger(__name__)
+
+from khoj.migrations.migrate_offline_chat_default_model import (
+    migrate_offline_chat_default_model,
+)
+from khoj.migrations.migrate_offline_chat_schema import migrate_offline_chat_schema
+from khoj.migrations.migrate_offline_model import migrate_offline_model
+from khoj.migrations.migrate_processor_config_openai import (
+    migrate_processor_conversation_schema,
+)
+from khoj.migrations.migrate_server_pg import migrate_server_pg
+from khoj.migrations.migrate_version import migrate_config_to_version
+from khoj.utils.helpers import in_debug_mode, is_env_var_true, resolve_absolute_path
+from khoj.utils.yaml import parse_config_from_file
+
+
+def cli(args=None):
+    # Setup Argument Parser for the Commandline Interface
+    parser = argparse.ArgumentParser(description="Start Khoj; An AI personal assistant for your Digital Brain")
+    parser.add_argument(
+        "--config-file", default="~/.khoj/khoj.yml", type=pathlib.Path, help="YAML file to configure Khoj"
+    )
+    parser.add_argument(
+        "--regenerate",
+        action="store_true",
+        default=False,
+        help="Regenerate model embeddings from source files. Default: false",
+    )
+    parser.add_argument("--verbose", "-v", action="count", default=0, help="Show verbose conversion logs. Default: 0")
+    parser.add_argument("--host", type=str, default="127.0.0.1", help="Host address of the server. Default: 127.0.0.1")
+    parser.add_argument("--port", "-p", type=int, default=42110, help="Port of the server. Default: 42110")
+    parser.add_argument(
+        "--socket",
+        type=pathlib.Path,
+        help="Path to UNIX socket for server. Use to run server behind reverse proxy. Default: /tmp/uvicorn.sock",
+    )
+    parser.add_argument("--sslcert", type=str, help="Path to SSL certificate file")
+    parser.add_argument("--sslkey", type=str, help="Path to SSL key file")
+    parser.add_argument("--version", "-V", action="store_true", help="Print the installed Khoj version and exit")
+    parser.add_argument(
+        "--disable-chat-on-gpu", action="store_true", default=False, help="Disable using GPU for the offline chat model"
+    )
+    parser.add_argument(
+        "--anonymous-mode",
+        action="store_true",
+        default=False,
+        help="Run Khoj in anonymous mode. This does not require any login for connecting users.",
+    )
+    parser.add_argument(
+        "--non-interactive",
+        action="store_true",
+        default=False,
+        help="Start Khoj in non-interactive mode. Assumes interactive shell unavailable for config. E.g when run via Docker.",
+    )
+
+    args, remaining_args = parser.parse_known_args(args)
+
+    if len(remaining_args) > 0:
+        logger.info(f"⚠️  Ignoring unknown commandline args: {remaining_args}")
+
+    # Set default values for arguments
+    args.chat_on_gpu = not args.disable_chat_on_gpu
+
+    args.version_no = version("khoj")
+    if args.version:
+        # Show version of khoj installed and exit
+        print(args.version_no)
+        exit(0)
+
+    # Normalize config_file path to absolute path
+    args.config_file = resolve_absolute_path(args.config_file)
+
+    if not args.config_file.exists():
+        args.config = None
+    else:
+        args = run_migrations(args)
+        args.config = parse_config_from_file(args.config_file)
+        if is_env_var_true("KHOJ_TELEMETRY_DISABLE") or in_debug_mode():
+            args.config.app.should_log_telemetry = False
+
+    return args
+
+
+def run_migrations(args):
+    migrations = [
+        migrate_config_to_version,
+        migrate_processor_conversation_schema,
+        migrate_offline_model,
+        migrate_offline_chat_schema,
+        migrate_offline_chat_default_model,
+        migrate_server_pg,
+    ]
+    for migration in migrations:
+        args = migration(args)
+    return args