khoj 1.16.1.dev15__py3-none-any.whl

khoj/utils/rawconfig.py

@@ -0,0 +1,160 @@
+# System Packages
+import json
+import uuid
+from pathlib import Path
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel
+
+from khoj.utils.helpers import to_snake_case_from_dash
+
+
+class ConfigBase(BaseModel):
+    class Config:
+        alias_generator = to_snake_case_from_dash
+        populate_by_name = True
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        return setattr(self, key, value)
+
+
+class LocationData(BaseModel):
+    city: Optional[str]
+    region: Optional[str]
+    country: Optional[str]
+
+
+class FilterRequest(BaseModel):
+    filename: str
+    conversation_id: str
+
+
+class TextConfigBase(ConfigBase):
+    compressed_jsonl: Path
+    embeddings_file: Path
+
+
+class TextContentConfig(ConfigBase):
+    input_files: Optional[List[Path]] = None
+    input_filter: Optional[List[str]] = None
+    index_heading_entries: Optional[bool] = False
+
+
+class GithubRepoConfig(ConfigBase):
+    name: str
+    owner: str
+    branch: Optional[str] = "master"
+
+
+class GithubContentConfig(ConfigBase):
+    pat_token: str
+    repos: List[GithubRepoConfig]
+
+
+class NotionContentConfig(ConfigBase):
+    token: str
+
+
+class ContentConfig(ConfigBase):
+    org: Optional[TextContentConfig] = None
+    markdown: Optional[TextContentConfig] = None
+    pdf: Optional[TextContentConfig] = None
+    plaintext: Optional[TextContentConfig] = None
+    github: Optional[GithubContentConfig] = None
+    notion: Optional[NotionContentConfig] = None
+    image: Optional[TextContentConfig] = None
+    docx: Optional[TextContentConfig] = None
+
+
+class ImageSearchConfig(ConfigBase):
+    encoder: str
+    encoder_type: Optional[str] = None
+    model_directory: Optional[Path] = None
+
+    class Config:
+        protected_namespaces = ()
+
+
+class SearchConfig(ConfigBase):
+    image: Optional[ImageSearchConfig] = None
+
+
+class OpenAIProcessorConfig(ConfigBase):
+    api_key: str
+    chat_model: Optional[str] = "gpt-3.5-turbo"
+
+
+class OfflineChatProcessorConfig(ConfigBase):
+    chat_model: Optional[str] = "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF"
+
+
+class ConversationProcessorConfig(ConfigBase):
+    openai: Optional[OpenAIProcessorConfig] = None
+    offline_chat: Optional[OfflineChatProcessorConfig] = None
+    max_prompt_size: Optional[int] = None
+    tokenizer: Optional[str] = None
+
+
+class ProcessorConfig(ConfigBase):
+    conversation: Optional[ConversationProcessorConfig] = None
+
+
+class AppConfig(ConfigBase):
+    should_log_telemetry: bool = True
+
+
+class FullConfig(ConfigBase):
+    content_type: Optional[ContentConfig] = None
+    search_type: Optional[SearchConfig] = None
+    processor: Optional[ProcessorConfig] = None
+    app: Optional[AppConfig] = AppConfig()
+    version: Optional[str] = None
+
+
+class SearchResponse(ConfigBase):
+    entry: str
+    score: float
+    cross_score: Optional[float] = None
+    additional: Optional[dict] = None
+    corpus_id: str
+
+
+class Entry:
+    raw: str
+    compiled: str
+    heading: Optional[str]
+    file: Optional[str]
+    corpus_id: str
+
+    def __init__(
+        self,
+        raw: str = None,
+        compiled: str = None,
+        heading: Optional[str] = None,
+        file: Optional[str] = None,
+        corpus_id: uuid.UUID = None,
+    ):
+        self.raw = raw
+        self.compiled = compiled
+        self.heading = heading
+        self.file = file
+        self.corpus_id = str(corpus_id)
+
+    def to_json(self) -> str:
+        return json.dumps(self.__dict__, ensure_ascii=False)
+
+    def __repr__(self) -> str:
+        return self.__dict__.__repr__()
+
+    @classmethod
+    def from_dict(cls, dictionary: dict):
+        return cls(
+            raw=dictionary["raw"],
+            compiled=dictionary["compiled"],
+            file=dictionary.get("file", None),
+            heading=dictionary.get("heading", None),
+            corpus_id=dictionary.get("corpus_id", None),
+        )

khoj/utils/state.py

@@ -0,0 +1,46 @@
+import os
+import threading
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List
+
+from apscheduler.schedulers.background import BackgroundScheduler
+from openai import OpenAI
+from whisper import Whisper
+
+from khoj.database.models import ProcessLock
+from khoj.processor.embeddings import CrossEncoderModel, EmbeddingsModel
+from khoj.utils import config as utils_config
+from khoj.utils.config import OfflineChatProcessorModel, SearchModels
+from khoj.utils.helpers import LRU, get_device
+from khoj.utils.rawconfig import FullConfig
+
+# Application Global State
+config = FullConfig()
+search_models = SearchModels()
+embeddings_model: Dict[str, EmbeddingsModel] = None
+cross_encoder_model: Dict[str, CrossEncoderModel] = None
+openai_client: OpenAI = None
+offline_chat_processor_config: OfflineChatProcessorModel = None
+whisper_model: Whisper = None
+config_file: Path = None
+verbose: int = 0
+host: str = None
+port: int = None
+cli_args: List[str] = None
+query_cache: Dict[str, LRU] = defaultdict(LRU)
+chat_lock = threading.Lock()
+SearchType = utils_config.SearchType
+scheduler: BackgroundScheduler = None
+schedule_leader_process_lock: ProcessLock = None
+telemetry: List[Dict[str, str]] = []
+khoj_version: str = None
+device = get_device()
+chat_on_gpu: bool = True
+anonymous_mode: bool = False
+pretrained_tokenizers: Dict[str, Any] = dict()
+billing_enabled: bool = (
+    os.getenv("STRIPE_API_KEY") is not None
+    and os.getenv("STRIPE_SIGNING_SECRET") is not None
+    and os.getenv("KHOJ_CLOUD_SUBSCRIPTION_URL") is not None
+)

khoj/utils/yaml.py

@@ -0,0 +1,43 @@
+from pathlib import Path
+
+import yaml
+
+from khoj.utils import state
+from khoj.utils.rawconfig import FullConfig
+
+# Do not emit tags when dumping to YAML
+yaml.emitter.Emitter.process_tag = lambda self, *args, **kwargs: None  # type: ignore[assignment]
+
+
+def save_config_to_file_updated_state():
+    with open(state.config_file, "w") as outfile:
+        yaml.dump(yaml.safe_load(state.config.json(by_alias=True)), outfile)
+        outfile.close()
+    return state.config
+
+
+def save_config_to_file(yaml_config: dict, yaml_config_file: Path):
+    "Write config to YML file"
+    # Create output directory, if it doesn't exist
+    yaml_config_file.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(yaml_config_file, "w", encoding="utf-8") as config_file:
+        yaml.safe_dump(yaml_config, config_file, allow_unicode=True)
+
+
+def load_config_from_file(yaml_config_file: Path) -> dict:
+    "Read config from YML file"
+    config_from_file = None
+    with open(yaml_config_file, "r", encoding="utf-8") as config_file:
+        config_from_file = yaml.safe_load(config_file)
+    return config_from_file
+
+
+def parse_config_from_string(yaml_config: dict) -> FullConfig:
+    "Parse and validate config in YML string"
+    return FullConfig.model_validate(yaml_config)
+
+
+def parse_config_from_file(yaml_config_file):
+    "Parse and validate config in YML file"
+    return parse_config_from_string(load_config_from_file(yaml_config_file))

khoj-1.16.1.dev15.dist-info/METADATA

@@ -0,0 +1,178 @@
+Metadata-Version: 2.3
+Name: khoj
+Version: 1.16.1.dev15
+Summary: An AI copilot for your Second Brain
+Project-URL: Homepage, https://khoj.dev
+Project-URL: Documentation, https://docs.khoj.dev
+Project-URL: Code, https://github.com/khoj-ai/khoj
+Author: Debanjum Singh Solanky, Saba Imran
+License-Expression: AGPL-3.0-or-later
+License-File: LICENSE
+Keywords: AI,NLP,images,markdown,org-mode,pdf,productivity,search,semantic-search
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Information Technology
+Classifier: License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Human Machine Interfaces
+Requires-Python: >=3.10
+Requires-Dist: aiohttp~=3.9.0
+Requires-Dist: anthropic==0.26.1
+Requires-Dist: anyio==3.7.1
+Requires-Dist: apscheduler~=3.10.0
+Requires-Dist: authlib==1.2.1
+Requires-Dist: beautifulsoup4~=4.12.3
+Requires-Dist: cron-descriptor==1.4.3
+Requires-Dist: dateparser>=1.1.1
+Requires-Dist: defusedxml==0.7.1
+Requires-Dist: django-apscheduler==0.6.2
+Requires-Dist: django-phonenumber-field==7.3.0
+Requires-Dist: django==5.0.7
+Requires-Dist: docx2txt==0.8
+Requires-Dist: einops==0.8.0
+Requires-Dist: fastapi>=0.104.1
+Requires-Dist: httpx==0.25.0
+Requires-Dist: huggingface-hub>=0.22.2
+Requires-Dist: itsdangerous==2.1.2
+Requires-Dist: jinja2==3.1.4
+Requires-Dist: langchain-community==0.2.5
+Requires-Dist: langchain-openai==0.1.7
+Requires-Dist: langchain==0.2.5
+Requires-Dist: llama-cpp-python==0.2.76
+Requires-Dist: lxml==4.9.3
+Requires-Dist: magika~=0.5.1
+Requires-Dist: markdown-it-py~=3.0.0
+Requires-Dist: markdownify~=0.11.6
+Requires-Dist: openai-whisper>=20231117
+Requires-Dist: openai>=1.0.0
+Requires-Dist: pgvector==0.2.4
+Requires-Dist: phonenumbers==8.13.27
+Requires-Dist: pillow~=10.0.0
+Requires-Dist: psutil>=5.8.0
+Requires-Dist: psycopg2-binary==2.9.9
+Requires-Dist: pydantic[email]>=2.0.0
+Requires-Dist: pymupdf>=1.23.5
+Requires-Dist: python-multipart>=0.0.7
+Requires-Dist: pytz~=2024.1
+Requires-Dist: pyyaml~=6.0
+Requires-Dist: rapidocr-onnxruntime==1.3.22
+Requires-Dist: requests>=2.26.0
+Requires-Dist: rich>=13.3.1
+Requires-Dist: schedule==1.1.0
+Requires-Dist: sentence-transformers==3.0.1
+Requires-Dist: tenacity==8.3.0
+Requires-Dist: tenacity>=8.2.2
+Requires-Dist: tiktoken>=0.3.2
+Requires-Dist: torch==2.2.2
+Requires-Dist: transformers>=4.28.0
+Requires-Dist: tzdata==2023.3
+Requires-Dist: uvicorn==0.17.6
+Requires-Dist: websockets==12.0
+Provides-Extra: dev
+Requires-Dist: black>=23.1.0; extra == 'dev'
+Requires-Dist: boto3>=1.34.57; extra == 'dev'
+Requires-Dist: factory-boy>=3.2.1; extra == 'dev'
+Requires-Dist: freezegun>=1.2.0; extra == 'dev'
+Requires-Dist: google-auth==2.23.3; extra == 'dev'
+Requires-Dist: gunicorn==22.0.0; extra == 'dev'
+Requires-Dist: mypy>=1.0.1; extra == 'dev'
+Requires-Dist: pre-commit>=3.0.4; extra == 'dev'
+Requires-Dist: pytest-asyncio==0.21.1; extra == 'dev'
+Requires-Dist: pytest-django==4.5.2; extra == 'dev'
+Requires-Dist: pytest-xdist[psutil]; extra == 'dev'
+Requires-Dist: pytest>=7.1.2; extra == 'dev'
+Requires-Dist: resend==1.0.1; extra == 'dev'
+Requires-Dist: stripe==7.3.0; extra == 'dev'
+Requires-Dist: twilio==8.11; extra == 'dev'
+Provides-Extra: prod
+Requires-Dist: boto3>=1.34.57; extra == 'prod'
+Requires-Dist: google-auth==2.23.3; extra == 'prod'
+Requires-Dist: gunicorn==22.0.0; extra == 'prod'
+Requires-Dist: resend==1.0.1; extra == 'prod'
+Requires-Dist: stripe==7.3.0; extra == 'prod'
+Requires-Dist: twilio==8.11; extra == 'prod'
+Description-Content-Type: text/markdown
+
+<p align="center"><img src="src/khoj/interface/web/assets/icons/khoj-logo-sideways-500.png" width="230" alt="Khoj Logo"></p>
+
+<div align="center">
+
+[![test](https://github.com/khoj-ai/khoj/actions/workflows/test.yml/badge.svg)](https://github.com/khoj-ai/khoj/actions/workflows/test.yml)
+[![dockerize](https://github.com/khoj-ai/khoj/actions/workflows/dockerize.yml/badge.svg)](https://github.com/khoj-ai/khoj/pkgs/container/khoj)
+[![pypi](https://github.com/khoj-ai/khoj/actions/workflows/pypi.yml/badge.svg)](https://pypi.org/project/khoj/)
+![Discord](https://img.shields.io/discord/1112065956647284756?style=plastic&label=discord)
+
+</div>
+
+<div align="center">
+<b>The open-source, personal AI for your digital brain</b>
+</div>
+
+<br />
+
+<div align="center">
+
+[🤖 Read Docs](https://docs.khoj.dev)
+<span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
+[🏮 Khoj Cloud](https://khoj.dev)
+<span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
+[💬 Get Involved](https://discord.gg/BDgyabRM6e)
+<span>&nbsp;&nbsp;•&nbsp;&nbsp;</span>
+[📚 Read Blog](https://blog.khoj.dev)
+
+</div>
+
+<div align="left">
+
+***
+
+Khoj is an application that creates always-available, personal AI agents for you to extend your capabilities.
+- You can share your notes and documents to extend your digital brain.
+- Your AI agents have access to the internet, allowing you to incorporate realtime information.
+- Khoj is accessible on Desktop, Emacs, Obsidian, Web and Whatsapp.
+- You can share pdf, markdown, org-mode, notion files and github repositories.
+- You'll get fast, accurate semantic search on top of your docs.
+- Your agents can create deeply personal images and understand your speech.
+- Khoj is open-source, self-hostable. Always.
+
+***
+
+</div>
+
+## See it in action
+
+<img src="https://github.com/khoj-ai/khoj/blob/master/documentation/assets/img/using_khoj_for_studying.gif?raw=true" alt="Khoj Demo">
+
+Go to https://app.khoj.dev to see Khoj live.
+
+## Full feature list
+You can see the full feature list [here](https://docs.khoj.dev/category/features).
+
+## Self-Host
+
+To get started with self-hosting Khoj, [read the docs](https://docs.khoj.dev/get-started/setup).
+
+## Contributors
+Cheers to our awesome contributors! 🎉
+
+<a href="https://github.com/khoj-ai/khoj/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=khoj-ai/khoj" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).
+
+### Interested in Contributing?
+
+We are always looking for contributors to help us build new features, improve the project documentation, or fix bugs. If you're interested, please see our [Contributing Guidelines](https://docs.khoj.dev/contributing/development) and check out our [Contributors Project Board](https://github.com/orgs/khoj-ai/projects/4).
+
+## [Sponsors](https://github.com/sponsors/khoj-ai)
+Shout out to our brilliant sponsors! 🌈
+
+<a href="http://github.com/beekeeb">
+  <img src="https://raw.githubusercontent.com/beekeeb/piantor/main/docs/beekeeb.png" width=250/>
+</a>