atlas-ragkit 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atlas_ragkit-1.0.0/LICENSE +21 -0
- atlas_ragkit-1.0.0/MANIFEST.in +22 -0
- atlas_ragkit-1.0.0/PKG-INFO +93 -0
- atlas_ragkit-1.0.0/README.md +51 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/__init__.py +2 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/main.py +205 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_article.md +130 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_benchmark.json +109 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_code.py +160 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.docx +0 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.pdf +80 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.txt +18 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_page.html +127 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_pipeline.yaml +94 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_presentation.pptx +0 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/code_base.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/default.v1.yaml +31 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/diversity_rrf.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/hybrid_search.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/micro_fast.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/precision_rerank.v1.yaml +31 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/sparse_tfidf.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/tech_docs.v1.yaml +26 -0
- atlas_ragkit-1.0.0/atlas-cli/pyproject.toml +21 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/__init__.py +2 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/execution_context/pipeline_manager.py +177 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/execution_context/services.py +259 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/experiment_context/services.py +158 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/ingestion_context/parsers.py +138 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/project_context/services.py +68 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/studio_context/chunking_lab.py +53 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/studio_context/embedding_studio.py +59 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/evaluation.py +29 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/pipeline.py +44 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/run.py +42 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/providers/storage/sqlite.py +234 -0
- atlas_ragkit-1.0.0/atlas-core/atlas_core/registry/plugin_registry.py +48 -0
- atlas_ragkit-1.0.0/atlas-core/pyproject.toml +22 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/__init__.py +2 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/code_aware.py +282 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/fixed_size.py +159 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/markdown_structure.py +337 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/recursive_character.py +176 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/semantic.py +228 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/sentence_window.py +163 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/embeddings/sentence_transformers_provider.py +33 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/evaluators/rag_triad.py +65 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/interfaces.py +60 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/__init__.py +15 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/error_handling.py +212 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/groq_provider.py +113 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/provider_factory.py +445 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/rerankers/cross_encoder.py +37 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/bm25.py +42 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/chroma.py +59 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/dense.py +41 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/hybrid.py +50 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/parent_document.py +41 -0
- atlas_ragkit-1.0.0/atlas-sdk-python/pyproject.toml +65 -0
- atlas_ragkit-1.0.0/atlas-server/atlas_server/main.py +1042 -0
- atlas_ragkit-1.0.0/atlas-server/atlas_server/static/index.html +4933 -0
- atlas_ragkit-1.0.0/atlas-server/pyproject.toml +20 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/PKG-INFO +93 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/SOURCES.txt +68 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/dependency_links.txt +1 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/entry_points.txt +41 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/requires.txt +21 -0
- atlas_ragkit-1.0.0/atlas_ragkit.egg-info/top_level.txt +4 -0
- atlas_ragkit-1.0.0/pyproject.toml +131 -0
- atlas_ragkit-1.0.0/setup.cfg +4 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Rishi Bhanushali
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2
|
+
# MANIFEST.in — Ensures non-Python files are included in the source distribution
|
|
3
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
4
|
+
|
|
5
|
+
# Web UI (served by atlas-server)
|
|
6
|
+
recursive-include atlas-server/atlas_server/static *
|
|
7
|
+
|
|
8
|
+
# Workspace scaffold templates (copied on atlas init)
|
|
9
|
+
recursive-include atlas-cli/atlas_cli/templates *
|
|
10
|
+
|
|
11
|
+
# Configuration & documentation
|
|
12
|
+
include README.md
|
|
13
|
+
include LICENSE
|
|
14
|
+
include atlas-server/pyproject.toml
|
|
15
|
+
include atlas-cli/pyproject.toml
|
|
16
|
+
include atlas-core/pyproject.toml
|
|
17
|
+
include atlas-sdk-python/pyproject.toml
|
|
18
|
+
|
|
19
|
+
# Exclude build artifacts and caches
|
|
20
|
+
global-exclude *.pyc
|
|
21
|
+
global-exclude __pycache__
|
|
22
|
+
global-exclude *.egg-info
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: atlas-ragkit
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Atlas RAGKit — Local-First RAG Engineering Platform
|
|
5
|
+
Author: Rishi Bhanushali
|
|
6
|
+
License: MIT
|
|
7
|
+
Keywords: rag,llm,ai,local,retrieval,augmented,generation,nlp
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
17
|
+
Requires-Python: >=3.10
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: fastapi>=0.100.0
|
|
21
|
+
Requires-Dist: uvicorn>=0.22.0
|
|
22
|
+
Requires-Dist: python-multipart>=0.0.6
|
|
23
|
+
Requires-Dist: typer>=0.9.0
|
|
24
|
+
Requires-Dist: rich>=13.0.0
|
|
25
|
+
Requires-Dist: pydantic>=2.0.0
|
|
26
|
+
Requires-Dist: pyyaml>=6.0
|
|
27
|
+
Requires-Dist: pymupdf>=1.23.0
|
|
28
|
+
Requires-Dist: python-docx>=1.1.0
|
|
29
|
+
Requires-Dist: beautifulsoup4>=4.12.0
|
|
30
|
+
Requires-Dist: gitpython>=3.1.0
|
|
31
|
+
Requires-Dist: sentence-transformers>=2.2.2
|
|
32
|
+
Requires-Dist: faiss-cpu>=1.7.4
|
|
33
|
+
Requires-Dist: chromadb>=0.4.0
|
|
34
|
+
Requires-Dist: rank-bm25>=0.2.2
|
|
35
|
+
Requires-Dist: groq>=0.9.0
|
|
36
|
+
Requires-Dist: tiktoken>=0.7.0
|
|
37
|
+
Requires-Dist: numpy>=1.24.0
|
|
38
|
+
Requires-Dist: scikit-learn>=1.2.0
|
|
39
|
+
Requires-Dist: umap-learn>=0.5.3
|
|
40
|
+
Requires-Dist: markdown-it-py>=3.0.0
|
|
41
|
+
Dynamic: license-file
|
|
42
|
+
|
|
43
|
+
# ⚡ Atlas — Local-First RAG Engineering Platform
|
|
44
|
+
|
|
45
|
+
[](https://pypi.org/project/atlas-ragkit/)
|
|
46
|
+
[](https://pypi.org/project/atlas-ragkit/)
|
|
47
|
+
[](https://opensource.org/licenses/MIT)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
Atlas is a **local-first, zero-cost** Retrieval-Augmented Generation (RAG) engineering platform with a full-featured visual web dashboard. Ingest documents, benchmark chunking strategies, visualize embeddings in 3D, run RAG queries, and evaluate pipeline quality — all from your browser with no cloud dependency.
|
|
51
|
+
|
|
52
|
+
---
|
|
53
|
+
|
|
54
|
+
## 🚀 Installation
|
|
55
|
+
|
|
56
|
+
`ash
|
|
57
|
+
pip install atlas-rag
|
|
58
|
+
`
|
|
59
|
+
|
|
60
|
+
## ▶️ Quick Start
|
|
61
|
+
|
|
62
|
+
1. Open a terminal and navigate to any directory.
|
|
63
|
+
2. Start the Atlas server:
|
|
64
|
+
`ash
|
|
65
|
+
atlas server
|
|
66
|
+
`
|
|
67
|
+
3. Your browser will open automatically at **http://localhost:8000**.
|
|
68
|
+
4. In the **System Config** tab, click **"Provision Workspace"** to initialize your workspace with demo files and pipelines.
|
|
69
|
+
|
|
70
|
+
## 🏗️ Architecture
|
|
71
|
+
|
|
72
|
+
Atlas is structured as a clean monorepo with four modules:
|
|
73
|
+
|
|
74
|
+
| Module | Role |
|
|
75
|
+
|--------|------|
|
|
76
|
+
| tlas_sdk | Plugin interfaces and reference implementations (chunkers, embedders, retrievers, LLMs) |
|
|
77
|
+
| tlas_core | Domain models, execution engine, ingestion, and studio services |
|
|
78
|
+
| tlas_server | FastAPI REST API + full-featured visual web dashboard (SPA) |
|
|
79
|
+
| tlas_cli | Minimal CLI — boots the server, manages workspace scaffold |
|
|
80
|
+
|
|
81
|
+
## 🔑 Supported LLM Providers
|
|
82
|
+
|
|
83
|
+
Atlas supports **Groq, OpenAI, Anthropic, Google Gemini, Mistral, Together AI, HuggingFace, Ollama, LM Studio**, and local GGUF models. Configure your API keys through the Atlas web UI — no terminal required.
|
|
84
|
+
|
|
85
|
+
## 🗑️ Uninstall
|
|
86
|
+
|
|
87
|
+
`ash
|
|
88
|
+
pip uninstall atlas-rag
|
|
89
|
+
`
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
> Built with ❤️ by Rishi Bhanushali
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# ⚡ Atlas — Local-First RAG Engineering Platform
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/atlas-ragkit/)
|
|
4
|
+
[](https://pypi.org/project/atlas-ragkit/)
|
|
5
|
+
[](https://opensource.org/licenses/MIT)
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
Atlas is a **local-first, zero-cost** Retrieval-Augmented Generation (RAG) engineering platform with a full-featured visual web dashboard. Ingest documents, benchmark chunking strategies, visualize embeddings in 3D, run RAG queries, and evaluate pipeline quality — all from your browser with no cloud dependency.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 🚀 Installation
|
|
13
|
+
|
|
14
|
+
`ash
|
|
15
|
+
pip install atlas-rag
|
|
16
|
+
`
|
|
17
|
+
|
|
18
|
+
## ▶️ Quick Start
|
|
19
|
+
|
|
20
|
+
1. Open a terminal and navigate to any directory.
|
|
21
|
+
2. Start the Atlas server:
|
|
22
|
+
`ash
|
|
23
|
+
atlas server
|
|
24
|
+
`
|
|
25
|
+
3. Your browser will open automatically at **http://localhost:8000**.
|
|
26
|
+
4. In the **System Config** tab, click **"Provision Workspace"** to initialize your workspace with demo files and pipelines.
|
|
27
|
+
|
|
28
|
+
## 🏗️ Architecture
|
|
29
|
+
|
|
30
|
+
Atlas is structured as a clean monorepo with four modules:
|
|
31
|
+
|
|
32
|
+
| Module | Role |
|
|
33
|
+
|--------|------|
|
|
34
|
+
| tlas_sdk | Plugin interfaces and reference implementations (chunkers, embedders, retrievers, LLMs) |
|
|
35
|
+
| tlas_core | Domain models, execution engine, ingestion, and studio services |
|
|
36
|
+
| tlas_server | FastAPI REST API + full-featured visual web dashboard (SPA) |
|
|
37
|
+
| tlas_cli | Minimal CLI — boots the server, manages workspace scaffold |
|
|
38
|
+
|
|
39
|
+
## 🔑 Supported LLM Providers
|
|
40
|
+
|
|
41
|
+
Atlas supports **Groq, OpenAI, Anthropic, Google Gemini, Mistral, Together AI, HuggingFace, Ollama, LM Studio**, and local GGUF models. Configure your API keys through the Atlas web UI — no terminal required.
|
|
42
|
+
|
|
43
|
+
## 🗑️ Uninstall
|
|
44
|
+
|
|
45
|
+
`ash
|
|
46
|
+
pip uninstall atlas-rag
|
|
47
|
+
`
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
> Built with ❤️ by Rishi Bhanushali
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
|
|
2
|
+
import os
|
|
3
|
+
import sys
|
|
4
|
+
import shutil
|
|
5
|
+
|
|
6
|
+
if sys.platform == "win32":
|
|
7
|
+
try:
|
|
8
|
+
sys.stdout.reconfigure(encoding="utf-8")
|
|
9
|
+
sys.stderr.reconfigure(encoding="utf-8")
|
|
10
|
+
except Exception:
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
from rich.console import Console
|
|
14
|
+
from rich.panel import Panel
|
|
15
|
+
from rich.align import Align
|
|
16
|
+
|
|
17
|
+
console = Console(safe_box=True)
|
|
18
|
+
|
|
19
|
+
ATLAS_BANNER_ASCII = r"""
|
|
20
|
+
█████╗ ████████╗ ██╗ █████╗ ███████╗
|
|
21
|
+
██╔══██╗ ╚══██╔══╝ ██║ ██╔══██╗ ██╔════╝
|
|
22
|
+
███████║ ██║ ██║ ███████║ ███████╗
|
|
23
|
+
██╔══██║ ██║ ██║ ██╔══██║ ╚════██║
|
|
24
|
+
██║ ██║ ██║ ███████╗ ██║ ██║ ███████║
|
|
25
|
+
╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝ ╚══════╝
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
DEFAULT_ATLAS_YAML = """atlas_version: "1.0"
|
|
29
|
+
project:
|
|
30
|
+
name: "default-project"
|
|
31
|
+
created_at: "2026-07-26T00:00:00Z"
|
|
32
|
+
|
|
33
|
+
defaults:
|
|
34
|
+
embedding_provider: sentence_transformers/bge-small-en-v1.5
|
|
35
|
+
llm_provider: groq/llama-3.1-8b-instant
|
|
36
|
+
vector_store: faiss
|
|
37
|
+
|
|
38
|
+
resource_limits:
|
|
39
|
+
max_document_mb: null
|
|
40
|
+
max_concurrent_experiments: null
|
|
41
|
+
|
|
42
|
+
active_pipeline: "default@v1"
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
DEFAULT_PIPELINE_YAML = """pipeline:
|
|
46
|
+
name: "default"
|
|
47
|
+
version: 1
|
|
48
|
+
|
|
49
|
+
steps:
|
|
50
|
+
- stage: chunk
|
|
51
|
+
plugin: atlas.chunkers.fixed_size
|
|
52
|
+
params:
|
|
53
|
+
chunk_size: 512
|
|
54
|
+
overlap: 64
|
|
55
|
+
|
|
56
|
+
- stage: embed
|
|
57
|
+
plugin: atlas.embeddings.sentence_transformers
|
|
58
|
+
params:
|
|
59
|
+
model: "BAAI/bge-small-en-v1.5"
|
|
60
|
+
|
|
61
|
+
- stage: retrieve
|
|
62
|
+
plugin: atlas.retrievers.dense
|
|
63
|
+
params:
|
|
64
|
+
top_k: 5
|
|
65
|
+
|
|
66
|
+
- stage: build_context
|
|
67
|
+
plugin: atlas.context.default
|
|
68
|
+
params:
|
|
69
|
+
max_tokens: 3000
|
|
70
|
+
dedupe: true
|
|
71
|
+
|
|
72
|
+
- stage: generate
|
|
73
|
+
plugin: atlas.llm.auto
|
|
74
|
+
params:
|
|
75
|
+
temperature: 0.2
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
DEFAULT_ATLAS_IGNORE = """*.pyc
|
|
79
|
+
__pycache__/
|
|
80
|
+
.env
|
|
81
|
+
cache/
|
|
82
|
+
logs/
|
|
83
|
+
"""
|
|
84
|
+
|
|
85
|
+
SUBDIRECTORIES = [
|
|
86
|
+
"data/raw", "data/processed", "indexes/faiss", "indexes/chroma",
|
|
87
|
+
"pipelines", "prompts", "runs", "traces", "evaluations",
|
|
88
|
+
"experiments", "reports", "exports", "assets", "cache", "logs"
|
|
89
|
+
]
|
|
90
|
+
|
|
91
|
+
def _load_env_file():
|
|
92
|
+
"""Load .env file into os.environ so API keys are always available."""
|
|
93
|
+
env_candidates = [".env", os.path.join(os.getcwd(), ".env")]
|
|
94
|
+
for env_path in env_candidates:
|
|
95
|
+
if os.path.exists(env_path):
|
|
96
|
+
try:
|
|
97
|
+
with open(env_path, "r", encoding="utf-8") as f:
|
|
98
|
+
for line in f:
|
|
99
|
+
line = line.strip()
|
|
100
|
+
if line and not line.startswith("#") and "=" in line:
|
|
101
|
+
k, v = line.split("=", 1)
|
|
102
|
+
k = k.strip()
|
|
103
|
+
v = v.strip().strip("'").strip('"')
|
|
104
|
+
if v: # Only set if non-empty
|
|
105
|
+
os.environ[k] = v
|
|
106
|
+
except Exception:
|
|
107
|
+
pass
|
|
108
|
+
break
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _auto_initialize():
|
|
112
|
+
"""Auto-initialize workspace folders if they do not exist."""
|
|
113
|
+
for folder in SUBDIRECTORIES:
|
|
114
|
+
os.makedirs(folder, exist_ok=True)
|
|
115
|
+
|
|
116
|
+
# Scaffolding: Copy bundled templates (demo_sandbox, pipelines) to the user's workspace
|
|
117
|
+
cli_dir = os.path.dirname(os.path.abspath(__file__))
|
|
118
|
+
templates_dir = os.path.join(cli_dir, "templates")
|
|
119
|
+
|
|
120
|
+
if os.path.exists(templates_dir):
|
|
121
|
+
# copy demo_sandbox
|
|
122
|
+
demo_src = os.path.join(templates_dir, "demo_sandbox")
|
|
123
|
+
if os.path.exists(demo_src) and not os.path.exists("demo_sandbox"):
|
|
124
|
+
shutil.copytree(demo_src, "demo_sandbox")
|
|
125
|
+
|
|
126
|
+
# copy pipelines
|
|
127
|
+
pipelines_src = os.path.join(templates_dir, "pipelines")
|
|
128
|
+
if os.path.exists(pipelines_src):
|
|
129
|
+
os.makedirs("pipelines", exist_ok=True)
|
|
130
|
+
for item in os.listdir(pipelines_src):
|
|
131
|
+
src_file = os.path.join(pipelines_src, item)
|
|
132
|
+
dst_file = os.path.join("pipelines", item)
|
|
133
|
+
if not os.path.exists(dst_file):
|
|
134
|
+
if os.path.isdir(src_file):
|
|
135
|
+
shutil.copytree(src_file, dst_file)
|
|
136
|
+
else:
|
|
137
|
+
shutil.copy2(src_file, dst_file)
|
|
138
|
+
|
|
139
|
+
if not os.path.exists("atlas.yaml"):
|
|
140
|
+
with open("atlas.yaml", "w", encoding="utf-8") as f:
|
|
141
|
+
f.write(DEFAULT_ATLAS_YAML)
|
|
142
|
+
|
|
143
|
+
pipeline_path = os.path.join("pipelines", "default.v1.yaml")
|
|
144
|
+
if not os.path.exists(pipeline_path):
|
|
145
|
+
os.makedirs("pipelines", exist_ok=True)
|
|
146
|
+
with open(pipeline_path, "w", encoding="utf-8") as f:
|
|
147
|
+
f.write(DEFAULT_PIPELINE_YAML)
|
|
148
|
+
|
|
149
|
+
if not os.path.exists(".atlasignore"):
|
|
150
|
+
with open(".atlasignore", "w", encoding="utf-8") as f:
|
|
151
|
+
f.write(DEFAULT_ATLAS_IGNORE)
|
|
152
|
+
|
|
153
|
+
from atlas_core.providers.storage.sqlite import SQLiteStorageProvider
|
|
154
|
+
SQLiteStorageProvider("atlas.db")
|
|
155
|
+
|
|
156
|
+
def print_banner():
|
|
157
|
+
lines = ATLAS_BANNER_ASCII.strip("\n").split("\n")
|
|
158
|
+
colors = [
|
|
159
|
+
"bold bright_cyan", "bold cyan", "bold sky_blue3",
|
|
160
|
+
"bold dodger_blue2", "bold blue", "bold bright_blue"
|
|
161
|
+
]
|
|
162
|
+
console.print("")
|
|
163
|
+
for line, color in zip(lines, colors):
|
|
164
|
+
console.print(Align.center(f"[{color}]{line}[/{color}]"))
|
|
165
|
+
console.print(Align.center("[bold bright_cyan]✨ ⚡ LOCAL-FIRST RAG ENGINEERING PLATFORM ⚡ ✨[/bold bright_cyan]"))
|
|
166
|
+
console.print(Align.center("[dim cyan]─────────────────────────────────────────────────────────────────────────────[/dim cyan]"))
|
|
167
|
+
console.print("")
|
|
168
|
+
console.print(Align.center("[bold bright_blue]═══ ⚡ ATLAS LOCAL WEB SERVER ⚡ ═══[/bold bright_blue]"))
|
|
169
|
+
console.print("")
|
|
170
|
+
|
|
171
|
+
def run_server():
|
|
172
|
+
_load_env_file()
|
|
173
|
+
print_banner()
|
|
174
|
+
|
|
175
|
+
port = 8000
|
|
176
|
+
for arg in sys.argv:
|
|
177
|
+
if arg.startswith("--port="):
|
|
178
|
+
port = int(arg.split("=")[1])
|
|
179
|
+
|
|
180
|
+
url = f"http://localhost:{port}"
|
|
181
|
+
|
|
182
|
+
console.print(Panel(
|
|
183
|
+
f"[bold bright_cyan]🌐 Server Web Dashboard & REST API URL:[/bold bright_cyan]\n"
|
|
184
|
+
f"[bold green] {url}[/bold green]\n\n"
|
|
185
|
+
f"[dim white]Press Ctrl+C in your terminal to stop the server.[/dim white]",
|
|
186
|
+
title="[bold bright_cyan]⚡ ATLAS LOCAL WEB SERVER ACTIVE[/bold bright_cyan]",
|
|
187
|
+
border_style="bright_cyan"
|
|
188
|
+
))
|
|
189
|
+
console.print("")
|
|
190
|
+
|
|
191
|
+
from atlas_server.main import start_server
|
|
192
|
+
start_server(port=port, open_browser=True)
|
|
193
|
+
|
|
194
|
+
def app(*args, **kwargs):
|
|
195
|
+
cli_args = sys.argv[1:]
|
|
196
|
+
if "--help" in cli_args or "-h" in cli_args:
|
|
197
|
+
sys.exit(0)
|
|
198
|
+
|
|
199
|
+
if len(cli_args) > 0 and cli_args[0] == "server":
|
|
200
|
+
run_server()
|
|
201
|
+
else:
|
|
202
|
+
sys.exit(1)
|
|
203
|
+
|
|
204
|
+
if __name__ == "__main__":
|
|
205
|
+
app()
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
# Distributed Consensus & Database Storage Architecture: LSM-Trees, Raft Protocol & Write-Ahead Logging
|
|
2
|
+
|
|
3
|
+
## Executive Summary & System Blueprint
|
|
4
|
+
|
|
5
|
+
Modern high-throughput distributed database engines (such as Apache Cassandra, RocksDB, CockroachDB, and TiKV) must guarantee high write performance, durability (ACID properties), and horizontal consistency across untrusted network partitions. To accomplish this, modern data stores decouple storage persistence from consensus coordination:
|
|
6
|
+
|
|
7
|
+
1. **Storage Subsystem**: Uses **Log-Structured Merge-Trees (LSM-Trees)** for append-only sequential disk writes paired with **B+ Trees** for random read acceleration.
|
|
8
|
+
2. **Consensus Subsystem**: Employs the **Raft Consensus Protocol** (or Multi-Raft) to achieve replicated state machine consistency across distributed cluster nodes.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## 1. Deep Dive into LSM-Tree Storage Engines
|
|
13
|
+
|
|
14
|
+
### 1.1 In-Memory MemTable and Write-Ahead Log (WAL)
|
|
15
|
+
When a write operation `Put(Key, Value)` arrives at a database node:
|
|
16
|
+
- The mutation is first appended sequentially to an on-disk **Write-Ahead Log (WAL)** using `fsync()` to guarantee durability across hardware crashes.
|
|
17
|
+
- Simultaneously, the key-value pair is inserted into an in-memory skip list or red-black tree called the **MemTable**.
|
|
18
|
+
|
|
19
|
+
$$\text{Write Latency} = T_{\text{WAL\_Append}} + T_{\text{MemTable\_Insert}}$$
|
|
20
|
+
|
|
21
|
+
Because both operations avoid disk seek overhead, write throughput often exceeds $100,000 \text{ ops/sec}$ per storage node.
|
|
22
|
+
|
|
23
|
+
### 1.2 SSTable Flushing and Multi-Tiered Compaction
|
|
24
|
+
When the MemTable reaches its capacity threshold (typically $64\text{ MB}$ to $256\text{ MB}$), it transitions into an **Immutable MemTable** and is flushed to disk as a **Sorted String Table (SSTable)** at Level 0 ($L_0$).
|
|
25
|
+
|
|
26
|
+
```text
|
|
27
|
+
+-----------------------------------------------------------------------+
|
|
28
|
+
| WRITE PATH |
|
|
29
|
+
| |
|
|
30
|
+
| Client Put(K,V) ---> [ WAL File (Disk) ] (Sequential Append) |
|
|
31
|
+
| | |
|
|
32
|
+
| v |
|
|
33
|
+
| [ MemTable (RAM) ] (SkipList / Red-Black Tree) |
|
|
34
|
+
| | |
|
|
35
|
+
| v (Flush when MemTable >= 64MB) |
|
|
36
|
+
| [ SSTable L0 (Disk) ] |
|
|
37
|
+
+-----------------------------------------------------------------------+
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
To limit read amplification, the storage engine runs background **Size-Tiered Compaction** or **Leveled Compaction**:
|
|
41
|
+
|
|
42
|
+
$$\text{Compaction Ratio} = \frac{\sum \text{SizeBytes}(L_{i+1})}{\sum \text{SizeBytes}(L_i)} \approx 10$$
|
|
43
|
+
|
|
44
|
+
### 1.3 Bloom Filter Optimization for Read Path Acceleration
|
|
45
|
+
To verify if a key exists within an SSTable without issuing expensive disk reads, every SSTable maintains an in-memory **Bloom Filter**:
|
|
46
|
+
|
|
47
|
+
$$m = -\frac{n \cdot \ln(p)}{(\ln 2)^2}$$
|
|
48
|
+
|
|
49
|
+
Where:
|
|
50
|
+
- $m$ is the bit array size.
|
|
51
|
+
- $n$ is the number of inserted keys.
|
|
52
|
+
- $p$ is the target false positive probability (e.g., $0.01$).
|
|
53
|
+
- $k = \frac{m}{n} \ln 2$ is the optimal number of hash functions.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## 2. Distributed Consensus: The Raft Protocol
|
|
58
|
+
|
|
59
|
+
### 2.1 Node States and Term Epochs
|
|
60
|
+
A Raft cluster consists of $2F + 1$ nodes to tolerate up to $F$ concurrent node failures. Each node transitions between three mutually exclusive states:
|
|
61
|
+
- **Leader**: Handles all client read/write requests and manages log replication.
|
|
62
|
+
- **Follower**: Passive state; responds to incoming RPCs from Leaders and Candidates.
|
|
63
|
+
- **Candidate**: Active state during leader elections.
|
|
64
|
+
|
|
65
|
+
```text
|
|
66
|
+
+------------------------+
|
|
67
|
+
| Times out, starts |
|
|
68
|
+
| Leader Election |
|
|
69
|
+
v |
|
|
70
|
+
+--------------+ +---------------+
|
|
71
|
+
| Follower | -----> | Candidate |
|
|
72
|
+
+--------------+ +---------------+
|
|
73
|
+
^ |
|
|
74
|
+
| Receives AppendEntries | Wins Election
|
|
75
|
+
| from valid Leader | with Majority
|
|
76
|
+
| v
|
|
77
|
+
+----------------+--------------+
|
|
78
|
+
| Leader |
|
|
79
|
+
+--------------+
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### 2.2 Replicated Log Matching Property
|
|
83
|
+
If two logs on different nodes contain an entry with the same index and term:
|
|
84
|
+
1. They store identical commands.
|
|
85
|
+
2. Their logs are identical in all preceding entries up to that index.
|
|
86
|
+
|
|
87
|
+
$$\forall i, j \quad (\text{Log}_A[i].\text{term} == \text{Log}_B[i].\text{term}) \implies \forall k \le i, \, \text{Log}_A[k] == \text{Log}_B[k]$$
|
|
88
|
+
|
|
89
|
+
### 2.3 Quorum Commit Rule
|
|
90
|
+
A log entry is considered **Committed** once it is safely replicated on a majority quorum of nodes:
|
|
91
|
+
|
|
92
|
+
$$\text{QuorumSize} = \left\lfloor \frac{N}{2} \right\rfloor + 1$$
|
|
93
|
+
|
|
94
|
+
---
|
|
95
|
+
|
|
96
|
+
## 3. Benchmarking Summary & Architectural Comparison
|
|
97
|
+
|
|
98
|
+
| Metric / Parameter | B+ Tree Storage (InnoDB) | LSM-Tree Storage (RocksDB) | Raft Replicated Log |
|
|
99
|
+
|---|---|---|---|
|
|
100
|
+
| **Write Pattern** | In-place random disk writes | Append-only sequential writes | Sequential log append |
|
|
101
|
+
| **Write Amplification** | Moderate ($2\times - 5\times$) | High ($10\times - 30\times$) | Low ($1\times - 2\times$) |
|
|
102
|
+
| **Read Amplification** | Low ($1\times - 2\times$) | High ($3\times - 10\times$) | N/A (Leader reads) |
|
|
103
|
+
| **Space Amplification** | Low ($1.1\times - 1.3\times$) | Moderate ($1.2\times - 2.0\times$) | $N \times$ (Replica count) |
|
|
104
|
+
| **Failure Recovery** | Crash recovery via REDO log | Flush MemTable via WAL | State catch-up via Snapshot |
|
|
105
|
+
|
|
106
|
+
---
|
|
107
|
+
|
|
108
|
+
## 4. Production Code Reference: Raft Log Truncation & Compaction
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
class RaftReplicatedLog:
|
|
112
|
+
def __init__(self, node_id: str, storage_path: str):
|
|
113
|
+
self.node_id = node_id
|
|
114
|
+
self.storage_path = storage_path
|
|
115
|
+
self.entries = []
|
|
116
|
+
self.commit_index = 0
|
|
117
|
+
self.last_applied = 0
|
|
118
|
+
|
|
119
|
+
def append_entry(self, term: int, command: dict) -> int:
|
|
120
|
+
index = len(self.entries) + 1
|
|
121
|
+
entry = {"index": index, "term": term, "command": command}
|
|
122
|
+
self.entries.append(entry)
|
|
123
|
+
return index
|
|
124
|
+
|
|
125
|
+
def truncate_uncommitted(self, from_index: int) -> None:
|
|
126
|
+
"""Removes conflicting uncommitted entries following a failed leader term."""
|
|
127
|
+
if from_index <= self.commit_index:
|
|
128
|
+
raise ValueError("Cannot truncate committed log entries!")
|
|
129
|
+
self.entries = self.entries[:from_index - 1]
|
|
130
|
+
```
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
{
|
|
2
|
+
"dataset_metadata": {
|
|
3
|
+
"benchmark_name": "MedQA-Clinical-EHR-Diagnostic-Eval-v4.2",
|
|
4
|
+
"domain": "Clinical Medicine & Electronic Health Record Diagnostics",
|
|
5
|
+
"release_date": "2026-06-15",
|
|
6
|
+
"compliance": ["HIPAA-Sec-164.312", "FDA-SaMD-Class-II", "EU-MDR-2017/745"],
|
|
7
|
+
"total_clinical_cases": 1250,
|
|
8
|
+
"evaluated_models": [
|
|
9
|
+
"Med-PaLM-2-70B",
|
|
10
|
+
"BioGPT-Large",
|
|
11
|
+
"Clinical-LLaMA-3-8B",
|
|
12
|
+
"Atlas-Medical-RAG-v1"
|
|
13
|
+
]
|
|
14
|
+
},
|
|
15
|
+
"evaluation_metrics_summary": {
|
|
16
|
+
"diagnostic_accuracy_top1": 0.884,
|
|
17
|
+
"diagnostic_accuracy_top3": 0.962,
|
|
18
|
+
"contraindication_detection_recall": 0.991,
|
|
19
|
+
"dosage_calculation_precision": 0.978,
|
|
20
|
+
"mean_retrieval_latency_ms": 142.6,
|
|
21
|
+
"p99_retrieval_latency_ms": 389.2,
|
|
22
|
+
"hipaa_audit_pass_rate": 1.0
|
|
23
|
+
},
|
|
24
|
+
"diagnostic_subcategories": [
|
|
25
|
+
{
|
|
26
|
+
"category_id": "CARDIO-001",
|
|
27
|
+
"category_name": "Cardiology & Electrocardiography Diagnostics",
|
|
28
|
+
"sample_count": 320,
|
|
29
|
+
"key_pathologies": [
|
|
30
|
+
"ST-Elevation Myocardial Infarction (STEMI)",
|
|
31
|
+
"Atrial Fibrillation with Rapid Ventricular Response",
|
|
32
|
+
"Hypertrophic Obstructive Cardiomyopathy",
|
|
33
|
+
"Decompensated Congestive Heart Failure"
|
|
34
|
+
],
|
|
35
|
+
"performance_breakdown": {
|
|
36
|
+
"precision": 0.912,
|
|
37
|
+
"recall": 0.895,
|
|
38
|
+
"f1_score": 0.903,
|
|
39
|
+
"hallucination_rate": 0.008
|
|
40
|
+
},
|
|
41
|
+
"ehr_structured_fields": {
|
|
42
|
+
"required_vitals": ["systolic_bp", "diastolic_bp", "heart_rate", "spo2", "troponin_i"],
|
|
43
|
+
"icd_10_codes": ["I21.0", "I48.0", "I42.1", "I50.9"]
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"category_id": "NEURO-002",
|
|
48
|
+
"category_name": "Neurology & Acute Stroke Protocols",
|
|
49
|
+
"sample_count": 280,
|
|
50
|
+
"key_pathologies": [
|
|
51
|
+
"Acute Ischemic Stroke (Middle Cerebral Artery Territory)",
|
|
52
|
+
"Subarachnoid Hemorrhage",
|
|
53
|
+
"Refractory Status Epilepticus",
|
|
54
|
+
"Guillain-Barré Syndrome"
|
|
55
|
+
],
|
|
56
|
+
"performance_breakdown": {
|
|
57
|
+
"precision": 0.879,
|
|
58
|
+
"recall": 0.921,
|
|
59
|
+
"f1_score": 0.899,
|
|
60
|
+
"hallucination_rate": 0.012
|
|
61
|
+
},
|
|
62
|
+
"ehr_structured_fields": {
|
|
63
|
+
"required_vitals": ["nihss_score", "gcs_score", "mean_arterial_pressure"],
|
|
64
|
+
"icd_10_codes": ["I63.5", "I60.9", "G41.9", "G61.0"]
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"category_id": "ONCO-003",
|
|
69
|
+
"category_name": "Oncology & Biomarker Therapeutics",
|
|
70
|
+
"sample_count": 450,
|
|
71
|
+
"key_pathologies": [
|
|
72
|
+
"Metastatic Non-Small Cell Lung Cancer (EGFR T790M+)",
|
|
73
|
+
"HER2-Positive Invasive Breast Carcinoma",
|
|
74
|
+
"Triple-Negative Breast Cancer",
|
|
75
|
+
"Multiple Myeloma"
|
|
76
|
+
],
|
|
77
|
+
"performance_breakdown": {
|
|
78
|
+
"precision": 0.941,
|
|
79
|
+
"recall": 0.935,
|
|
80
|
+
"f1_score": 0.938,
|
|
81
|
+
"hallucination_rate": 0.003
|
|
82
|
+
},
|
|
83
|
+
"ehr_structured_fields": {
|
|
84
|
+
"required_vitals": ["pd_l1_expression_percent", "alk_rearrangement", "brca1_mutation"],
|
|
85
|
+
"icd_10_codes": ["C34.90", "C50.911", "C90.00"]
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
],
|
|
89
|
+
"retrieval_augmented_generation_telemetry": {
|
|
90
|
+
"vector_database_config": {
|
|
91
|
+
"index_type": "HNSW",
|
|
92
|
+
"distance_metric": "Cosine",
|
|
93
|
+
"m_parameter": 32,
|
|
94
|
+
"ef_construction": 200,
|
|
95
|
+
"embedding_model": "BAAI/bge-m3",
|
|
96
|
+
"dimensions": 1024
|
|
97
|
+
},
|
|
98
|
+
"lexical_bm25_config": {
|
|
99
|
+
"k1": 1.4,
|
|
100
|
+
"b": 0.75,
|
|
101
|
+
"tokenizer": "UnicodeMedicalSubwordTokenizer"
|
|
102
|
+
},
|
|
103
|
+
"rrf_fusion_parameters": {
|
|
104
|
+
"rank_constant_k": 60,
|
|
105
|
+
"weight_dense": 0.65,
|
|
106
|
+
"weight_sparse": 0.35
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|