atlas-ragkit 1.0.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. atlas_ragkit-1.0.0/LICENSE +21 -0
  2. atlas_ragkit-1.0.0/MANIFEST.in +22 -0
  3. atlas_ragkit-1.0.0/PKG-INFO +93 -0
  4. atlas_ragkit-1.0.0/README.md +51 -0
  5. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/__init__.py +2 -0
  6. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/main.py +205 -0
  7. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_article.md +130 -0
  8. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_benchmark.json +109 -0
  9. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_code.py +160 -0
  10. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.docx +0 -0
  11. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.pdf +80 -0
  12. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_document.txt +18 -0
  13. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_page.html +127 -0
  14. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_pipeline.yaml +94 -0
  15. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/demo_sandbox/demo_presentation.pptx +0 -0
  16. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/code_base.v1.yaml +26 -0
  17. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/default.v1.yaml +31 -0
  18. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/diversity_rrf.v1.yaml +26 -0
  19. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/hybrid_search.v1.yaml +26 -0
  20. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/micro_fast.v1.yaml +26 -0
  21. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/precision_rerank.v1.yaml +31 -0
  22. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/sparse_tfidf.v1.yaml +26 -0
  23. atlas_ragkit-1.0.0/atlas-cli/atlas_cli/templates/pipelines/tech_docs.v1.yaml +26 -0
  24. atlas_ragkit-1.0.0/atlas-cli/pyproject.toml +21 -0
  25. atlas_ragkit-1.0.0/atlas-core/atlas_core/__init__.py +2 -0
  26. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/execution_context/pipeline_manager.py +177 -0
  27. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/execution_context/services.py +259 -0
  28. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/experiment_context/services.py +158 -0
  29. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/ingestion_context/parsers.py +138 -0
  30. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/project_context/services.py +68 -0
  31. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/studio_context/chunking_lab.py +53 -0
  32. atlas_ragkit-1.0.0/atlas-core/atlas_core/contexts/studio_context/embedding_studio.py +59 -0
  33. atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/evaluation.py +29 -0
  34. atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/pipeline.py +44 -0
  35. atlas_ragkit-1.0.0/atlas-core/atlas_core/domain/run.py +42 -0
  36. atlas_ragkit-1.0.0/atlas-core/atlas_core/providers/storage/sqlite.py +234 -0
  37. atlas_ragkit-1.0.0/atlas-core/atlas_core/registry/plugin_registry.py +48 -0
  38. atlas_ragkit-1.0.0/atlas-core/pyproject.toml +22 -0
  39. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/__init__.py +2 -0
  40. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/code_aware.py +282 -0
  41. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/fixed_size.py +159 -0
  42. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/markdown_structure.py +337 -0
  43. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/recursive_character.py +176 -0
  44. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/semantic.py +228 -0
  45. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/chunkers/sentence_window.py +163 -0
  46. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/embeddings/sentence_transformers_provider.py +33 -0
  47. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/evaluators/rag_triad.py +65 -0
  48. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/interfaces.py +60 -0
  49. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/__init__.py +15 -0
  50. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/error_handling.py +212 -0
  51. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/groq_provider.py +113 -0
  52. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/llm/provider_factory.py +445 -0
  53. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/rerankers/cross_encoder.py +37 -0
  54. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/bm25.py +42 -0
  55. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/chroma.py +59 -0
  56. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/dense.py +41 -0
  57. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/hybrid.py +50 -0
  58. atlas_ragkit-1.0.0/atlas-sdk-python/atlas_sdk/retrievers/parent_document.py +41 -0
  59. atlas_ragkit-1.0.0/atlas-sdk-python/pyproject.toml +65 -0
  60. atlas_ragkit-1.0.0/atlas-server/atlas_server/main.py +1042 -0
  61. atlas_ragkit-1.0.0/atlas-server/atlas_server/static/index.html +4933 -0
  62. atlas_ragkit-1.0.0/atlas-server/pyproject.toml +20 -0
  63. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/PKG-INFO +93 -0
  64. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/SOURCES.txt +68 -0
  65. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/dependency_links.txt +1 -0
  66. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/entry_points.txt +41 -0
  67. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/requires.txt +21 -0
  68. atlas_ragkit-1.0.0/atlas_ragkit.egg-info/top_level.txt +4 -0
  69. atlas_ragkit-1.0.0/pyproject.toml +131 -0
  70. atlas_ragkit-1.0.0/setup.cfg +4 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Rishi Bhanushali
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,22 @@
1
+ # ─────────────────────────────────────────────────────────────────────────────
2
+ # MANIFEST.in — Ensures non-Python files are included in the source distribution
3
+ # ─────────────────────────────────────────────────────────────────────────────
4
+
5
+ # Web UI (served by atlas-server)
6
+ recursive-include atlas-server/atlas_server/static *
7
+
8
+ # Workspace scaffold templates (copied on atlas init)
9
+ recursive-include atlas-cli/atlas_cli/templates *
10
+
11
+ # Configuration & documentation
12
+ include README.md
13
+ include LICENSE
14
+ include atlas-server/pyproject.toml
15
+ include atlas-cli/pyproject.toml
16
+ include atlas-core/pyproject.toml
17
+ include atlas-sdk-python/pyproject.toml
18
+
19
+ # Exclude build artifacts and caches
20
+ global-exclude *.pyc
21
+ global-exclude __pycache__
22
+ global-exclude *.egg-info
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: atlas-ragkit
3
+ Version: 1.0.0
4
+ Summary: Atlas RAGKit — Local-First RAG Engineering Platform
5
+ Author: Rishi Bhanushali
6
+ License: MIT
7
+ Keywords: rag,llm,ai,local,retrieval,augmented,generation,nlp
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: MIT License
12
+ Classifier: Programming Language :: Python :: 3
13
+ Classifier: Programming Language :: Python :: 3.10
14
+ Classifier: Programming Language :: Python :: 3.11
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
17
+ Requires-Python: >=3.10
18
+ Description-Content-Type: text/markdown
19
+ License-File: LICENSE
20
+ Requires-Dist: fastapi>=0.100.0
21
+ Requires-Dist: uvicorn>=0.22.0
22
+ Requires-Dist: python-multipart>=0.0.6
23
+ Requires-Dist: typer>=0.9.0
24
+ Requires-Dist: rich>=13.0.0
25
+ Requires-Dist: pydantic>=2.0.0
26
+ Requires-Dist: pyyaml>=6.0
27
+ Requires-Dist: pymupdf>=1.23.0
28
+ Requires-Dist: python-docx>=1.1.0
29
+ Requires-Dist: beautifulsoup4>=4.12.0
30
+ Requires-Dist: gitpython>=3.1.0
31
+ Requires-Dist: sentence-transformers>=2.2.2
32
+ Requires-Dist: faiss-cpu>=1.7.4
33
+ Requires-Dist: chromadb>=0.4.0
34
+ Requires-Dist: rank-bm25>=0.2.2
35
+ Requires-Dist: groq>=0.9.0
36
+ Requires-Dist: tiktoken>=0.7.0
37
+ Requires-Dist: numpy>=1.24.0
38
+ Requires-Dist: scikit-learn>=1.2.0
39
+ Requires-Dist: umap-learn>=0.5.3
40
+ Requires-Dist: markdown-it-py>=3.0.0
41
+ Dynamic: license-file
42
+
43
+ # ⚡ Atlas — Local-First RAG Engineering Platform
44
+
45
+ [![PyPI version](https://img.shields.io/pypi/v/atlas-ragkit)](https://pypi.org/project/atlas-ragkit/)
46
+ [![Python](https://img.shields.io/pypi/pyversions/atlas-ragkit)](https://pypi.org/project/atlas-ragkit/)
47
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
48
+
49
+
50
+ Atlas is a **local-first, zero-cost** Retrieval-Augmented Generation (RAG) engineering platform with a full-featured visual web dashboard. Ingest documents, benchmark chunking strategies, visualize embeddings in 3D, run RAG queries, and evaluate pipeline quality — all from your browser with no cloud dependency.
51
+
52
+ ---
53
+
54
+ ## 🚀 Installation
55
+
56
+ `ash
57
+ pip install atlas-rag
58
+ `
59
+
60
+ ## ▶️ Quick Start
61
+
62
+ 1. Open a terminal and navigate to any directory.
63
+ 2. Start the Atlas server:
64
+ `ash
65
+ atlas server
66
+ `
67
+ 3. Your browser will open automatically at **http://localhost:8000**.
68
+ 4. In the **System Config** tab, click **"Provision Workspace"** to initialize your workspace with demo files and pipelines.
69
+
70
+ ## 🏗️ Architecture
71
+
72
+ Atlas is structured as a clean monorepo with four modules:
73
+
74
+ | Module | Role |
75
+ |--------|------|
76
+ | tlas_sdk | Plugin interfaces and reference implementations (chunkers, embedders, retrievers, LLMs) |
77
+ | tlas_core | Domain models, execution engine, ingestion, and studio services |
78
+ | tlas_server | FastAPI REST API + full-featured visual web dashboard (SPA) |
79
+ | tlas_cli | Minimal CLI — boots the server, manages workspace scaffold |
80
+
81
+ ## 🔑 Supported LLM Providers
82
+
83
+ Atlas supports **Groq, OpenAI, Anthropic, Google Gemini, Mistral, Together AI, HuggingFace, Ollama, LM Studio**, and local GGUF models. Configure your API keys through the Atlas web UI — no terminal required.
84
+
85
+ ## 🗑️ Uninstall
86
+
87
+ `ash
88
+ pip uninstall atlas-rag
89
+ `
90
+
91
+ ---
92
+
93
+ > Built with ❤️ by Rishi Bhanushali
@@ -0,0 +1,51 @@
1
+ # ⚡ Atlas — Local-First RAG Engineering Platform
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/atlas-ragkit)](https://pypi.org/project/atlas-ragkit/)
4
+ [![Python](https://img.shields.io/pypi/pyversions/atlas-ragkit)](https://pypi.org/project/atlas-ragkit/)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
6
+
7
+
8
+ Atlas is a **local-first, zero-cost** Retrieval-Augmented Generation (RAG) engineering platform with a full-featured visual web dashboard. Ingest documents, benchmark chunking strategies, visualize embeddings in 3D, run RAG queries, and evaluate pipeline quality — all from your browser with no cloud dependency.
9
+
10
+ ---
11
+
12
+ ## 🚀 Installation
13
+
14
+ `ash
15
+ pip install atlas-rag
16
+ `
17
+
18
+ ## ▶️ Quick Start
19
+
20
+ 1. Open a terminal and navigate to any directory.
21
+ 2. Start the Atlas server:
22
+ `ash
23
+ atlas server
24
+ `
25
+ 3. Your browser will open automatically at **http://localhost:8000**.
26
+ 4. In the **System Config** tab, click **"Provision Workspace"** to initialize your workspace with demo files and pipelines.
27
+
28
+ ## 🏗️ Architecture
29
+
30
+ Atlas is structured as a clean monorepo with four modules:
31
+
32
+ | Module | Role |
33
+ |--------|------|
34
+ | tlas_sdk | Plugin interfaces and reference implementations (chunkers, embedders, retrievers, LLMs) |
35
+ | tlas_core | Domain models, execution engine, ingestion, and studio services |
36
+ | tlas_server | FastAPI REST API + full-featured visual web dashboard (SPA) |
37
+ | tlas_cli | Minimal CLI — boots the server, manages workspace scaffold |
38
+
39
+ ## 🔑 Supported LLM Providers
40
+
41
+ Atlas supports **Groq, OpenAI, Anthropic, Google Gemini, Mistral, Together AI, HuggingFace, Ollama, LM Studio**, and local GGUF models. Configure your API keys through the Atlas web UI — no terminal required.
42
+
43
+ ## 🗑️ Uninstall
44
+
45
+ `ash
46
+ pip uninstall atlas-rag
47
+ `
48
+
49
+ ---
50
+
51
+ > Built with ❤️ by Rishi Bhanushali
@@ -0,0 +1,2 @@
1
+ """Atlas CLI package."""
2
+ __version__ = "0.1.0"
@@ -0,0 +1,205 @@
1
+
2
+ import os
3
+ import sys
4
+ import shutil
5
+
6
+ if sys.platform == "win32":
7
+ try:
8
+ sys.stdout.reconfigure(encoding="utf-8")
9
+ sys.stderr.reconfigure(encoding="utf-8")
10
+ except Exception:
11
+ pass
12
+
13
+ from rich.console import Console
14
+ from rich.panel import Panel
15
+ from rich.align import Align
16
+
17
+ console = Console(safe_box=True)
18
+
19
+ ATLAS_BANNER_ASCII = r"""
20
+ █████╗ ████████╗ ██╗ █████╗ ███████╗
21
+ ██╔══██╗ ╚══██╔══╝ ██║ ██╔══██╗ ██╔════╝
22
+ ███████║ ██║ ██║ ███████║ ███████╗
23
+ ██╔══██║ ██║ ██║ ██╔══██║ ╚════██║
24
+ ██║ ██║ ██║ ███████╗ ██║ ██║ ███████║
25
+ ╚═╝ ╚═╝ ╚═╝ ╚══════╝ ╚═╝ ╚═╝ ╚══════╝
26
+ """
27
+
28
+ DEFAULT_ATLAS_YAML = """atlas_version: "1.0"
29
+ project:
30
+ name: "default-project"
31
+ created_at: "2026-07-26T00:00:00Z"
32
+
33
+ defaults:
34
+ embedding_provider: sentence_transformers/bge-small-en-v1.5
35
+ llm_provider: groq/llama-3.1-8b-instant
36
+ vector_store: faiss
37
+
38
+ resource_limits:
39
+ max_document_mb: null
40
+ max_concurrent_experiments: null
41
+
42
+ active_pipeline: "default@v1"
43
+ """
44
+
45
+ DEFAULT_PIPELINE_YAML = """pipeline:
46
+ name: "default"
47
+ version: 1
48
+
49
+ steps:
50
+ - stage: chunk
51
+ plugin: atlas.chunkers.fixed_size
52
+ params:
53
+ chunk_size: 512
54
+ overlap: 64
55
+
56
+ - stage: embed
57
+ plugin: atlas.embeddings.sentence_transformers
58
+ params:
59
+ model: "BAAI/bge-small-en-v1.5"
60
+
61
+ - stage: retrieve
62
+ plugin: atlas.retrievers.dense
63
+ params:
64
+ top_k: 5
65
+
66
+ - stage: build_context
67
+ plugin: atlas.context.default
68
+ params:
69
+ max_tokens: 3000
70
+ dedupe: true
71
+
72
+ - stage: generate
73
+ plugin: atlas.llm.auto
74
+ params:
75
+ temperature: 0.2
76
+ """
77
+
78
+ DEFAULT_ATLAS_IGNORE = """*.pyc
79
+ __pycache__/
80
+ .env
81
+ cache/
82
+ logs/
83
+ """
84
+
85
+ SUBDIRECTORIES = [
86
+ "data/raw", "data/processed", "indexes/faiss", "indexes/chroma",
87
+ "pipelines", "prompts", "runs", "traces", "evaluations",
88
+ "experiments", "reports", "exports", "assets", "cache", "logs"
89
+ ]
90
+
91
+ def _load_env_file():
92
+ """Load .env file into os.environ so API keys are always available."""
93
+ env_candidates = [".env", os.path.join(os.getcwd(), ".env")]
94
+ for env_path in env_candidates:
95
+ if os.path.exists(env_path):
96
+ try:
97
+ with open(env_path, "r", encoding="utf-8") as f:
98
+ for line in f:
99
+ line = line.strip()
100
+ if line and not line.startswith("#") and "=" in line:
101
+ k, v = line.split("=", 1)
102
+ k = k.strip()
103
+ v = v.strip().strip("'").strip('"')
104
+ if v: # Only set if non-empty
105
+ os.environ[k] = v
106
+ except Exception:
107
+ pass
108
+ break
109
+
110
+
111
+ def _auto_initialize():
112
+ """Auto-initialize workspace folders if they do not exist."""
113
+ for folder in SUBDIRECTORIES:
114
+ os.makedirs(folder, exist_ok=True)
115
+
116
+ # Scaffolding: Copy bundled templates (demo_sandbox, pipelines) to the user's workspace
117
+ cli_dir = os.path.dirname(os.path.abspath(__file__))
118
+ templates_dir = os.path.join(cli_dir, "templates")
119
+
120
+ if os.path.exists(templates_dir):
121
+ # copy demo_sandbox
122
+ demo_src = os.path.join(templates_dir, "demo_sandbox")
123
+ if os.path.exists(demo_src) and not os.path.exists("demo_sandbox"):
124
+ shutil.copytree(demo_src, "demo_sandbox")
125
+
126
+ # copy pipelines
127
+ pipelines_src = os.path.join(templates_dir, "pipelines")
128
+ if os.path.exists(pipelines_src):
129
+ os.makedirs("pipelines", exist_ok=True)
130
+ for item in os.listdir(pipelines_src):
131
+ src_file = os.path.join(pipelines_src, item)
132
+ dst_file = os.path.join("pipelines", item)
133
+ if not os.path.exists(dst_file):
134
+ if os.path.isdir(src_file):
135
+ shutil.copytree(src_file, dst_file)
136
+ else:
137
+ shutil.copy2(src_file, dst_file)
138
+
139
+ if not os.path.exists("atlas.yaml"):
140
+ with open("atlas.yaml", "w", encoding="utf-8") as f:
141
+ f.write(DEFAULT_ATLAS_YAML)
142
+
143
+ pipeline_path = os.path.join("pipelines", "default.v1.yaml")
144
+ if not os.path.exists(pipeline_path):
145
+ os.makedirs("pipelines", exist_ok=True)
146
+ with open(pipeline_path, "w", encoding="utf-8") as f:
147
+ f.write(DEFAULT_PIPELINE_YAML)
148
+
149
+ if not os.path.exists(".atlasignore"):
150
+ with open(".atlasignore", "w", encoding="utf-8") as f:
151
+ f.write(DEFAULT_ATLAS_IGNORE)
152
+
153
+ from atlas_core.providers.storage.sqlite import SQLiteStorageProvider
154
+ SQLiteStorageProvider("atlas.db")
155
+
156
+ def print_banner():
157
+ lines = ATLAS_BANNER_ASCII.strip("\n").split("\n")
158
+ colors = [
159
+ "bold bright_cyan", "bold cyan", "bold sky_blue3",
160
+ "bold dodger_blue2", "bold blue", "bold bright_blue"
161
+ ]
162
+ console.print("")
163
+ for line, color in zip(lines, colors):
164
+ console.print(Align.center(f"[{color}]{line}[/{color}]"))
165
+ console.print(Align.center("[bold bright_cyan]✨ ⚡ LOCAL-FIRST RAG ENGINEERING PLATFORM ⚡ ✨[/bold bright_cyan]"))
166
+ console.print(Align.center("[dim cyan]─────────────────────────────────────────────────────────────────────────────[/dim cyan]"))
167
+ console.print("")
168
+ console.print(Align.center("[bold bright_blue]═══ ⚡ ATLAS LOCAL WEB SERVER ⚡ ═══[/bold bright_blue]"))
169
+ console.print("")
170
+
171
+ def run_server():
172
+ _load_env_file()
173
+ print_banner()
174
+
175
+ port = 8000
176
+ for arg in sys.argv:
177
+ if arg.startswith("--port="):
178
+ port = int(arg.split("=")[1])
179
+
180
+ url = f"http://localhost:{port}"
181
+
182
+ console.print(Panel(
183
+ f"[bold bright_cyan]🌐 Server Web Dashboard & REST API URL:[/bold bright_cyan]\n"
184
+ f"[bold green] {url}[/bold green]\n\n"
185
+ f"[dim white]Press Ctrl+C in your terminal to stop the server.[/dim white]",
186
+ title="[bold bright_cyan]⚡ ATLAS LOCAL WEB SERVER ACTIVE[/bold bright_cyan]",
187
+ border_style="bright_cyan"
188
+ ))
189
+ console.print("")
190
+
191
+ from atlas_server.main import start_server
192
+ start_server(port=port, open_browser=True)
193
+
194
+ def app(*args, **kwargs):
195
+ cli_args = sys.argv[1:]
196
+ if "--help" in cli_args or "-h" in cli_args:
197
+ sys.exit(0)
198
+
199
+ if len(cli_args) > 0 and cli_args[0] == "server":
200
+ run_server()
201
+ else:
202
+ sys.exit(1)
203
+
204
+ if __name__ == "__main__":
205
+ app()
@@ -0,0 +1,130 @@
1
+ # Distributed Consensus & Database Storage Architecture: LSM-Trees, Raft Protocol & Write-Ahead Logging
2
+
3
+ ## Executive Summary & System Blueprint
4
+
5
+ Modern high-throughput distributed database engines (such as Apache Cassandra, RocksDB, CockroachDB, and TiKV) must guarantee high write performance, durability (ACID properties), and horizontal consistency across untrusted network partitions. To accomplish this, modern data stores decouple storage persistence from consensus coordination:
6
+
7
+ 1. **Storage Subsystem**: Uses **Log-Structured Merge-Trees (LSM-Trees)** for append-only sequential disk writes paired with **B+ Trees** for random read acceleration.
8
+ 2. **Consensus Subsystem**: Employs the **Raft Consensus Protocol** (or Multi-Raft) to achieve replicated state machine consistency across distributed cluster nodes.
9
+
10
+ ---
11
+
12
+ ## 1. Deep Dive into LSM-Tree Storage Engines
13
+
14
+ ### 1.1 In-Memory MemTable and Write-Ahead Log (WAL)
15
+ When a write operation `Put(Key, Value)` arrives at a database node:
16
+ - The mutation is first appended sequentially to an on-disk **Write-Ahead Log (WAL)** using `fsync()` to guarantee durability across hardware crashes.
17
+ - Simultaneously, the key-value pair is inserted into an in-memory skip list or red-black tree called the **MemTable**.
18
+
19
+ $$\text{Write Latency} = T_{\text{WAL\_Append}} + T_{\text{MemTable\_Insert}}$$
20
+
21
+ Because both operations avoid disk seek overhead, write throughput often exceeds $100,000 \text{ ops/sec}$ per storage node.
22
+
23
+ ### 1.2 SSTable Flushing and Multi-Tiered Compaction
24
+ When the MemTable reaches its capacity threshold (typically $64\text{ MB}$ to $256\text{ MB}$), it transitions into an **Immutable MemTable** and is flushed to disk as a **Sorted String Table (SSTable)** at Level 0 ($L_0$).
25
+
26
+ ```text
27
+ +-----------------------------------------------------------------------+
28
+ | WRITE PATH |
29
+ | |
30
+ | Client Put(K,V) ---> [ WAL File (Disk) ] (Sequential Append) |
31
+ | | |
32
+ | v |
33
+ | [ MemTable (RAM) ] (SkipList / Red-Black Tree) |
34
+ | | |
35
+ | v (Flush when MemTable >= 64MB) |
36
+ | [ SSTable L0 (Disk) ] |
37
+ +-----------------------------------------------------------------------+
38
+ ```
39
+
40
+ To limit read amplification, the storage engine runs background **Size-Tiered Compaction** or **Leveled Compaction**:
41
+
42
+ $$\text{Compaction Ratio} = \frac{\sum \text{SizeBytes}(L_{i+1})}{\sum \text{SizeBytes}(L_i)} \approx 10$$
43
+
44
+ ### 1.3 Bloom Filter Optimization for Read Path Acceleration
45
+ To verify if a key exists within an SSTable without issuing expensive disk reads, every SSTable maintains an in-memory **Bloom Filter**:
46
+
47
+ $$m = -\frac{n \cdot \ln(p)}{(\ln 2)^2}$$
48
+
49
+ Where:
50
+ - $m$ is the bit array size.
51
+ - $n$ is the number of inserted keys.
52
+ - $p$ is the target false positive probability (e.g., $0.01$).
53
+ - $k = \frac{m}{n} \ln 2$ is the optimal number of hash functions.
54
+
55
+ ---
56
+
57
+ ## 2. Distributed Consensus: The Raft Protocol
58
+
59
+ ### 2.1 Node States and Term Epochs
60
+ A Raft cluster consists of $2F + 1$ nodes to tolerate up to $F$ concurrent node failures. Each node transitions between three mutually exclusive states:
61
+ - **Leader**: Handles all client read/write requests and manages log replication.
62
+ - **Follower**: Passive state; responds to incoming RPCs from Leaders and Candidates.
63
+ - **Candidate**: Active state during leader elections.
64
+
65
+ ```text
66
+ +------------------------+
67
+ | Times out, starts |
68
+ | Leader Election |
69
+ v |
70
+ +--------------+ +---------------+
71
+ | Follower | -----> | Candidate |
72
+ +--------------+ +---------------+
73
+ ^ |
74
+ | Receives AppendEntries | Wins Election
75
+ | from valid Leader | with Majority
76
+ | v
77
+ +----------------+--------------+
78
+ | Leader |
79
+ +--------------+
80
+ ```
81
+
82
+ ### 2.2 Replicated Log Matching Property
83
+ If two logs on different nodes contain an entry with the same index and term:
84
+ 1. They store identical commands.
85
+ 2. Their logs are identical in all preceding entries up to that index.
86
+
87
+ $$\forall i, j \quad (\text{Log}_A[i].\text{term} == \text{Log}_B[i].\text{term}) \implies \forall k \le i, \, \text{Log}_A[k] == \text{Log}_B[k]$$
88
+
89
+ ### 2.3 Quorum Commit Rule
90
+ A log entry is considered **Committed** once it is safely replicated on a majority quorum of nodes:
91
+
92
+ $$\text{QuorumSize} = \left\lfloor \frac{N}{2} \right\rfloor + 1$$
93
+
94
+ ---
95
+
96
+ ## 3. Benchmarking Summary & Architectural Comparison
97
+
98
+ | Metric / Parameter | B+ Tree Storage (InnoDB) | LSM-Tree Storage (RocksDB) | Raft Replicated Log |
99
+ |---|---|---|---|
100
+ | **Write Pattern** | In-place random disk writes | Append-only sequential writes | Sequential log append |
101
+ | **Write Amplification** | Moderate ($2\times - 5\times$) | High ($10\times - 30\times$) | Low ($1\times - 2\times$) |
102
+ | **Read Amplification** | Low ($1\times - 2\times$) | High ($3\times - 10\times$) | N/A (Leader reads) |
103
+ | **Space Amplification** | Low ($1.1\times - 1.3\times$) | Moderate ($1.2\times - 2.0\times$) | $N \times$ (Replica count) |
104
+ | **Failure Recovery** | Crash recovery via REDO log | Flush MemTable via WAL | State catch-up via Snapshot |
105
+
106
+ ---
107
+
108
+ ## 4. Production Code Reference: Raft Log Truncation & Compaction
109
+
110
+ ```python
111
+ class RaftReplicatedLog:
112
+ def __init__(self, node_id: str, storage_path: str):
113
+ self.node_id = node_id
114
+ self.storage_path = storage_path
115
+ self.entries = []
116
+ self.commit_index = 0
117
+ self.last_applied = 0
118
+
119
+ def append_entry(self, term: int, command: dict) -> int:
120
+ index = len(self.entries) + 1
121
+ entry = {"index": index, "term": term, "command": command}
122
+ self.entries.append(entry)
123
+ return index
124
+
125
+ def truncate_uncommitted(self, from_index: int) -> None:
126
+ """Removes conflicting uncommitted entries following a failed leader term."""
127
+ if from_index <= self.commit_index:
128
+ raise ValueError("Cannot truncate committed log entries!")
129
+ self.entries = self.entries[:from_index - 1]
130
+ ```
@@ -0,0 +1,109 @@
1
+ {
2
+ "dataset_metadata": {
3
+ "benchmark_name": "MedQA-Clinical-EHR-Diagnostic-Eval-v4.2",
4
+ "domain": "Clinical Medicine & Electronic Health Record Diagnostics",
5
+ "release_date": "2026-06-15",
6
+ "compliance": ["HIPAA-Sec-164.312", "FDA-SaMD-Class-II", "EU-MDR-2017/745"],
7
+ "total_clinical_cases": 1250,
8
+ "evaluated_models": [
9
+ "Med-PaLM-2-70B",
10
+ "BioGPT-Large",
11
+ "Clinical-LLaMA-3-8B",
12
+ "Atlas-Medical-RAG-v1"
13
+ ]
14
+ },
15
+ "evaluation_metrics_summary": {
16
+ "diagnostic_accuracy_top1": 0.884,
17
+ "diagnostic_accuracy_top3": 0.962,
18
+ "contraindication_detection_recall": 0.991,
19
+ "dosage_calculation_precision": 0.978,
20
+ "mean_retrieval_latency_ms": 142.6,
21
+ "p99_retrieval_latency_ms": 389.2,
22
+ "hipaa_audit_pass_rate": 1.0
23
+ },
24
+ "diagnostic_subcategories": [
25
+ {
26
+ "category_id": "CARDIO-001",
27
+ "category_name": "Cardiology & Electrocardiography Diagnostics",
28
+ "sample_count": 320,
29
+ "key_pathologies": [
30
+ "ST-Elevation Myocardial Infarction (STEMI)",
31
+ "Atrial Fibrillation with Rapid Ventricular Response",
32
+ "Hypertrophic Obstructive Cardiomyopathy",
33
+ "Decompensated Congestive Heart Failure"
34
+ ],
35
+ "performance_breakdown": {
36
+ "precision": 0.912,
37
+ "recall": 0.895,
38
+ "f1_score": 0.903,
39
+ "hallucination_rate": 0.008
40
+ },
41
+ "ehr_structured_fields": {
42
+ "required_vitals": ["systolic_bp", "diastolic_bp", "heart_rate", "spo2", "troponin_i"],
43
+ "icd_10_codes": ["I21.0", "I48.0", "I42.1", "I50.9"]
44
+ }
45
+ },
46
+ {
47
+ "category_id": "NEURO-002",
48
+ "category_name": "Neurology & Acute Stroke Protocols",
49
+ "sample_count": 280,
50
+ "key_pathologies": [
51
+ "Acute Ischemic Stroke (Middle Cerebral Artery Territory)",
52
+ "Subarachnoid Hemorrhage",
53
+ "Refractory Status Epilepticus",
54
+ "Guillain-Barré Syndrome"
55
+ ],
56
+ "performance_breakdown": {
57
+ "precision": 0.879,
58
+ "recall": 0.921,
59
+ "f1_score": 0.899,
60
+ "hallucination_rate": 0.012
61
+ },
62
+ "ehr_structured_fields": {
63
+ "required_vitals": ["nihss_score", "gcs_score", "mean_arterial_pressure"],
64
+ "icd_10_codes": ["I63.5", "I60.9", "G41.9", "G61.0"]
65
+ }
66
+ },
67
+ {
68
+ "category_id": "ONCO-003",
69
+ "category_name": "Oncology & Biomarker Therapeutics",
70
+ "sample_count": 450,
71
+ "key_pathologies": [
72
+ "Metastatic Non-Small Cell Lung Cancer (EGFR T790M+)",
73
+ "HER2-Positive Invasive Breast Carcinoma",
74
+ "Triple-Negative Breast Cancer",
75
+ "Multiple Myeloma"
76
+ ],
77
+ "performance_breakdown": {
78
+ "precision": 0.941,
79
+ "recall": 0.935,
80
+ "f1_score": 0.938,
81
+ "hallucination_rate": 0.003
82
+ },
83
+ "ehr_structured_fields": {
84
+ "required_vitals": ["pd_l1_expression_percent", "alk_rearrangement", "brca1_mutation"],
85
+ "icd_10_codes": ["C34.90", "C50.911", "C90.00"]
86
+ }
87
+ }
88
+ ],
89
+ "retrieval_augmented_generation_telemetry": {
90
+ "vector_database_config": {
91
+ "index_type": "HNSW",
92
+ "distance_metric": "Cosine",
93
+ "m_parameter": 32,
94
+ "ef_construction": 200,
95
+ "embedding_model": "BAAI/bge-m3",
96
+ "dimensions": 1024
97
+ },
98
+ "lexical_bm25_config": {
99
+ "k1": 1.4,
100
+ "b": 0.75,
101
+ "tokenizer": "UnicodeMedicalSubwordTokenizer"
102
+ },
103
+ "rrf_fusion_parameters": {
104
+ "rank_constant_k": 60,
105
+ "weight_dense": 0.65,
106
+ "weight_sparse": 0.35
107
+ }
108
+ }
109
+ }