lumenrag 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lumenrag-0.1.2/LICENSE +21 -0
- lumenrag-0.1.2/PKG-INFO +22 -0
- lumenrag-0.1.2/lumenrag/__init__.py +3 -0
- lumenrag-0.1.2/lumenrag/__main__.py +3 -0
- lumenrag-0.1.2/lumenrag/bundled.py +80 -0
- lumenrag-0.1.2/lumenrag/cli.py +80 -0
- lumenrag-0.1.2/lumenrag/launcher.py +325 -0
- lumenrag-0.1.2/lumenrag.egg-info/PKG-INFO +22 -0
- lumenrag-0.1.2/lumenrag.egg-info/SOURCES.txt +45 -0
- lumenrag-0.1.2/lumenrag.egg-info/dependency_links.txt +1 -0
- lumenrag-0.1.2/lumenrag.egg-info/entry_points.txt +2 -0
- lumenrag-0.1.2/lumenrag.egg-info/requires.txt +7 -0
- lumenrag-0.1.2/lumenrag.egg-info/top_level.txt +2 -0
- lumenrag-0.1.2/pyproject.toml +62 -0
- lumenrag-0.1.2/rag_lumenvec/__init__.py +1 -0
- lumenrag-0.1.2/rag_lumenvec/api/__init__.py +2 -0
- lumenrag-0.1.2/rag_lumenvec/api/app.py +96 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/__init__.py +2 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/ai.py +42 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/chat.py +68 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/documents.py +155 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/errors.py +27 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/evaluation.py +103 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/health.py +46 -0
- lumenrag-0.1.2/rag_lumenvec/api/routes/status.py +99 -0
- lumenrag-0.1.2/rag_lumenvec/core/__init__.py +2 -0
- lumenrag-0.1.2/rag_lumenvec/core/config.py +34 -0
- lumenrag-0.1.2/rag_lumenvec/core/logging.py +17 -0
- lumenrag-0.1.2/rag_lumenvec/models.py +39 -0
- lumenrag-0.1.2/rag_lumenvec/repositories/__init__.py +2 -0
- lumenrag-0.1.2/rag_lumenvec/repositories/files.py +234 -0
- lumenrag-0.1.2/rag_lumenvec/schemas.py +50 -0
- lumenrag-0.1.2/rag_lumenvec/services/__init__.py +2 -0
- lumenrag-0.1.2/rag_lumenvec/services/ai.py +241 -0
- lumenrag-0.1.2/rag_lumenvec/services/chat.py +43 -0
- lumenrag-0.1.2/rag_lumenvec/services/evaluation.py +60 -0
- lumenrag-0.1.2/rag_lumenvec/services/ingestion.py +171 -0
- lumenrag-0.1.2/rag_lumenvec/services/lumenvec.py +175 -0
- lumenrag-0.1.2/rag_lumenvec/services/network.py +60 -0
- lumenrag-0.1.2/rag_lumenvec/services/retrieval.py +118 -0
- lumenrag-0.1.2/setup.cfg +4 -0
- lumenrag-0.1.2/setup.py +76 -0
- lumenrag-0.1.2/tests/test_api.py +138 -0
- lumenrag-0.1.2/tests/test_bundled.py +57 -0
- lumenrag-0.1.2/tests/test_launcher.py +220 -0
- lumenrag-0.1.2/tests/test_network.py +56 -0
- lumenrag-0.1.2/tests/test_rag_service.py +84 -0
lumenrag-0.1.2/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 brma-tech
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
lumenrag-0.1.2/PKG-INFO
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lumenrag
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Local-first RAG studio powered by LumenVec
|
|
5
|
+
Author: brma-tech
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/brma-tech/lumenrag
|
|
8
|
+
Project-URL: Repository, https://github.com/brma-tech/lumenrag
|
|
9
|
+
Project-URL: Issues, https://github.com/brma-tech/lumenrag/issues
|
|
10
|
+
Requires-Python: >=3.11
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: fastapi>=0.115.0
|
|
14
|
+
Requires-Dist: uvicorn[standard]>=0.30.0
|
|
15
|
+
Requires-Dist: openai<3,>=1.99.0
|
|
16
|
+
Requires-Dist: pypdf>=5.0.0
|
|
17
|
+
Requires-Dist: python-docx>=1.1.0
|
|
18
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
19
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
LumenRAG is a local-first RAG studio powered by LumenVec.
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
import stat
|
|
7
|
+
from functools import lru_cache
|
|
8
|
+
from importlib import resources
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BundledAssetError(RuntimeError):
|
|
14
|
+
"""Raised when packaged product assets fail integrity validation."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _assets_root() -> Path | None:
|
|
18
|
+
candidate = resources.files("lumenrag").joinpath("assets")
|
|
19
|
+
try:
|
|
20
|
+
path = Path(os.fspath(candidate))
|
|
21
|
+
except TypeError:
|
|
22
|
+
return None
|
|
23
|
+
return path if path.is_dir() else None
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _sha256(path: Path) -> str:
|
|
27
|
+
digest = hashlib.sha256()
|
|
28
|
+
with path.open("rb") as stream:
|
|
29
|
+
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
|
30
|
+
digest.update(chunk)
|
|
31
|
+
return digest.hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@lru_cache(maxsize=1)
|
|
35
|
+
def verified_manifest() -> tuple[Path, dict[str, Any]] | None:
|
|
36
|
+
assets = _assets_root()
|
|
37
|
+
if assets is None:
|
|
38
|
+
return None
|
|
39
|
+
manifest_path = assets / "manifest.json"
|
|
40
|
+
if not manifest_path.is_file():
|
|
41
|
+
raise BundledAssetError("Packaged asset manifest is missing.")
|
|
42
|
+
try:
|
|
43
|
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
44
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
45
|
+
raise BundledAssetError("Packaged asset manifest is invalid.") from exc
|
|
46
|
+
if manifest.get("schema_version") != 1:
|
|
47
|
+
raise BundledAssetError("Unsupported packaged asset manifest.")
|
|
48
|
+
files = manifest.get("files")
|
|
49
|
+
if not isinstance(files, dict) or not files:
|
|
50
|
+
raise BundledAssetError("Packaged asset manifest contains no files.")
|
|
51
|
+
for relative, expected in files.items():
|
|
52
|
+
if not isinstance(relative, str) or not isinstance(expected, str):
|
|
53
|
+
raise BundledAssetError("Packaged asset manifest has invalid entries.")
|
|
54
|
+
path = (assets / relative).resolve()
|
|
55
|
+
if assets.resolve() not in path.parents or not path.is_file():
|
|
56
|
+
raise BundledAssetError(f"Packaged asset is missing: {relative}")
|
|
57
|
+
if _sha256(path) != expected:
|
|
58
|
+
raise BundledAssetError(f"Packaged asset integrity failed: {relative}")
|
|
59
|
+
return assets, manifest
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def bundled_binary() -> Path | None:
|
|
63
|
+
verified = verified_manifest()
|
|
64
|
+
if verified is None:
|
|
65
|
+
return None
|
|
66
|
+
assets, manifest = verified
|
|
67
|
+
binary = assets / str(manifest["binary"])
|
|
68
|
+
if os.name != "nt":
|
|
69
|
+
binary.chmod(
|
|
70
|
+
binary.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
|
|
71
|
+
)
|
|
72
|
+
return binary
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def bundled_studio() -> Path | None:
|
|
76
|
+
verified = verified_manifest()
|
|
77
|
+
if verified is None:
|
|
78
|
+
return None
|
|
79
|
+
assets, manifest = verified
|
|
80
|
+
return assets / Path(str(manifest["studio"])).parent
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from lumenrag import __version__
|
|
9
|
+
from lumenrag.launcher import LauncherError, StartOptions, default_workspace, start
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
13
|
+
parser = argparse.ArgumentParser(
|
|
14
|
+
prog="lumenrag",
|
|
15
|
+
description="Local-first RAG studio powered by LumenVec.",
|
|
16
|
+
)
|
|
17
|
+
parser.add_argument("--version", action="version", version=__version__)
|
|
18
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
19
|
+
|
|
20
|
+
start_parser = commands.add_parser("start", help="Start the local RAG studio")
|
|
21
|
+
start_parser.add_argument("--host", default="127.0.0.1", help="Studio host")
|
|
22
|
+
start_parser.add_argument("--port", type=int, default=8000, help="Studio port")
|
|
23
|
+
start_parser.add_argument(
|
|
24
|
+
"--workspace",
|
|
25
|
+
type=Path,
|
|
26
|
+
default=default_workspace(),
|
|
27
|
+
help="Directory used for local data and logs",
|
|
28
|
+
)
|
|
29
|
+
start_parser.add_argument(
|
|
30
|
+
"--lumenvec-url",
|
|
31
|
+
default=os.getenv("LUMENVEC_BASE_URL", "http://127.0.0.1:19190"),
|
|
32
|
+
help="Existing or locally managed LumenVec HTTP endpoint",
|
|
33
|
+
)
|
|
34
|
+
start_parser.add_argument(
|
|
35
|
+
"--lumenvec-binary",
|
|
36
|
+
type=Path,
|
|
37
|
+
help="Path to a LumenVec executable",
|
|
38
|
+
)
|
|
39
|
+
start_parser.add_argument(
|
|
40
|
+
"--external-lumenvec",
|
|
41
|
+
action="store_true",
|
|
42
|
+
help="Require an already running LumenVec instead of starting one",
|
|
43
|
+
)
|
|
44
|
+
start_parser.add_argument(
|
|
45
|
+
"--no-browser",
|
|
46
|
+
action="store_true",
|
|
47
|
+
help="Do not open the Studio in the default browser",
|
|
48
|
+
)
|
|
49
|
+
start_parser.add_argument(
|
|
50
|
+
"--startup-timeout",
|
|
51
|
+
type=float,
|
|
52
|
+
default=20.0,
|
|
53
|
+
help="Seconds to wait for LumenVec startup",
|
|
54
|
+
)
|
|
55
|
+
return parser
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def main(argv: list[str] | None = None) -> int:
|
|
59
|
+
args = build_parser().parse_args(argv)
|
|
60
|
+
if args.command != "start": # pragma: no cover - argparse enforces this
|
|
61
|
+
return 2
|
|
62
|
+
|
|
63
|
+
options = StartOptions(
|
|
64
|
+
host=args.host,
|
|
65
|
+
port=args.port,
|
|
66
|
+
workspace=args.workspace.expanduser().resolve(),
|
|
67
|
+
lumenvec_url=args.lumenvec_url,
|
|
68
|
+
lumenvec_binary=args.lumenvec_binary,
|
|
69
|
+
external_lumenvec=args.external_lumenvec,
|
|
70
|
+
open_browser=not args.no_browser,
|
|
71
|
+
startup_timeout=args.startup_timeout,
|
|
72
|
+
)
|
|
73
|
+
try:
|
|
74
|
+
start(options)
|
|
75
|
+
except LauncherError as exc:
|
|
76
|
+
print(f"LumenRAG could not start: {exc}", file=sys.stderr)
|
|
77
|
+
return 1
|
|
78
|
+
except KeyboardInterrupt:
|
|
79
|
+
return 130
|
|
80
|
+
return 0
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import shutil
|
|
5
|
+
import signal
|
|
6
|
+
import subprocess
|
|
7
|
+
import sys
|
|
8
|
+
import threading
|
|
9
|
+
import time
|
|
10
|
+
import urllib.error
|
|
11
|
+
import urllib.request
|
|
12
|
+
import webbrowser
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from contextlib import nullcontext
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Protocol
|
|
18
|
+
from urllib.parse import urlparse
|
|
19
|
+
|
|
20
|
+
from lumenrag.bundled import BundledAssetError, bundled_binary, bundled_studio
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class LauncherError(RuntimeError):
|
|
24
|
+
"""Raised when the local product stack cannot be started safely."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class Process(Protocol):
|
|
28
|
+
def poll(self) -> int | None: ...
|
|
29
|
+
|
|
30
|
+
def wait(self, timeout: float | None = None) -> int: ...
|
|
31
|
+
|
|
32
|
+
def terminate(self) -> None: ...
|
|
33
|
+
|
|
34
|
+
def kill(self) -> None: ...
|
|
35
|
+
|
|
36
|
+
def send_signal(self, sig: int) -> None: ...
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class StartOptions:
|
|
41
|
+
host: str = "127.0.0.1"
|
|
42
|
+
port: int = 8000
|
|
43
|
+
workspace: Path = Path(".lumenrag")
|
|
44
|
+
lumenvec_url: str = "http://127.0.0.1:19190"
|
|
45
|
+
lumenvec_binary: Path | None = None
|
|
46
|
+
external_lumenvec: bool = False
|
|
47
|
+
open_browser: bool = True
|
|
48
|
+
startup_timeout: float = 20.0
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class ManagedProcess:
|
|
53
|
+
process: Process
|
|
54
|
+
use_process_group: bool = False
|
|
55
|
+
|
|
56
|
+
def stop(self, timeout: float = 5.0) -> None:
|
|
57
|
+
if self.process.poll() is not None:
|
|
58
|
+
return
|
|
59
|
+
try:
|
|
60
|
+
if self.use_process_group and os.name == "nt":
|
|
61
|
+
self.process.send_signal(signal.CTRL_BREAK_EVENT)
|
|
62
|
+
else:
|
|
63
|
+
self.process.terminate()
|
|
64
|
+
self.process.wait(timeout=timeout)
|
|
65
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
66
|
+
self.process.kill()
|
|
67
|
+
self.process.wait(timeout=timeout)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def default_workspace() -> Path:
|
|
71
|
+
override = os.getenv("LUMENRAG_HOME")
|
|
72
|
+
if override:
|
|
73
|
+
return Path(override).expanduser()
|
|
74
|
+
if os.name == "nt":
|
|
75
|
+
base = Path(os.getenv("LOCALAPPDATA", Path.home() / "AppData" / "Local"))
|
|
76
|
+
return base / "LumenRAG"
|
|
77
|
+
base = Path(os.getenv("XDG_DATA_HOME", Path.home() / ".local" / "share"))
|
|
78
|
+
return base / "lumenrag"
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def is_healthy(base_url: str, timeout: float = 1.0) -> bool:
|
|
82
|
+
health_url = f"{base_url.rstrip('/')}/health"
|
|
83
|
+
try:
|
|
84
|
+
with urllib.request.urlopen(health_url, timeout=timeout) as response:
|
|
85
|
+
return 200 <= response.status < 300
|
|
86
|
+
except (OSError, urllib.error.URLError, ValueError):
|
|
87
|
+
return False
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def is_url_ready(url: str, timeout: float = 1.0) -> bool:
|
|
91
|
+
try:
|
|
92
|
+
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
93
|
+
return 200 <= response.status < 300
|
|
94
|
+
except (OSError, urllib.error.URLError, ValueError):
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def discover_lumenvec_binary(explicit: Path | None = None) -> Path:
|
|
99
|
+
configured = explicit or _configured_binary()
|
|
100
|
+
if configured:
|
|
101
|
+
candidate = configured.expanduser().resolve()
|
|
102
|
+
if candidate.is_file():
|
|
103
|
+
return candidate
|
|
104
|
+
raise LauncherError(f"LumenVec binary does not exist: {candidate}")
|
|
105
|
+
|
|
106
|
+
try:
|
|
107
|
+
packaged = bundled_binary()
|
|
108
|
+
except BundledAssetError as exc:
|
|
109
|
+
raise LauncherError(str(exc)) from exc
|
|
110
|
+
if packaged is not None:
|
|
111
|
+
return packaged.resolve()
|
|
112
|
+
|
|
113
|
+
names = ["lumenvec.exe", "lumenvec"] if os.name == "nt" else ["lumenvec"]
|
|
114
|
+
for name in names:
|
|
115
|
+
found = shutil.which(name)
|
|
116
|
+
if found:
|
|
117
|
+
return Path(found).resolve()
|
|
118
|
+
|
|
119
|
+
for candidate in _development_binary_candidates():
|
|
120
|
+
if candidate.is_file():
|
|
121
|
+
return candidate.resolve()
|
|
122
|
+
|
|
123
|
+
raise LauncherError(
|
|
124
|
+
"LumenVec was not found. Pass --lumenvec-binary, set LUMENVEC_BINARY, "
|
|
125
|
+
"or add lumenvec to PATH."
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _configured_binary() -> Path | None:
|
|
130
|
+
raw = os.getenv("LUMENVEC_BINARY", "").strip()
|
|
131
|
+
return Path(raw) if raw else None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _development_binary_candidates() -> list[Path]:
|
|
135
|
+
project_root = Path(__file__).resolve().parents[2]
|
|
136
|
+
database_root = project_root.parents[1] if len(project_root.parents) > 1 else None
|
|
137
|
+
if database_root is None:
|
|
138
|
+
return []
|
|
139
|
+
lumenvec_root = database_root / "lumenvec"
|
|
140
|
+
platform_pattern = "*windows*http*" if os.name == "nt" else f"*{sys.platform}*"
|
|
141
|
+
executable = "lumenvec.exe" if os.name == "nt" else "lumenvec"
|
|
142
|
+
release_candidates = sorted(
|
|
143
|
+
lumenvec_root.glob(f"dist/release/{platform_pattern}/{executable}"),
|
|
144
|
+
reverse=True,
|
|
145
|
+
)
|
|
146
|
+
direct_candidates = sorted(
|
|
147
|
+
lumenvec_root.glob(f"dist/lumenvec-*-{sys.platform}-*"), reverse=True
|
|
148
|
+
)
|
|
149
|
+
return release_candidates + direct_candidates + [lumenvec_root / executable]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _lumenvec_port(base_url: str) -> int:
|
|
153
|
+
parsed = urlparse(base_url)
|
|
154
|
+
local_hosts = {"127.0.0.1", "localhost", "::1"}
|
|
155
|
+
if parsed.scheme != "http" or parsed.hostname not in local_hosts:
|
|
156
|
+
raise LauncherError(
|
|
157
|
+
"A managed LumenVec endpoint must use HTTP on localhost. "
|
|
158
|
+
"Use --external-lumenvec for remote endpoints."
|
|
159
|
+
)
|
|
160
|
+
return parsed.port or 80
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def lumenvec_environment(workspace: Path, base_url: str) -> dict[str, str]:
|
|
164
|
+
data_dir = workspace / "lumenvec"
|
|
165
|
+
vector_dir = data_dir / "vectors"
|
|
166
|
+
vector_dir.mkdir(parents=True, exist_ok=True)
|
|
167
|
+
environment = os.environ.copy()
|
|
168
|
+
environment.update(
|
|
169
|
+
{
|
|
170
|
+
"VECTOR_DB_PROTOCOL": "http",
|
|
171
|
+
"VECTOR_DB_PORT": str(_lumenvec_port(base_url)),
|
|
172
|
+
"VECTOR_DB_SNAPSHOT_PATH": str(data_dir / "snapshot.json"),
|
|
173
|
+
"VECTOR_DB_WAL_PATH": str(data_dir / "wal.log"),
|
|
174
|
+
"VECTOR_DB_VECTOR_PATH": str(vector_dir),
|
|
175
|
+
"VECTOR_DB_SEARCH_MODE": environment.get("VECTOR_DB_SEARCH_MODE", "ann"),
|
|
176
|
+
}
|
|
177
|
+
)
|
|
178
|
+
return environment
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def launch_lumenvec(
|
|
182
|
+
binary: Path,
|
|
183
|
+
workspace: Path,
|
|
184
|
+
base_url: str,
|
|
185
|
+
*,
|
|
186
|
+
popen: Callable[..., Process] = subprocess.Popen,
|
|
187
|
+
) -> ManagedProcess:
|
|
188
|
+
log_dir = workspace / "logs"
|
|
189
|
+
process_dir = workspace / "lumenvec"
|
|
190
|
+
log_dir.mkdir(parents=True, exist_ok=True)
|
|
191
|
+
process_dir.mkdir(parents=True, exist_ok=True)
|
|
192
|
+
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0
|
|
193
|
+
with (log_dir / "lumenvec.stdout.log").open("ab") as stdout, (
|
|
194
|
+
log_dir / "lumenvec.stderr.log"
|
|
195
|
+
).open("ab") as stderr:
|
|
196
|
+
process = popen(
|
|
197
|
+
[str(binary)],
|
|
198
|
+
cwd=process_dir,
|
|
199
|
+
env=lumenvec_environment(workspace, base_url),
|
|
200
|
+
stdout=stdout,
|
|
201
|
+
stderr=stderr,
|
|
202
|
+
creationflags=creationflags,
|
|
203
|
+
)
|
|
204
|
+
return ManagedProcess(process, use_process_group=os.name == "nt")
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def wait_until_healthy(
|
|
208
|
+
base_url: str,
|
|
209
|
+
process: Process,
|
|
210
|
+
timeout: float,
|
|
211
|
+
*,
|
|
212
|
+
health_check: Callable[[str, float], bool] = is_healthy,
|
|
213
|
+
) -> None:
|
|
214
|
+
deadline = time.monotonic() + timeout
|
|
215
|
+
while time.monotonic() < deadline:
|
|
216
|
+
exit_code = process.poll()
|
|
217
|
+
if exit_code is not None:
|
|
218
|
+
raise LauncherError(
|
|
219
|
+
f"LumenVec exited during startup with code {exit_code}."
|
|
220
|
+
)
|
|
221
|
+
if health_check(base_url, 1.0):
|
|
222
|
+
return
|
|
223
|
+
time.sleep(0.15)
|
|
224
|
+
raise LauncherError(f"LumenVec did not become healthy within {timeout:g} seconds.")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def ensure_lumenvec(
|
|
228
|
+
options: StartOptions,
|
|
229
|
+
*,
|
|
230
|
+
health_check: Callable[[str, float], bool] = is_healthy,
|
|
231
|
+
process_launcher: Callable[[Path, Path, str], ManagedProcess] = launch_lumenvec,
|
|
232
|
+
) -> ManagedProcess | None:
|
|
233
|
+
if health_check(options.lumenvec_url, 1.0):
|
|
234
|
+
print(f"[ok] Reusing LumenVec at {options.lumenvec_url}")
|
|
235
|
+
return None
|
|
236
|
+
if options.external_lumenvec:
|
|
237
|
+
raise LauncherError(f"LumenVec is not reachable at {options.lumenvec_url}.")
|
|
238
|
+
|
|
239
|
+
binary = discover_lumenvec_binary(options.lumenvec_binary)
|
|
240
|
+
managed = process_launcher(binary, options.workspace, options.lumenvec_url)
|
|
241
|
+
try:
|
|
242
|
+
wait_until_healthy(
|
|
243
|
+
options.lumenvec_url,
|
|
244
|
+
managed.process,
|
|
245
|
+
options.startup_timeout,
|
|
246
|
+
health_check=health_check,
|
|
247
|
+
)
|
|
248
|
+
except Exception:
|
|
249
|
+
managed.stop()
|
|
250
|
+
raise
|
|
251
|
+
print(f"[ok] LumenVec started at {options.lumenvec_url}")
|
|
252
|
+
return managed
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _open_browser_when_ready(url: str) -> None:
|
|
256
|
+
for _ in range(80):
|
|
257
|
+
if is_url_ready(url, 0.5):
|
|
258
|
+
webbrowser.open(url)
|
|
259
|
+
return
|
|
260
|
+
time.sleep(0.1)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _run_studio(options: StartOptions) -> None:
|
|
264
|
+
import uvicorn
|
|
265
|
+
|
|
266
|
+
server = uvicorn.Server(
|
|
267
|
+
uvicorn.Config(
|
|
268
|
+
"rag_lumenvec.api.app:app",
|
|
269
|
+
host=options.host,
|
|
270
|
+
port=options.port,
|
|
271
|
+
log_level="info",
|
|
272
|
+
)
|
|
273
|
+
)
|
|
274
|
+
server.capture_signals = nullcontext
|
|
275
|
+
|
|
276
|
+
watched_signals = [signal.SIGINT, signal.SIGTERM]
|
|
277
|
+
if os.name == "nt" and hasattr(signal, "SIGBREAK"):
|
|
278
|
+
watched_signals.append(signal.SIGBREAK)
|
|
279
|
+
previous_handlers: dict[int, object] = {}
|
|
280
|
+
|
|
281
|
+
def request_shutdown(_signum: int, _frame: object) -> None:
|
|
282
|
+
server.should_exit = True
|
|
283
|
+
|
|
284
|
+
try:
|
|
285
|
+
for watched_signal in watched_signals:
|
|
286
|
+
previous_handlers[watched_signal] = signal.getsignal(watched_signal)
|
|
287
|
+
signal.signal(watched_signal, request_shutdown)
|
|
288
|
+
server.run()
|
|
289
|
+
finally:
|
|
290
|
+
for watched_signal, previous_handler in previous_handlers.items():
|
|
291
|
+
signal.signal(watched_signal, previous_handler)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def start(options: StartOptions) -> None:
|
|
295
|
+
if not 1 <= options.port <= 65535:
|
|
296
|
+
raise LauncherError("Studio port must be between 1 and 65535.")
|
|
297
|
+
options.workspace.mkdir(parents=True, exist_ok=True)
|
|
298
|
+
os.environ["LUMENVEC_BASE_URL"] = options.lumenvec_url
|
|
299
|
+
os.environ["RAG_DATA_DIR"] = str(options.workspace / "rag_data")
|
|
300
|
+
if not os.getenv("LUMENRAG_FRONTEND_DIST"):
|
|
301
|
+
try:
|
|
302
|
+
studio = bundled_studio()
|
|
303
|
+
except BundledAssetError as exc:
|
|
304
|
+
raise LauncherError(str(exc)) from exc
|
|
305
|
+
if studio is not None:
|
|
306
|
+
os.environ["LUMENRAG_FRONTEND_DIST"] = str(studio)
|
|
307
|
+
|
|
308
|
+
managed = ensure_lumenvec(options)
|
|
309
|
+
studio_url = f"http://{options.host}:{options.port}"
|
|
310
|
+
try:
|
|
311
|
+
if options.open_browser:
|
|
312
|
+
threading.Thread(
|
|
313
|
+
target=_open_browser_when_ready,
|
|
314
|
+
args=(studio_url,),
|
|
315
|
+
daemon=True,
|
|
316
|
+
).start()
|
|
317
|
+
print(f"[ok] Workspace: {options.workspace}")
|
|
318
|
+
print(f"[ok] LumenRAG Studio: {studio_url}")
|
|
319
|
+
print("Press Ctrl+C to stop.")
|
|
320
|
+
|
|
321
|
+
_run_studio(options)
|
|
322
|
+
finally:
|
|
323
|
+
if managed is not None:
|
|
324
|
+
print("Stopping managed LumenVec...")
|
|
325
|
+
managed.stop()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: lumenrag
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: Local-first RAG studio powered by LumenVec
|
|
5
|
+
Author: brma-tech
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/brma-tech/lumenrag
|
|
8
|
+
Project-URL: Repository, https://github.com/brma-tech/lumenrag
|
|
9
|
+
Project-URL: Issues, https://github.com/brma-tech/lumenrag/issues
|
|
10
|
+
Requires-Python: >=3.11
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Requires-Dist: fastapi>=0.115.0
|
|
14
|
+
Requires-Dist: uvicorn[standard]>=0.30.0
|
|
15
|
+
Requires-Dist: openai<3,>=1.99.0
|
|
16
|
+
Requires-Dist: pypdf>=5.0.0
|
|
17
|
+
Requires-Dist: python-docx>=1.1.0
|
|
18
|
+
Requires-Dist: python-dotenv>=1.0.1
|
|
19
|
+
Requires-Dist: python-multipart>=0.0.9
|
|
20
|
+
Dynamic: license-file
|
|
21
|
+
|
|
22
|
+
LumenRAG is a local-first RAG studio powered by LumenVec.
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
pyproject.toml
|
|
3
|
+
setup.py
|
|
4
|
+
lumenrag/__init__.py
|
|
5
|
+
lumenrag/__main__.py
|
|
6
|
+
lumenrag/bundled.py
|
|
7
|
+
lumenrag/cli.py
|
|
8
|
+
lumenrag/launcher.py
|
|
9
|
+
lumenrag.egg-info/PKG-INFO
|
|
10
|
+
lumenrag.egg-info/SOURCES.txt
|
|
11
|
+
lumenrag.egg-info/dependency_links.txt
|
|
12
|
+
lumenrag.egg-info/entry_points.txt
|
|
13
|
+
lumenrag.egg-info/requires.txt
|
|
14
|
+
lumenrag.egg-info/top_level.txt
|
|
15
|
+
rag_lumenvec/__init__.py
|
|
16
|
+
rag_lumenvec/models.py
|
|
17
|
+
rag_lumenvec/schemas.py
|
|
18
|
+
rag_lumenvec/api/__init__.py
|
|
19
|
+
rag_lumenvec/api/app.py
|
|
20
|
+
rag_lumenvec/api/routes/__init__.py
|
|
21
|
+
rag_lumenvec/api/routes/ai.py
|
|
22
|
+
rag_lumenvec/api/routes/chat.py
|
|
23
|
+
rag_lumenvec/api/routes/documents.py
|
|
24
|
+
rag_lumenvec/api/routes/errors.py
|
|
25
|
+
rag_lumenvec/api/routes/evaluation.py
|
|
26
|
+
rag_lumenvec/api/routes/health.py
|
|
27
|
+
rag_lumenvec/api/routes/status.py
|
|
28
|
+
rag_lumenvec/core/__init__.py
|
|
29
|
+
rag_lumenvec/core/config.py
|
|
30
|
+
rag_lumenvec/core/logging.py
|
|
31
|
+
rag_lumenvec/repositories/__init__.py
|
|
32
|
+
rag_lumenvec/repositories/files.py
|
|
33
|
+
rag_lumenvec/services/__init__.py
|
|
34
|
+
rag_lumenvec/services/ai.py
|
|
35
|
+
rag_lumenvec/services/chat.py
|
|
36
|
+
rag_lumenvec/services/evaluation.py
|
|
37
|
+
rag_lumenvec/services/ingestion.py
|
|
38
|
+
rag_lumenvec/services/lumenvec.py
|
|
39
|
+
rag_lumenvec/services/network.py
|
|
40
|
+
rag_lumenvec/services/retrieval.py
|
|
41
|
+
tests/test_api.py
|
|
42
|
+
tests/test_bundled.py
|
|
43
|
+
tests/test_launcher.py
|
|
44
|
+
tests/test_network.py
|
|
45
|
+
tests/test_rag_service.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=75", "wheel>=0.45"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "lumenrag"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
description = "Local-first RAG studio powered by LumenVec"
|
|
9
|
+
license = "MIT"
|
|
10
|
+
license-files = ["LICENSE"]
|
|
11
|
+
authors = [{name = "brma-tech"}]
|
|
12
|
+
readme = {text = "LumenRAG is a local-first RAG studio powered by LumenVec.", content-type = "text/markdown"}
|
|
13
|
+
requires-python = ">=3.11"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"fastapi>=0.115.0",
|
|
16
|
+
"uvicorn[standard]>=0.30.0",
|
|
17
|
+
"openai>=1.99.0,<3",
|
|
18
|
+
"pypdf>=5.0.0",
|
|
19
|
+
"python-docx>=1.1.0",
|
|
20
|
+
"python-dotenv>=1.0.1",
|
|
21
|
+
"python-multipart>=0.0.9",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.scripts]
|
|
25
|
+
lumenrag = "lumenrag.cli:main"
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/brma-tech/lumenrag"
|
|
29
|
+
Repository = "https://github.com/brma-tech/lumenrag"
|
|
30
|
+
Issues = "https://github.com/brma-tech/lumenrag/issues"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages.find]
|
|
33
|
+
where = ["."]
|
|
34
|
+
include = ["lumenrag*", "rag_lumenvec*"]
|
|
35
|
+
|
|
36
|
+
[tool.setuptools.package-data]
|
|
37
|
+
lumenrag = ["assets/**/*"]
|
|
38
|
+
|
|
39
|
+
[tool.pytest.ini_options]
|
|
40
|
+
testpaths = ["tests"]
|
|
41
|
+
pythonpath = ["."]
|
|
42
|
+
addopts = "-q"
|
|
43
|
+
|
|
44
|
+
[tool.ruff]
|
|
45
|
+
line-length = 88
|
|
46
|
+
target-version = "py311"
|
|
47
|
+
|
|
48
|
+
[tool.ruff.lint]
|
|
49
|
+
select = [
|
|
50
|
+
"E",
|
|
51
|
+
"F",
|
|
52
|
+
"I",
|
|
53
|
+
"UP",
|
|
54
|
+
"B",
|
|
55
|
+
]
|
|
56
|
+
ignore = [
|
|
57
|
+
"B008",
|
|
58
|
+
]
|
|
59
|
+
|
|
60
|
+
[tool.ruff.format]
|
|
61
|
+
quote-style = "double"
|
|
62
|
+
indent-style = "space"
|