smartrag 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- smartrag/__about__.py +3 -0
- smartrag/__init__.py +10 -0
- smartrag/__main__.py +3 -0
- smartrag/api/__init__.py +5 -0
- smartrag/api/app.py +42 -0
- smartrag/api/auth.py +115 -0
- smartrag/api/errors.py +57 -0
- smartrag/api/ops.py +39 -0
- smartrag/api/query.py +154 -0
- smartrag/cli/__init__.py +0 -0
- smartrag/cli/main.py +78 -0
- smartrag/engine/__init__.py +7 -0
- smartrag/engine/base.py +78 -0
- smartrag/engine/demo.py +69 -0
- smartrag/engine/memory.py +424 -0
- smartrag/errors.py +70 -0
- smartrag/models.py +229 -0
- smartrag/paging.py +51 -0
- smartrag/settings.py +25 -0
- smartrag-0.0.1.dist-info/METADATA +86 -0
- smartrag-0.0.1.dist-info/RECORD +23 -0
- smartrag-0.0.1.dist-info/WHEEL +4 -0
- smartrag-0.0.1.dist-info/entry_points.txt +2 -0
smartrag/__about__.py
ADDED
smartrag/__init__.py
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
"""Smart RAG — a knowledge source any agent can use.
|
|
2
|
+
|
|
3
|
+
Search returns evidence (snippets, stable ids, citations); the agent, or a person at the
|
|
4
|
+
search box, writes the answer. See `smartrag.api` for the query-plane Web API and
|
|
5
|
+
`smartrag.engine` for the retrieval-engine seam behind it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from smartrag.__about__ import __version__
|
|
9
|
+
|
|
10
|
+
__all__ = ["__version__"]
|
smartrag/__main__.py
ADDED
smartrag/api/__init__.py
ADDED
smartrag/api/app.py
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""Build the query-plane FastAPI app around a `RetrievalEngine`."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from fastapi import FastAPI
|
|
6
|
+
|
|
7
|
+
from smartrag.__about__ import __version__
|
|
8
|
+
from smartrag.api import ops, query
|
|
9
|
+
from smartrag.api.auth import FileKeyStore, KeyStore
|
|
10
|
+
from smartrag.api.errors import install_error_handlers
|
|
11
|
+
from smartrag.engine.base import RetrievalEngine
|
|
12
|
+
from smartrag.settings import Settings
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def create_app(
|
|
16
|
+
engine: RetrievalEngine,
|
|
17
|
+
settings: Settings | None = None,
|
|
18
|
+
key_store: KeyStore | None = None,
|
|
19
|
+
) -> FastAPI:
|
|
20
|
+
"""The Smart RAG Web API (query plane only).
|
|
21
|
+
|
|
22
|
+
Auth: `key_store` if given, else the hashed key file `settings.api_keys_file`. With
|
|
23
|
+
neither, the app refuses to start unless `settings.auth_disabled` is set — an
|
|
24
|
+
unauthenticated server is never the silent default.
|
|
25
|
+
"""
|
|
26
|
+
settings = settings or Settings()
|
|
27
|
+
if key_store is None and not settings.auth_disabled:
|
|
28
|
+
if settings.api_keys_file is None:
|
|
29
|
+
raise ValueError("no API keys configured: set SMARTRAG_API_KEYS_FILE "
|
|
30
|
+
"(or auth_disabled for local development)")
|
|
31
|
+
key_store = FileKeyStore(settings.api_keys_file)
|
|
32
|
+
|
|
33
|
+
app = FastAPI(title="Smart RAG", version=__version__,
|
|
34
|
+
description="Search returns evidence with citations; the agent writes "
|
|
35
|
+
"the answer.")
|
|
36
|
+
app.state.engine = engine
|
|
37
|
+
app.state.settings = settings
|
|
38
|
+
app.state.key_store = None if settings.auth_disabled else key_store
|
|
39
|
+
install_error_handlers(app)
|
|
40
|
+
app.include_router(query.router)
|
|
41
|
+
app.include_router(ops.router)
|
|
42
|
+
return app
|
smartrag/api/auth.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""API-key authentication and the end-user identity slot.
|
|
2
|
+
|
|
3
|
+
Clients send `Authorization: Bearer <api key>`. Keys are stored hashed (SHA-256) in a
|
|
4
|
+
JSON file that the CLI writes, so the query server never holds a plaintext key:
|
|
5
|
+
|
|
6
|
+
[{"key_id": "ui-search", "sha256": "<hex>", "role": "reader", "delegate": true}]
|
|
7
|
+
|
|
8
|
+
The file is re-read when it changes, so a key issued or revoked by the CLI takes effect
|
|
9
|
+
without restarting the server.
|
|
10
|
+
|
|
11
|
+
`X-SmartRAG-On-Behalf-Of: <user id>` names the end user an agent or UI acts for
|
|
12
|
+
(CommonRAG P2). Only keys with `delegate: true` may send it; v1 logs it and passes it to
|
|
13
|
+
the engine, and document permissions will use it later.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import hashlib
|
|
19
|
+
import hmac
|
|
20
|
+
import json
|
|
21
|
+
import logging
|
|
22
|
+
import threading
|
|
23
|
+
from dataclasses import dataclass
|
|
24
|
+
from pathlib import Path
|
|
25
|
+
from typing import Literal, Protocol
|
|
26
|
+
|
|
27
|
+
from fastapi import Header, Request
|
|
28
|
+
|
|
29
|
+
from smartrag.engine.base import RequestContext
|
|
30
|
+
from smartrag.errors import Forbidden, Unauthorized
|
|
31
|
+
|
|
32
|
+
log = logging.getLogger("smartrag.api.auth")
|
|
33
|
+
|
|
34
|
+
ON_BEHALF_OF_HEADER = "X-SmartRAG-On-Behalf-Of"
|
|
35
|
+
|
|
36
|
+
Role = Literal["reader", "admin"]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass(frozen=True)
|
|
40
|
+
class Principal:
|
|
41
|
+
key_id: str
|
|
42
|
+
role: Role
|
|
43
|
+
delegate: bool = False
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
ANONYMOUS = Principal(key_id="anonymous", role="reader", delegate=True)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def hash_key(key: str) -> str:
|
|
50
|
+
return hashlib.sha256(key.encode()).hexdigest()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class KeyStore(Protocol):
|
|
54
|
+
def authenticate(self, key: str) -> Principal | None: ...
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class FileKeyStore:
|
|
58
|
+
"""Hashed keys from a JSON file, reloaded when the file's mtime changes."""
|
|
59
|
+
|
|
60
|
+
def __init__(self, path: Path) -> None:
|
|
61
|
+
self.path = Path(path)
|
|
62
|
+
self._lock = threading.Lock()
|
|
63
|
+
self._mtime: float | None = None
|
|
64
|
+
self._entries: list[tuple[str, Principal]] = []
|
|
65
|
+
self._reload()
|
|
66
|
+
|
|
67
|
+
def _reload(self) -> None:
|
|
68
|
+
mtime = self.path.stat().st_mtime
|
|
69
|
+
if mtime == self._mtime:
|
|
70
|
+
return
|
|
71
|
+
try:
|
|
72
|
+
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
73
|
+
entries = [(str(e["sha256"]).lower(),
|
|
74
|
+
Principal(key_id=str(e["key_id"]), role=e.get("role", "reader"),
|
|
75
|
+
delegate=bool(e.get("delegate", False))))
|
|
76
|
+
for e in raw]
|
|
77
|
+
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
78
|
+
raise ValueError(f"invalid API key file {self.path}: {exc}") from exc
|
|
79
|
+
self._entries, self._mtime = entries, mtime
|
|
80
|
+
log.info("loaded %d API keys from %s", len(entries), self.path)
|
|
81
|
+
|
|
82
|
+
def authenticate(self, key: str) -> Principal | None:
|
|
83
|
+
with self._lock:
|
|
84
|
+
try:
|
|
85
|
+
self._reload()
|
|
86
|
+
except (OSError, ValueError):
|
|
87
|
+
log.exception("cannot reload %s; keeping the previous keys", self.path)
|
|
88
|
+
digest = hash_key(key)
|
|
89
|
+
match = None
|
|
90
|
+
for stored, principal in self._entries: # compare all: no early exit on timing
|
|
91
|
+
if hmac.compare_digest(stored, digest):
|
|
92
|
+
match = principal
|
|
93
|
+
return match
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
async def request_context(
|
|
97
|
+
request: Request,
|
|
98
|
+
authorization: str | None = Header(default=None),
|
|
99
|
+
on_behalf_of: str | None = Header(default=None, alias=ON_BEHALF_OF_HEADER),
|
|
100
|
+
) -> RequestContext:
|
|
101
|
+
"""FastAPI dependency: authenticate the caller and build the engine's context."""
|
|
102
|
+
store: KeyStore | None = request.app.state.key_store
|
|
103
|
+
if store is None: # auth disabled (development)
|
|
104
|
+
principal = ANONYMOUS
|
|
105
|
+
else:
|
|
106
|
+
scheme, _, key = (authorization or "").partition(" ")
|
|
107
|
+
if scheme.lower() != "bearer" or not key.strip():
|
|
108
|
+
raise Unauthorized("an API key is required: Authorization: Bearer <key>")
|
|
109
|
+
found = store.authenticate(key.strip())
|
|
110
|
+
if found is None:
|
|
111
|
+
raise Unauthorized("the API key is not valid")
|
|
112
|
+
principal = found
|
|
113
|
+
if on_behalf_of is not None and not principal.delegate:
|
|
114
|
+
raise Forbidden(f"this API key may not act on behalf of users ({ON_BEHALF_OF_HEADER})")
|
|
115
|
+
return RequestContext(key_id=principal.key_id, on_behalf_of=on_behalf_of or None)
|
smartrag/api/errors.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""One error shape for every failure: `{"error": {"code", "message", "details"}}`.
|
|
2
|
+
|
|
3
|
+
Validation errors are `400 invalid_request` (not FastAPI's default 422), unknown paths
|
|
4
|
+
`404 not_found`, and unexpected exceptions `500 internal_error` with a generic message
|
|
5
|
+
(the traceback goes to the log, never to the client).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from fastapi import FastAPI, Request
|
|
14
|
+
from fastapi.exceptions import RequestValidationError
|
|
15
|
+
from fastapi.responses import JSONResponse
|
|
16
|
+
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
17
|
+
|
|
18
|
+
from smartrag.errors import EmbedderUnavailable, SmartRAGError, Unauthorized
|
|
19
|
+
|
|
20
|
+
log = logging.getLogger("smartrag.api")
|
|
21
|
+
|
|
22
|
+
_HTTP_CODES = {404: "not_found", 405: "method_not_allowed"}
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def error_response(status: int, code: str, message: str,
|
|
26
|
+
details: dict[str, Any] | None = None,
|
|
27
|
+
headers: dict[str, str] | None = None) -> JSONResponse:
|
|
28
|
+
body = {"error": {"code": code, "message": message, "details": details or {}}}
|
|
29
|
+
return JSONResponse(body, status_code=status, headers=headers)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def install_error_handlers(app: FastAPI) -> None:
|
|
33
|
+
@app.exception_handler(SmartRAGError)
|
|
34
|
+
async def _smartrag_error(_: Request, exc: SmartRAGError) -> JSONResponse:
|
|
35
|
+
headers = None
|
|
36
|
+
if isinstance(exc, Unauthorized):
|
|
37
|
+
headers = {"WWW-Authenticate": "Bearer"}
|
|
38
|
+
elif isinstance(exc, EmbedderUnavailable) and exc.retry_after:
|
|
39
|
+
headers = {"Retry-After": str(exc.retry_after)}
|
|
40
|
+
return error_response(exc.status, exc.code, exc.message, exc.details, headers)
|
|
41
|
+
|
|
42
|
+
@app.exception_handler(RequestValidationError)
|
|
43
|
+
async def _validation_error(_: Request, exc: RequestValidationError) -> JSONResponse:
|
|
44
|
+
errors = [{"loc": list(e.get("loc", ())), "msg": e.get("msg", ""),
|
|
45
|
+
"type": e.get("type", "")} for e in exc.errors()]
|
|
46
|
+
return error_response(400, "invalid_request", "the request is not valid",
|
|
47
|
+
{"errors": errors})
|
|
48
|
+
|
|
49
|
+
@app.exception_handler(StarletteHTTPException)
|
|
50
|
+
async def _http_error(_: Request, exc: StarletteHTTPException) -> JSONResponse:
|
|
51
|
+
code = _HTTP_CODES.get(exc.status_code, "http_error")
|
|
52
|
+
return error_response(exc.status_code, code, str(exc.detail))
|
|
53
|
+
|
|
54
|
+
@app.exception_handler(Exception)
|
|
55
|
+
async def _unexpected(request: Request, exc: Exception) -> JSONResponse:
|
|
56
|
+
log.exception("unhandled error on %s %s", request.method, request.url.path)
|
|
57
|
+
return error_response(500, "internal_error", "an internal error occurred")
|
smartrag/api/ops.py
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Ops endpoints (13), unauthenticated so probes and load balancers can call them.
|
|
2
|
+
|
|
3
|
+
GET /healthz liveness: the process answers
|
|
4
|
+
GET /readyz readiness: the index and the embedder can serve (503 otherwise)
|
|
5
|
+
GET /version package and API version
|
|
6
|
+
|
|
7
|
+
They reveal no data: `/readyz` reports check names and short details only.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from fastapi import APIRouter, Request
|
|
13
|
+
from fastapi.responses import JSONResponse
|
|
14
|
+
|
|
15
|
+
from smartrag.__about__ import __version__
|
|
16
|
+
from smartrag.models import ReadinessResponse, VersionResponse
|
|
17
|
+
|
|
18
|
+
router = APIRouter()
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@router.get("/healthz")
|
|
22
|
+
async def healthz() -> dict[str, str]:
|
|
23
|
+
"""liveness"""
|
|
24
|
+
return {"status": "ok"}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@router.get("/readyz", response_model=ReadinessResponse,
|
|
28
|
+
responses={503: {"model": ReadinessResponse}})
|
|
29
|
+
async def readyz(request: Request) -> JSONResponse:
|
|
30
|
+
"""readiness of the index and the embedder"""
|
|
31
|
+
readiness: ReadinessResponse = await request.app.state.engine.readiness()
|
|
32
|
+
return JSONResponse(readiness.model_dump(mode="json"),
|
|
33
|
+
status_code=200 if readiness.ready else 503)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@router.get("/version", response_model=VersionResponse)
|
|
37
|
+
async def version() -> VersionResponse:
|
|
38
|
+
"""package and API version"""
|
|
39
|
+
return VersionResponse(version=__version__)
|
smartrag/api/query.py
ADDED
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""Query-plane endpoints 1–6: the only operations agents, the UI and MCP get.
|
|
2
|
+
|
|
3
|
+
GET /v1/sources describe_sources
|
|
4
|
+
POST /v1/search search (canonical form)
|
|
5
|
+
GET /v1/search?q= search for the search box (bookmarkable URLs)
|
|
6
|
+
GET /v1/chunks/{chunk_id} read: chunk -> neighbours -> section -> document
|
|
7
|
+
GET /v1/documents/{doc_id} metadata, outline, text in pages
|
|
8
|
+
POST /v1/feedback click / relevant / irrelevant
|
|
9
|
+
|
|
10
|
+
Admin operations (sources, sync, jobs, index, query logs, API keys) are not here by
|
|
11
|
+
design: they run through the CLI on the operator host.
|
|
12
|
+
|
|
13
|
+
Every search is logged with its `query_id` (the server-side query log, CommonRAG P3),
|
|
14
|
+
so a wrong answer can be traced to retrieval or to the agent.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
import logging
|
|
21
|
+
import time
|
|
22
|
+
import uuid
|
|
23
|
+
from typing import Literal
|
|
24
|
+
|
|
25
|
+
from fastapi import APIRouter, Depends, Path, Query, Request
|
|
26
|
+
|
|
27
|
+
from smartrag.api.auth import request_context
|
|
28
|
+
from smartrag.engine.base import RequestContext, RetrievalEngine
|
|
29
|
+
from smartrag.models import (
|
|
30
|
+
DocumentResponse,
|
|
31
|
+
FeedbackRequest,
|
|
32
|
+
FeedbackResponse,
|
|
33
|
+
ReadResponse,
|
|
34
|
+
ReadScope,
|
|
35
|
+
SearchFilters,
|
|
36
|
+
SearchRequest,
|
|
37
|
+
SearchResponse,
|
|
38
|
+
SourcesResponse,
|
|
39
|
+
)
|
|
40
|
+
from smartrag.paging import encode_cursor, search_binding
|
|
41
|
+
|
|
42
|
+
log = logging.getLogger("smartrag.query")
|
|
43
|
+
|
|
44
|
+
router = APIRouter(prefix="/v1", dependencies=[Depends(request_context)])
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _engine(request: Request) -> RetrievalEngine:
|
|
48
|
+
return request.app.state.engine
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _document_link(request: Request, doc_id: str, page: int | None) -> str:
|
|
52
|
+
base = request.app.state.settings.public_base_url or str(request.base_url)
|
|
53
|
+
link = f"{base.rstrip('/')}/v1/documents/{doc_id}"
|
|
54
|
+
return f"{link}?page={page}" if page is not None else link
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@router.get("/sources", response_model=SourcesResponse, response_model_exclude_none=True)
|
|
58
|
+
async def describe_sources(
|
|
59
|
+
request: Request, ctx: RequestContext = Depends(request_context)
|
|
60
|
+
) -> SourcesResponse:
|
|
61
|
+
"""what is indexed, which fields can be filtered, and how fresh it is"""
|
|
62
|
+
return await _engine(request).describe_sources(ctx)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@router.post("/search", response_model=SearchResponse, response_model_exclude_none=True)
|
|
66
|
+
async def search(
|
|
67
|
+
body: SearchRequest, request: Request, ctx: RequestContext = Depends(request_context)
|
|
68
|
+
) -> SearchResponse:
|
|
69
|
+
"""search with a natural-language query; returns evidence with citations"""
|
|
70
|
+
return await _search(body, request, ctx)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@router.get("/search", response_model=SearchResponse, response_model_exclude_none=True)
|
|
74
|
+
async def search_get(
|
|
75
|
+
request: Request,
|
|
76
|
+
q: str = Query(min_length=1, max_length=2000),
|
|
77
|
+
source: list[str] | None = Query(default=None),
|
|
78
|
+
type: list[str] | None = Query(default=None), # the URL's own name
|
|
79
|
+
limit: int = Query(default=10, ge=1, le=50),
|
|
80
|
+
page: int = Query(default=1, ge=1, le=100),
|
|
81
|
+
ctx: RequestContext = Depends(request_context),
|
|
82
|
+
) -> SearchResponse:
|
|
83
|
+
"""search for the search box: one result per document, highlights on"""
|
|
84
|
+
filters = SearchFilters(source_id=source, file_type=type) if (source or type) else None
|
|
85
|
+
body = SearchRequest(query=q, filters=filters, limit=limit, group_by="document",
|
|
86
|
+
include=["highlights"])
|
|
87
|
+
if page > 1:
|
|
88
|
+
body.cursor = encode_cursor((page - 1) * limit, search_binding(body))
|
|
89
|
+
return await _search(body, request, ctx)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def _search(body: SearchRequest, request: Request, ctx: RequestContext) -> SearchResponse:
|
|
93
|
+
query_id = f"q_{uuid.uuid4().hex}"
|
|
94
|
+
started = time.perf_counter()
|
|
95
|
+
response = await _engine(request).search(body, ctx, query_id)
|
|
96
|
+
response.took_ms = round((time.perf_counter() - started) * 1000)
|
|
97
|
+
for result in response.results:
|
|
98
|
+
result.citation.link = _document_link(request, result.doc_id, result.citation.page)
|
|
99
|
+
log.info("search %s", json.dumps({
|
|
100
|
+
"query_id": query_id,
|
|
101
|
+
"key_id": ctx.key_id,
|
|
102
|
+
"on_behalf_of": ctx.on_behalf_of,
|
|
103
|
+
"query": body.query,
|
|
104
|
+
"filters": body.filters.model_dump(mode="json", exclude_none=True)
|
|
105
|
+
if body.filters else None,
|
|
106
|
+
"mode": body.mode,
|
|
107
|
+
"group_by": body.group_by,
|
|
108
|
+
"results": [r.chunk_id for r in response.results],
|
|
109
|
+
"took_ms": response.took_ms,
|
|
110
|
+
}, ensure_ascii=False))
|
|
111
|
+
return response
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@router.get("/chunks/{chunk_id}", response_model=ReadResponse, response_model_exclude_none=True)
|
|
115
|
+
async def read_chunk(
|
|
116
|
+
request: Request,
|
|
117
|
+
chunk_id: str = Path(max_length=200),
|
|
118
|
+
scope: ReadScope = "chunk",
|
|
119
|
+
window: int = Query(default=1, ge=0, le=5),
|
|
120
|
+
max_chars: int = Query(default=8000, ge=500, le=50000),
|
|
121
|
+
cursor: str | None = None,
|
|
122
|
+
ctx: RequestContext = Depends(request_context),
|
|
123
|
+
) -> ReadResponse:
|
|
124
|
+
"""read a chunk, widened to its neighbours, section or whole document"""
|
|
125
|
+
response = await _engine(request).read_chunk(chunk_id, scope, window, max_chars, cursor, ctx)
|
|
126
|
+
response.citation.link = _document_link(request, response.doc_id, response.citation.page)
|
|
127
|
+
return response
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@router.get("/documents/{doc_id}", response_model=DocumentResponse,
|
|
131
|
+
response_model_exclude_none=True)
|
|
132
|
+
async def get_document(
|
|
133
|
+
request: Request,
|
|
134
|
+
doc_id: str = Path(max_length=100),
|
|
135
|
+
include: list[Literal["text"]] = Query(default=[]),
|
|
136
|
+
section: str | None = Query(default=None, max_length=50),
|
|
137
|
+
max_chars: int = Query(default=8000, ge=500, le=50000),
|
|
138
|
+
cursor: str | None = None,
|
|
139
|
+
ctx: RequestContext = Depends(request_context),
|
|
140
|
+
) -> DocumentResponse:
|
|
141
|
+
"""document metadata and outline; text in pages with include=text"""
|
|
142
|
+
response = await _engine(request).get_document(
|
|
143
|
+
doc_id, "text" in include, section, max_chars, cursor, ctx)
|
|
144
|
+
response.link = _document_link(request, response.doc_id, None)
|
|
145
|
+
return response
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
@router.post("/feedback", status_code=202, response_model=FeedbackResponse)
|
|
149
|
+
async def feedback(
|
|
150
|
+
body: FeedbackRequest, request: Request, ctx: RequestContext = Depends(request_context)
|
|
151
|
+
) -> FeedbackResponse:
|
|
152
|
+
"""record whether a result was clicked, relevant or irrelevant"""
|
|
153
|
+
await _engine(request).record_feedback(body, ctx)
|
|
154
|
+
return FeedbackResponse()
|
smartrag/cli/__init__.py
ADDED
|
File without changes
|
smartrag/cli/main.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""The `smartrag` CLI.
|
|
2
|
+
|
|
3
|
+
Now: `serve` (the query-plane Web API) and `version`. Admin commands — `source`, `jobs`,
|
|
4
|
+
`docs`, `index`, `logs`, `apikey`, `eval` — will be added here: by design, admin runs
|
|
5
|
+
through the CLI on the operator host, never through the Web API.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import ipaddress
|
|
11
|
+
import logging
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
import typer
|
|
15
|
+
|
|
16
|
+
from smartrag.__about__ import __version__
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(help="Smart RAG: a knowledge source any agent can use.",
|
|
19
|
+
no_args_is_help=True, add_completion=False)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _is_loopback(host: str) -> bool:
|
|
23
|
+
if host == "localhost":
|
|
24
|
+
return True
|
|
25
|
+
try:
|
|
26
|
+
return ipaddress.ip_address(host).is_loopback
|
|
27
|
+
except ValueError:
|
|
28
|
+
return False
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@app.command()
|
|
32
|
+
def serve(
|
|
33
|
+
host: str = typer.Option(None, help="Bind address [default: $SMARTRAG_HOST or 127.0.0.1]"),
|
|
34
|
+
port: int = typer.Option(None, help="Port [default: $SMARTRAG_PORT or 8710]"),
|
|
35
|
+
api_keys_file: Path = typer.Option(None, help="Hashed API key file "
|
|
36
|
+
"[default: $SMARTRAG_API_KEYS_FILE]"),
|
|
37
|
+
demo: bool = typer.Option(False, help="Serve the built-in in-memory demo corpus."),
|
|
38
|
+
no_auth: bool = typer.Option(False, help="Disable API keys (loopback only, development)."),
|
|
39
|
+
log_level: str = typer.Option("info"),
|
|
40
|
+
) -> None:
|
|
41
|
+
"""Run the query-plane Web API."""
|
|
42
|
+
import uvicorn
|
|
43
|
+
|
|
44
|
+
from smartrag.api import create_app
|
|
45
|
+
from smartrag.engine import demo_engine
|
|
46
|
+
from smartrag.settings import Settings
|
|
47
|
+
|
|
48
|
+
overrides = {k: v for k, v in {"host": host, "port": port,
|
|
49
|
+
"api_keys_file": api_keys_file}.items() if v is not None}
|
|
50
|
+
if no_auth:
|
|
51
|
+
overrides["auth_disabled"] = True
|
|
52
|
+
settings = Settings(**overrides)
|
|
53
|
+
if settings.auth_disabled and not _is_loopback(settings.host):
|
|
54
|
+
raise typer.BadParameter(f"--no-auth is only allowed on a loopback address, not "
|
|
55
|
+
f"{settings.host}", param_hint="--no-auth")
|
|
56
|
+
if not demo:
|
|
57
|
+
# TODO: build the PostgreSQL engine from settings once the index exists.
|
|
58
|
+
typer.echo("smartrag: no index backend is implemented yet; run with --demo", err=True)
|
|
59
|
+
raise typer.Exit(2)
|
|
60
|
+
|
|
61
|
+
logging.basicConfig(level=log_level.upper(),
|
|
62
|
+
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
|
63
|
+
try:
|
|
64
|
+
web_app = create_app(demo_engine(), settings)
|
|
65
|
+
except ValueError as exc:
|
|
66
|
+
typer.echo(f"smartrag: {exc}", err=True)
|
|
67
|
+
raise typer.Exit(2) from exc
|
|
68
|
+
uvicorn.run(web_app, host=settings.host, port=settings.port, log_level=log_level)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@app.command()
|
|
72
|
+
def version() -> None:
|
|
73
|
+
"""Print the version."""
|
|
74
|
+
typer.echo(__version__)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def main() -> None:
|
|
78
|
+
app()
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Retrieval engines behind the query-plane API."""
|
|
2
|
+
|
|
3
|
+
from smartrag.engine.base import RequestContext, RetrievalEngine
|
|
4
|
+
from smartrag.engine.demo import demo_engine
|
|
5
|
+
from smartrag.engine.memory import InMemoryEngine
|
|
6
|
+
|
|
7
|
+
__all__ = ["InMemoryEngine", "RequestContext", "RetrievalEngine", "demo_engine"]
|
smartrag/engine/base.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""The retrieval-engine seam: what the query-plane API needs from an index.
|
|
2
|
+
|
|
3
|
+
The Web API depends only on this protocol. The v1 engine (PostgreSQL + pgvector +
|
|
4
|
+
PGroonga, query embeddings from OumiGo's `/v1/embeddings`) and the in-memory demo engine
|
|
5
|
+
both implement it, so the API never changes when the engine does.
|
|
6
|
+
|
|
7
|
+
Every call receives a `RequestContext`: who is calling (the API key) and, optionally, on
|
|
8
|
+
whose behalf (the end user, CommonRAG P2). v1 engines only log `on_behalf_of`; document
|
|
9
|
+
permissions will use it later without an interface change.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import Protocol
|
|
16
|
+
|
|
17
|
+
from smartrag.models import (
|
|
18
|
+
DocumentResponse,
|
|
19
|
+
FeedbackRequest,
|
|
20
|
+
ReadinessResponse,
|
|
21
|
+
ReadResponse,
|
|
22
|
+
ReadScope,
|
|
23
|
+
SearchRequest,
|
|
24
|
+
SearchResponse,
|
|
25
|
+
SourcesResponse,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class RequestContext:
|
|
31
|
+
key_id: str
|
|
32
|
+
on_behalf_of: str | None = None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class RetrievalEngine(Protocol):
|
|
36
|
+
async def describe_sources(self, ctx: RequestContext) -> SourcesResponse:
|
|
37
|
+
"""What is indexed, which fields can be filtered, how fresh it is."""
|
|
38
|
+
...
|
|
39
|
+
|
|
40
|
+
async def search(
|
|
41
|
+
self, request: SearchRequest, ctx: RequestContext, query_id: str
|
|
42
|
+
) -> SearchResponse:
|
|
43
|
+
"""Ranked evidence for a natural-language query. `query_id` is issued by the
|
|
44
|
+
API; the engine records it so feedback can refer to this search."""
|
|
45
|
+
...
|
|
46
|
+
|
|
47
|
+
async def read_chunk(
|
|
48
|
+
self,
|
|
49
|
+
chunk_id: str,
|
|
50
|
+
scope: ReadScope,
|
|
51
|
+
window: int,
|
|
52
|
+
max_chars: int,
|
|
53
|
+
cursor: str | None,
|
|
54
|
+
ctx: RequestContext,
|
|
55
|
+
) -> ReadResponse:
|
|
56
|
+
"""A chunk widened to `scope`. Raises `NotFound`, or `ChunkSuperseded` for a
|
|
57
|
+
chunk of an older document version."""
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
async def get_document(
|
|
61
|
+
self,
|
|
62
|
+
doc_id: str,
|
|
63
|
+
include_text: bool,
|
|
64
|
+
section: str | None,
|
|
65
|
+
max_chars: int,
|
|
66
|
+
cursor: str | None,
|
|
67
|
+
ctx: RequestContext,
|
|
68
|
+
) -> DocumentResponse:
|
|
69
|
+
"""Document metadata and outline; text (paged) with `include_text`."""
|
|
70
|
+
...
|
|
71
|
+
|
|
72
|
+
async def record_feedback(self, feedback: FeedbackRequest, ctx: RequestContext) -> None:
|
|
73
|
+
"""Store a relevance signal. Raises `NotFound` for an unknown `query_id`."""
|
|
74
|
+
...
|
|
75
|
+
|
|
76
|
+
async def readiness(self) -> ReadinessResponse:
|
|
77
|
+
"""Whether the index and the embedder can serve queries."""
|
|
78
|
+
...
|
smartrag/engine/demo.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""A small demo corpus: company rules and an expense manual (Japanese), a product README
|
|
2
|
+
(English), across two sources. Used by `smartrag serve --demo` and the API tests."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
|
|
8
|
+
from smartrag.engine.memory import Chunk, Document, InMemoryEngine, Source
|
|
9
|
+
|
|
10
|
+
_SYNCED = datetime(2026, 11, 8, 3, 0, tzinfo=timezone.utc)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def demo_engine() -> InMemoryEngine:
|
|
14
|
+
leave = ("第5章 休暇", "第22条 年次有給休暇")
|
|
15
|
+
return InMemoryEngine(
|
|
16
|
+
sources=[
|
|
17
|
+
Source("nas-kitei", "社内規程", "smb", _SYNCED),
|
|
18
|
+
Source("s3-manuals", "製品マニュアル", "s3", _SYNCED),
|
|
19
|
+
],
|
|
20
|
+
documents=[
|
|
21
|
+
Document(
|
|
22
|
+
source_id="nas-kitei",
|
|
23
|
+
path="就業規則/就業規則.pdf",
|
|
24
|
+
title="就業規則",
|
|
25
|
+
file_type="pdf",
|
|
26
|
+
version=3,
|
|
27
|
+
modified_at=datetime(2026, 4, 1, tzinfo=timezone.utc),
|
|
28
|
+
chunks=[
|
|
29
|
+
Chunk("年次有給休暇は、入社後6か月間継続勤務し、所定労働日の8割以上出勤した"
|
|
30
|
+
"従業員に10日付与する。", leave, 14),
|
|
31
|
+
Chunk("年次有給休暇は、翌年度に限り最大20日まで繰り越すことができる。"
|
|
32
|
+
"繰り越した休暇は、当年度に付与された休暇より先に取得したものとみなす。",
|
|
33
|
+
leave, 14),
|
|
34
|
+
Chunk("従業員が結婚するときは5日、配偶者が出産するときは3日の特別休暇を与える。",
|
|
35
|
+
("第5章 休暇", "第23条 特別休暇"), 15),
|
|
36
|
+
Chunk("従業員は、会社の許可を得た場合に限り、副業を行うことができる。",
|
|
37
|
+
("第6章 服務", "第30条 副業"), 18),
|
|
38
|
+
],
|
|
39
|
+
),
|
|
40
|
+
Document(
|
|
41
|
+
source_id="nas-kitei",
|
|
42
|
+
path="経理/経費精算マニュアル.docx",
|
|
43
|
+
title="経費精算マニュアル",
|
|
44
|
+
file_type="docx",
|
|
45
|
+
version=1,
|
|
46
|
+
modified_at=datetime(2025, 10, 15, tzinfo=timezone.utc),
|
|
47
|
+
chunks=[
|
|
48
|
+
Chunk("経費は発生から30日以内に経費精算システムで申請する。"
|
|
49
|
+
"領収書の画像を必ず添付すること。", ("1. 申請の流れ",)),
|
|
50
|
+
Chunk("出張の交通費は最も経済的な経路で計算する。"
|
|
51
|
+
"新幹線のグリーン車は部長以上に限り利用できる。", ("2. 交通費",)),
|
|
52
|
+
],
|
|
53
|
+
),
|
|
54
|
+
Document(
|
|
55
|
+
source_id="s3-manuals",
|
|
56
|
+
path="gotobox/README.md",
|
|
57
|
+
title="GotoBox Quick Start",
|
|
58
|
+
file_type="md",
|
|
59
|
+
version=2,
|
|
60
|
+
modified_at=datetime(2026, 9, 1, tzinfo=timezone.utc),
|
|
61
|
+
chunks=[
|
|
62
|
+
Chunk("GotoBox connects to your file server and indexes documents "
|
|
63
|
+
"every night at 3:00.", ("Quick Start",)),
|
|
64
|
+
Chunk("If indexing fails, check that the service account can read the "
|
|
65
|
+
"shared folder.", ("Quick Start", "Troubleshooting")),
|
|
66
|
+
],
|
|
67
|
+
),
|
|
68
|
+
],
|
|
69
|
+
)
|