cited-rag 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cited_rag-0.1.0/LICENSE +21 -0
- cited_rag-0.1.0/PKG-INFO +130 -0
- cited_rag-0.1.0/README.md +114 -0
- cited_rag-0.1.0/pyproject.toml +32 -0
- cited_rag-0.1.0/setup.cfg +4 -0
- cited_rag-0.1.0/src/cited_rag/__init__.py +4 -0
- cited_rag-0.1.0/src/cited_rag/chunking.py +147 -0
- cited_rag-0.1.0/src/cited_rag/cli.py +217 -0
- cited_rag-0.1.0/src/cited_rag/eval.py +104 -0
- cited_rag-0.1.0/src/cited_rag/generate.py +190 -0
- cited_rag-0.1.0/src/cited_rag/index.py +224 -0
- cited_rag-0.1.0/src/cited_rag/llm_adapter.py +28 -0
- cited_rag-0.1.0/src/cited_rag/retrieval.py +207 -0
- cited_rag-0.1.0/src/cited_rag/tokenize.py +96 -0
- cited_rag-0.1.0/src/cited_rag.egg-info/PKG-INFO +130 -0
- cited_rag-0.1.0/src/cited_rag.egg-info/SOURCES.txt +24 -0
- cited_rag-0.1.0/src/cited_rag.egg-info/dependency_links.txt +1 -0
- cited_rag-0.1.0/src/cited_rag.egg-info/entry_points.txt +2 -0
- cited_rag-0.1.0/src/cited_rag.egg-info/top_level.txt +1 -0
- cited_rag-0.1.0/tests/test_chunking.py +97 -0
- cited_rag-0.1.0/tests/test_cli.py +146 -0
- cited_rag-0.1.0/tests/test_eval.py +100 -0
- cited_rag-0.1.0/tests/test_generate.py +101 -0
- cited_rag-0.1.0/tests/test_index.py +79 -0
- cited_rag-0.1.0/tests/test_retrieval.py +55 -0
- cited_rag-0.1.0/tests/test_tokenize.py +31 -0
cited_rag-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 cited-rag contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
cited_rag-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cited-rag
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Local RAG you can trust: every claim cited, or the answer refused.
|
|
5
|
+
Author: cited-rag
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/shenn667/cited-rag
|
|
8
|
+
Keywords: rag,search,bm25,offline,citations
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Environment :: Console
|
|
12
|
+
Requires-Python: >=3.10
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
Dynamic: license-file
|
|
16
|
+
|
|
17
|
+
# cited-rag
|
|
18
|
+
|
|
19
|
+
[](https://github.com/shenn667/cited-rag/actions/workflows/ci.yml)
|
|
20
|
+
[](https://www.python.org/downloads/)
|
|
21
|
+
[](LICENSE)
|
|
22
|
+
[](https://github.com/shenn667/cited-rag)
|
|
23
|
+
|
|
24
|
+
Local RAG you can trust. Zero hallucination by design: every claim is cited
|
|
25
|
+
(`[file:L1-L2]`) or the answer is refused with `Insufficient evidence`.
|
|
26
|
+
|
|
27
|
+
Stdlib-only. Offline-first. No data leaves your machine unless you pass `--llm`
|
|
28
|
+
explicitly. Install with `pip install cited-rag` — no heavy dependencies.
|
|
29
|
+
|
|
30
|
+
## Quickstart (3 commands)
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install cited-rag
|
|
34
|
+
cited-rag index ./docs --db ./index.db
|
|
35
|
+
cited-rag ask "how does RRF fusion work?" --k 5 --min-score 0.35
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
Ask with machine-readable output:
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
cited-rag ask "what is the chunk overlap?" --k 5 --json
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Run the benchmark (reproduces the numbers below in one command):
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
cited-rag eval --golden eval/golden.jsonl
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## How it works
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
docs (.txt/.md)
|
|
54
|
+
|
|
|
55
|
+
ingestion + line-based chunking (40 lines, 10 overlap, hash + line offsets)
|
|
56
|
+
|
|
|
57
|
+
+----+----+
|
|
58
|
+
| |
|
|
59
|
+
BM25 TF-IDF/cosine (optional dense adapter: OFF by default)
|
|
60
|
+
(k1=1.5, (sublinear tf,
|
|
61
|
+
b=0.75) stopwords, L2-norm)
|
|
62
|
+
| |
|
|
63
|
+
+----+----+
|
|
64
|
+
|
|
|
65
|
+
Reciprocal Rank Fusion RRF(d) = sum_r 1 / (60 + rank_r(d))
|
|
66
|
+
|
|
|
67
|
+
light rerank (exact-phrase boost) -> top-k
|
|
68
|
+
|
|
|
69
|
+
extractive generation (verbatim phrases + [file:L1-L2])
|
|
70
|
+
|
|
|
71
|
+
fidelity gate (>= 0.5) + min-score gate (>= 0.35)
|
|
72
|
+
|
|
|
73
|
+
answer | `Insufficient evidence` (exit 0, zero citations)
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Why RRF: BM25 and cosine live on incompatible scales. Fusion on ranks only,
|
|
77
|
+
with k=60 per Cormac et al., keeps small-corpus behavior stable without tuning.
|
|
78
|
+
|
|
79
|
+
## Benchmark
|
|
80
|
+
|
|
81
|
+
Measured with `cited-rag eval --golden eval/golden.jsonl` on the bundled
|
|
82
|
+
corpus (12 docs in `examples/docs/`, 52 questions: 40 in-corpus + 12
|
|
83
|
+
out-of-corpus). Release gates (PRODUCT_BRIEF C4): Recall@5 >= 0.80,
|
|
84
|
+
MRR >= 0.60, correct-refusal rate >= 0.90.
|
|
85
|
+
|
|
86
|
+
| Metric | Target | Measured |
|
|
87
|
+
|---|---|---|
|
|
88
|
+
| Recall@5 | >= 0.80 | see `cited-rag eval` output |
|
|
89
|
+
| MRR | >= 0.60 | see `cited-rag eval` output |
|
|
90
|
+
| Correct-refusal (12 off-topic Q) | >= 0.90 | see `cited-rag eval` output |
|
|
91
|
+
|
|
92
|
+
Numbers are reproduced deterministically from `eval/golden.jsonl` — run the
|
|
93
|
+
single eval command above rather than trusting this table.
|
|
94
|
+
|
|
95
|
+
Golden format (one JSON object per line, 40 in-corpus + 12 out-of-corpus):
|
|
96
|
+
|
|
97
|
+
```json
|
|
98
|
+
{"question": "What are the default BM25 parameters?", "expected_doc": "01-bm25-ranking.md", "should_refuse": false}
|
|
99
|
+
{"question": "What is the best sourdough recipe?", "expected_doc": null, "should_refuse": true}
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
## Refusal contract
|
|
103
|
+
|
|
104
|
+
- Trigger: `max_score < --min-score` (default 0.35) or fidelity < 0.5.
|
|
105
|
+
- Output: the literal string `Insufficient evidence`, exit code 0.
|
|
106
|
+
- Guarantee: refused answers carry zero `[file:L1-L2]` markers — no invented
|
|
107
|
+
citations, ever. Assert it in tests, not by inspection.
|
|
108
|
+
|
|
109
|
+
## Limitations (honest)
|
|
110
|
+
|
|
111
|
+
- Lexical retrieval only by default: BM25 + TF-IDF miss paraphrases and
|
|
112
|
+
cross-lingual queries. The dense adapter (`--llm`) is opt-in and unevaluated
|
|
113
|
+
against the gates above.
|
|
114
|
+
- Small-corpus IDF: with ~12 demo docs, rare-term statistics are noisy;
|
|
115
|
+
scores stabilize as the corpus grows. Re-index after every corpus change.
|
|
116
|
+
- Extractive answers: verbatim quoting is faithful but stilted — no
|
|
117
|
+
synthesis, no multi-hop reasoning, fixed fidelity threshold (0.5).
|
|
118
|
+
- Secrets: the indexer warns on key-like patterns (`AKIA...`, `ghp_...`)
|
|
119
|
+
but does not redact. Scrub corpora before indexing.
|
|
120
|
+
- Scale: single-machine SQLite/JSON index; 50 docs index in < 60 s, but this
|
|
121
|
+
is not a distributed search engine and makes no such claim.
|
|
122
|
+
|
|
123
|
+
## Project context
|
|
124
|
+
|
|
125
|
+
Third portfolio pillar after `agent-context-bridge` and `llm-eval-ci`.
|
|
126
|
+
See `PRODUCT_BRIEF.md` for the testable criteria (C1–C6) this release targets.
|
|
127
|
+
|
|
128
|
+
## License
|
|
129
|
+
|
|
130
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# cited-rag
|
|
2
|
+
|
|
3
|
+
[](https://github.com/shenn667/cited-rag/actions/workflows/ci.yml)
|
|
4
|
+
[](https://www.python.org/downloads/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://github.com/shenn667/cited-rag)
|
|
7
|
+
|
|
8
|
+
Local RAG you can trust. Zero hallucination by design: every claim is cited
|
|
9
|
+
(`[file:L1-L2]`) or the answer is refused with `Insufficient evidence`.
|
|
10
|
+
|
|
11
|
+
Stdlib-only. Offline-first. No data leaves your machine unless you pass `--llm`
|
|
12
|
+
explicitly. Install with `pip install cited-rag` — no heavy dependencies.
|
|
13
|
+
|
|
14
|
+
## Quickstart (3 commands)
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install cited-rag
|
|
18
|
+
cited-rag index ./docs --db ./index.db
|
|
19
|
+
cited-rag ask "how does RRF fusion work?" --k 5 --min-score 0.35
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
Ask with machine-readable output:
|
|
23
|
+
|
|
24
|
+
```bash
|
|
25
|
+
cited-rag ask "what is the chunk overlap?" --k 5 --json
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
Run the benchmark (reproduces the numbers below in one command):
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
cited-rag eval --golden eval/golden.jsonl
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## How it works
|
|
35
|
+
|
|
36
|
+
```text
|
|
37
|
+
docs (.txt/.md)
|
|
38
|
+
|
|
|
39
|
+
ingestion + line-based chunking (40 lines, 10 overlap, hash + line offsets)
|
|
40
|
+
|
|
|
41
|
+
+----+----+
|
|
42
|
+
| |
|
|
43
|
+
BM25 TF-IDF/cosine (optional dense adapter: OFF by default)
|
|
44
|
+
(k1=1.5, (sublinear tf,
|
|
45
|
+
b=0.75) stopwords, L2-norm)
|
|
46
|
+
| |
|
|
47
|
+
+----+----+
|
|
48
|
+
|
|
|
49
|
+
Reciprocal Rank Fusion RRF(d) = sum_r 1 / (60 + rank_r(d))
|
|
50
|
+
|
|
|
51
|
+
light rerank (exact-phrase boost) -> top-k
|
|
52
|
+
|
|
|
53
|
+
extractive generation (verbatim phrases + [file:L1-L2])
|
|
54
|
+
|
|
|
55
|
+
fidelity gate (>= 0.5) + min-score gate (>= 0.35)
|
|
56
|
+
|
|
|
57
|
+
answer | `Insufficient evidence` (exit 0, zero citations)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Why RRF: BM25 and cosine live on incompatible scales. Fusion on ranks only,
|
|
61
|
+
with k=60 per Cormac et al., keeps small-corpus behavior stable without tuning.
|
|
62
|
+
|
|
63
|
+
## Benchmark
|
|
64
|
+
|
|
65
|
+
Measured with `cited-rag eval --golden eval/golden.jsonl` on the bundled
|
|
66
|
+
corpus (12 docs in `examples/docs/`, 52 questions: 40 in-corpus + 12
|
|
67
|
+
out-of-corpus). Release gates (PRODUCT_BRIEF C4): Recall@5 >= 0.80,
|
|
68
|
+
MRR >= 0.60, correct-refusal rate >= 0.90.
|
|
69
|
+
|
|
70
|
+
| Metric | Target | Measured |
|
|
71
|
+
|---|---|---|
|
|
72
|
+
| Recall@5 | >= 0.80 | see `cited-rag eval` output |
|
|
73
|
+
| MRR | >= 0.60 | see `cited-rag eval` output |
|
|
74
|
+
| Correct-refusal (12 off-topic Q) | >= 0.90 | see `cited-rag eval` output |
|
|
75
|
+
|
|
76
|
+
Numbers are reproduced deterministically from `eval/golden.jsonl` — run the
|
|
77
|
+
single eval command above rather than trusting this table.
|
|
78
|
+
|
|
79
|
+
Golden format (one JSON object per line, 40 in-corpus + 12 out-of-corpus):
|
|
80
|
+
|
|
81
|
+
```json
|
|
82
|
+
{"question": "What are the default BM25 parameters?", "expected_doc": "01-bm25-ranking.md", "should_refuse": false}
|
|
83
|
+
{"question": "What is the best sourdough recipe?", "expected_doc": null, "should_refuse": true}
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
## Refusal contract
|
|
87
|
+
|
|
88
|
+
- Trigger: `max_score < --min-score` (default 0.35) or fidelity < 0.5.
|
|
89
|
+
- Output: the literal string `Insufficient evidence`, exit code 0.
|
|
90
|
+
- Guarantee: refused answers carry zero `[file:L1-L2]` markers — no invented
|
|
91
|
+
citations, ever. Assert it in tests, not by inspection.
|
|
92
|
+
|
|
93
|
+
## Limitations (honest)
|
|
94
|
+
|
|
95
|
+
- Lexical retrieval only by default: BM25 + TF-IDF miss paraphrases and
|
|
96
|
+
cross-lingual queries. The dense adapter (`--llm`) is opt-in and unevaluated
|
|
97
|
+
against the gates above.
|
|
98
|
+
- Small-corpus IDF: with ~12 demo docs, rare-term statistics are noisy;
|
|
99
|
+
scores stabilize as the corpus grows. Re-index after every corpus change.
|
|
100
|
+
- Extractive answers: verbatim quoting is faithful but stilted — no
|
|
101
|
+
synthesis, no multi-hop reasoning, fixed fidelity threshold (0.5).
|
|
102
|
+
- Secrets: the indexer warns on key-like patterns (`AKIA...`, `ghp_...`)
|
|
103
|
+
but does not redact. Scrub corpora before indexing.
|
|
104
|
+
- Scale: single-machine SQLite/JSON index; 50 docs index in < 60 s, but this
|
|
105
|
+
is not a distributed search engine and makes no such claim.
|
|
106
|
+
|
|
107
|
+
## Project context
|
|
108
|
+
|
|
109
|
+
Third portfolio pillar after `agent-context-bridge` and `llm-eval-ci`.
|
|
110
|
+
See `PRODUCT_BRIEF.md` for the testable criteria (C1–C6) this release targets.
|
|
111
|
+
|
|
112
|
+
## License
|
|
113
|
+
|
|
114
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cited-rag"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Local RAG you can trust: every claim cited, or the answer refused."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "cited-rag" }]
|
|
13
|
+
keywords = ["rag", "search", "bm25", "offline", "citations"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Programming Language :: Python :: 3",
|
|
16
|
+
"License :: OSI Approved :: MIT License",
|
|
17
|
+
"Environment :: Console",
|
|
18
|
+
]
|
|
19
|
+
dependencies = []
|
|
20
|
+
|
|
21
|
+
[project.scripts]
|
|
22
|
+
cited-rag = "cited_rag.cli:main"
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/shenn667/cited-rag"
|
|
26
|
+
|
|
27
|
+
[tool.setuptools.packages.find]
|
|
28
|
+
where = ["src"]
|
|
29
|
+
|
|
30
|
+
[tool.pytest.ini_options]
|
|
31
|
+
testpaths = ["tests"]
|
|
32
|
+
addopts = "-q"
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Traceable line-based chunking with security confinement (stdlib only).
|
|
2
|
+
|
|
3
|
+
Every chunk records ``file`` (path relative to the indexed root),
|
|
4
|
+
1-indexed ``start_line``/``end_line`` offsets, and a SHA-256 ``hash`` of
|
|
5
|
+
its exact text, so any ``[file:L1-L2]`` citation is verifiable verbatim.
|
|
6
|
+
|
|
7
|
+
Rules (PRODUCT_BRIEF C1 / DESIGN):
|
|
8
|
+
- line-based: 40 lines per chunk, 10-line fixed overlap;
|
|
9
|
+
- a markdown heading is never orphaned as the last line of a chunk;
|
|
10
|
+
- only ``.txt``/``.md`` files under the declared root are ingested;
|
|
11
|
+
- symlinks are never followed; absolute paths / ``..`` escapes rejected;
|
|
12
|
+
- key-like secret patterns emit warnings (never redaction).
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
CHUNK_LINES = 40
|
|
21
|
+
OVERLAP = 10
|
|
22
|
+
ACCEPTED_SUFFIXES = (".txt", ".md")
|
|
23
|
+
|
|
24
|
+
# Key-like patterns worth warning about (index stores them as-is;
|
|
25
|
+
# the operator must scrub corpora -- see README "Limitations").
|
|
26
|
+
SECRET_PATTERNS = [
|
|
27
|
+
re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
28
|
+
re.compile(r"ghp_[A-Za-z0-9]{36}"),
|
|
29
|
+
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
30
|
+
re.compile(r"xox[bap]-[A-Za-z0-9-]+"),
|
|
31
|
+
re.compile(r"sk-(live|test)-[A-Za-z0-9]{16,}"),
|
|
32
|
+
re.compile(r"-----BEGIN (?:RSA )?PRIVATE KEY-----"),
|
|
33
|
+
re.compile(r"(?i)\bapi[_-]?key\b\s*[:=]\s*['\"]?[A-Za-z0-9_\-]{16,}"),
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def is_heading(line: str) -> bool:
|
|
38
|
+
"""True for ATX markdown headings (``# ...`` up to ``###### ...``)."""
|
|
39
|
+
stripped = line.lstrip()
|
|
40
|
+
if not stripped.startswith("#"):
|
|
41
|
+
return False
|
|
42
|
+
hashes = stripped.split(" ", 1)[0] if " " in stripped else stripped
|
|
43
|
+
hashes = hashes.rstrip("\n")
|
|
44
|
+
return bool(hashes) and all(c == "#" for c in hashes) and len(hashes) <= 6
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def chunk_text_lines(
|
|
48
|
+
lines: list[str],
|
|
49
|
+
source: str,
|
|
50
|
+
chunk_lines: int = CHUNK_LINES,
|
|
51
|
+
overlap: int = OVERLAP,
|
|
52
|
+
) -> list[dict]:
|
|
53
|
+
"""Split pre-read lines into traceable chunk dicts.
|
|
54
|
+
|
|
55
|
+
``lines``: file lines *without* trailing newlines. Offsets are
|
|
56
|
+
1-indexed and inclusive on both ends.
|
|
57
|
+
"""
|
|
58
|
+
if chunk_lines <= 0:
|
|
59
|
+
raise ValueError("chunk_lines must be > 0")
|
|
60
|
+
if not 0 <= overlap < chunk_lines:
|
|
61
|
+
raise ValueError("overlap must satisfy 0 <= overlap < chunk_lines")
|
|
62
|
+
|
|
63
|
+
chunks: list[dict] = []
|
|
64
|
+
n = len(lines)
|
|
65
|
+
start = 0
|
|
66
|
+
while start < n:
|
|
67
|
+
end = min(start + chunk_lines, n)
|
|
68
|
+
# Never orphan a heading as the chunk's last line: extend the
|
|
69
|
+
# boundary so the heading stays with the line that follows it.
|
|
70
|
+
while end < n and is_heading(lines[end - 1]):
|
|
71
|
+
end += 1
|
|
72
|
+
text = "\n".join(lines[start:end])
|
|
73
|
+
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
74
|
+
chunks.append(
|
|
75
|
+
{
|
|
76
|
+
"file": source,
|
|
77
|
+
"start_line": start + 1,
|
|
78
|
+
"end_line": end,
|
|
79
|
+
"hash": digest,
|
|
80
|
+
"text": text,
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
if end >= n:
|
|
84
|
+
break
|
|
85
|
+
start = max(end - overlap, start + 1)
|
|
86
|
+
return chunks
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def chunk_file(
|
|
90
|
+
abs_path: str,
|
|
91
|
+
rel_path: str,
|
|
92
|
+
chunk_lines: int = CHUNK_LINES,
|
|
93
|
+
overlap: int = OVERLAP,
|
|
94
|
+
) -> list[dict]:
|
|
95
|
+
"""Read a file and chunk it. Returns [] for empty files."""
|
|
96
|
+
with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
|
|
97
|
+
content = fh.read()
|
|
98
|
+
lines = content.splitlines()
|
|
99
|
+
if not lines:
|
|
100
|
+
return []
|
|
101
|
+
return chunk_text_lines(lines, rel_path, chunk_lines, overlap)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def iter_corpus_files(root: str) -> list[tuple[str, str]]:
|
|
105
|
+
"""List ``(rel_path, abs_path)`` of ingestible files under ``root``.
|
|
106
|
+
|
|
107
|
+
Sorted for determinism. Symlinks (files and dirs) are skipped,
|
|
108
|
+
and any path escaping the root (traversal) is rejected.
|
|
109
|
+
"""
|
|
110
|
+
if not os.path.isdir(root):
|
|
111
|
+
raise FileNotFoundError(f"{root} not found")
|
|
112
|
+
root_real = os.path.realpath(root)
|
|
113
|
+
found: list[tuple[str, str]] = []
|
|
114
|
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
115
|
+
# Never descend into symlinked directories.
|
|
116
|
+
dirnames[:] = sorted(
|
|
117
|
+
d
|
|
118
|
+
for d in dirnames
|
|
119
|
+
if not os.path.islink(os.path.join(dirpath, d))
|
|
120
|
+
)
|
|
121
|
+
for name in sorted(filenames):
|
|
122
|
+
abs_path = os.path.join(dirpath, name)
|
|
123
|
+
if os.path.islink(abs_path):
|
|
124
|
+
continue
|
|
125
|
+
if not name.lower().endswith(ACCEPTED_SUFFIXES):
|
|
126
|
+
continue
|
|
127
|
+
real = os.path.realpath(abs_path)
|
|
128
|
+
try:
|
|
129
|
+
common = os.path.commonpath([root_real, real])
|
|
130
|
+
except ValueError:
|
|
131
|
+
continue
|
|
132
|
+
if common != root_real:
|
|
133
|
+
continue
|
|
134
|
+
rel = os.path.relpath(abs_path, root)
|
|
135
|
+
if rel.startswith("..") or os.path.isabs(rel):
|
|
136
|
+
continue
|
|
137
|
+
found.append((rel, abs_path))
|
|
138
|
+
return sorted(found)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def find_secrets(text: str) -> list[str]:
|
|
142
|
+
"""Return the matched secret-like substrings (for warnings)."""
|
|
143
|
+
hits: list[str] = []
|
|
144
|
+
for pattern in SECRET_PATTERNS:
|
|
145
|
+
for match in pattern.finditer(text):
|
|
146
|
+
hits.append(match.group(0)[:64])
|
|
147
|
+
return hits
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""CLI: ``cited-rag index|ask|eval`` (stdlib only, argparse).
|
|
2
|
+
|
|
3
|
+
Exit codes (DESIGN sect. 5):
|
|
4
|
+
- 0: success OR nominal ``Insufficient evidence`` refusal;
|
|
5
|
+
- 1: runtime failure (missing/corrupt index, unreadable golden);
|
|
6
|
+
- 2: invalid usage (bad flag, missing dir, ``--k`` outside [1, 20]).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from . import __version__
|
|
16
|
+
from .eval import format_text as format_eval_text
|
|
17
|
+
from .eval import run_eval
|
|
18
|
+
from .generate import (
|
|
19
|
+
format_json_payload,
|
|
20
|
+
format_text as format_answer_text,
|
|
21
|
+
generate_answer,
|
|
22
|
+
)
|
|
23
|
+
from .index import index_corpus, load_index
|
|
24
|
+
from .llm_adapter import describe as describe_llm
|
|
25
|
+
from .llm_adapter import is_enabled as llm_enabled
|
|
26
|
+
from .retrieval import Retriever
|
|
27
|
+
|
|
28
|
+
DEFAULT_INDEX = os.path.join(".cited-rag", "index.json")
|
|
29
|
+
DEFAULT_GOLDEN = os.path.join("eval", "golden.jsonl")
|
|
30
|
+
DEFAULT_K = 5
|
|
31
|
+
DEFAULT_MIN_SCORE = 0.35
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _add_index_arg(parser: argparse.ArgumentParser) -> None:
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--index",
|
|
37
|
+
"--db",
|
|
38
|
+
dest="index",
|
|
39
|
+
default=DEFAULT_INDEX,
|
|
40
|
+
help=f"path to the local index file (default: {DEFAULT_INDEX})",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
45
|
+
parser = argparse.ArgumentParser(
|
|
46
|
+
prog="cited-rag",
|
|
47
|
+
description="Local RAG you can trust: every claim cited, or refused.",
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
50
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
51
|
+
|
|
52
|
+
p_index = sub.add_parser("index", help="build the local index from docs")
|
|
53
|
+
p_index.add_argument("docs", help="directory containing .txt/.md files")
|
|
54
|
+
_add_index_arg(p_index)
|
|
55
|
+
|
|
56
|
+
p_ask = sub.add_parser("ask", help="answer a question with citations")
|
|
57
|
+
p_ask.add_argument("question", help="question to answer")
|
|
58
|
+
p_ask.add_argument("--k", type=int, default=DEFAULT_K,
|
|
59
|
+
help="top-k chunks used (1-20, default: 5)")
|
|
60
|
+
p_ask.add_argument("--min-score", type=float, default=DEFAULT_MIN_SCORE,
|
|
61
|
+
dest="min_score",
|
|
62
|
+
help="refuse below this max_score (default: 0.35)")
|
|
63
|
+
p_ask.add_argument("--json", action="store_true",
|
|
64
|
+
help="emit machine-readable JSON")
|
|
65
|
+
p_ask.add_argument("--llm", action="store_true", default=False,
|
|
66
|
+
help="opt-in to the external LLM adapter (OFF by default)")
|
|
67
|
+
p_ask.add_argument("--llm-endpoint", default=None, dest="llm_endpoint",
|
|
68
|
+
help="external LLM endpoint URL (requires --llm)")
|
|
69
|
+
_add_index_arg(p_ask)
|
|
70
|
+
|
|
71
|
+
p_eval = sub.add_parser("eval", help="benchmark on a golden Q/A file")
|
|
72
|
+
p_eval.add_argument("--golden", default=DEFAULT_GOLDEN,
|
|
73
|
+
help=f"golden JSONL file (default: {DEFAULT_GOLDEN})")
|
|
74
|
+
p_eval.add_argument("--k", type=int, default=DEFAULT_K,
|
|
75
|
+
help="retrieval depth for Recall@k (1-20, default: 5)")
|
|
76
|
+
p_eval.add_argument("--min-score", type=float, default=DEFAULT_MIN_SCORE,
|
|
77
|
+
dest="min_score",
|
|
78
|
+
help="refusal gate used during eval (default: 0.35)")
|
|
79
|
+
p_eval.add_argument("--min-recall", type=float, default=None,
|
|
80
|
+
dest="min_recall",
|
|
81
|
+
help="fail (exit 1) if Recall@k is below this (0-1, default: no gate)")
|
|
82
|
+
p_eval.add_argument("--min-mrr", type=float, default=None,
|
|
83
|
+
dest="min_mrr",
|
|
84
|
+
help="fail (exit 1) if MRR is below this (0-1, default: no gate)")
|
|
85
|
+
p_eval.add_argument("--json", action="store_true",
|
|
86
|
+
help="emit machine-readable JSON")
|
|
87
|
+
_add_index_arg(p_eval)
|
|
88
|
+
return parser
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _check_k(k: int) -> int:
|
|
92
|
+
if k is None or not 1 <= k <= 20:
|
|
93
|
+
print("Error: --k must be in [1, 20]", file=sys.stderr)
|
|
94
|
+
raise SystemExit(2)
|
|
95
|
+
return k
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _check_gate(value: float | None, name: str) -> float | None:
|
|
99
|
+
if value is None:
|
|
100
|
+
return None
|
|
101
|
+
if not 0.0 <= value <= 1.0:
|
|
102
|
+
print(f"Error: {name} must be in [0, 1]", file=sys.stderr)
|
|
103
|
+
raise SystemExit(2)
|
|
104
|
+
return value
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _check_min_score(value: float) -> float:
|
|
108
|
+
if value is None or not 0.0 <= value <= 1.0:
|
|
109
|
+
print("Error: --min-score must be in [0, 1]", file=sys.stderr)
|
|
110
|
+
raise SystemExit(2)
|
|
111
|
+
return value
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def cmd_index(args: argparse.Namespace) -> int:
|
|
115
|
+
if not os.path.isdir(args.docs):
|
|
116
|
+
print(f"Error: {args.docs} not found", file=sys.stderr)
|
|
117
|
+
return 2
|
|
118
|
+
try:
|
|
119
|
+
summary = index_corpus(args.docs, args.index)
|
|
120
|
+
except (OSError, ValueError) as exc:
|
|
121
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
122
|
+
return 1
|
|
123
|
+
size_mb = summary["size_bytes"] / (1024 * 1024)
|
|
124
|
+
print(
|
|
125
|
+
f"Indexed {summary['num_files']} files -> "
|
|
126
|
+
f"{summary['num_chunks']} chunks -> {summary['index_path']} "
|
|
127
|
+
f"({size_mb:.1f} MB) in {summary['elapsed_s']:.1f}s"
|
|
128
|
+
)
|
|
129
|
+
for warning in summary["warnings"]:
|
|
130
|
+
print(f"Warning: {warning}", file=sys.stderr)
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _load_retriever(index_path: str) -> Retriever | None:
|
|
135
|
+
try:
|
|
136
|
+
return Retriever(load_index(index_path))
|
|
137
|
+
except FileNotFoundError as exc:
|
|
138
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
139
|
+
return None
|
|
140
|
+
except (ValueError, OSError, KeyError) as exc:
|
|
141
|
+
print(f"Error: corrupt index: {index_path} ({exc})", file=sys.stderr)
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def cmd_ask(args: argparse.Namespace) -> int:
|
|
146
|
+
_check_k(args.k)
|
|
147
|
+
_check_min_score(args.min_score)
|
|
148
|
+
retriever = _load_retriever(args.index)
|
|
149
|
+
if retriever is None:
|
|
150
|
+
return 1
|
|
151
|
+
if llm_enabled(args):
|
|
152
|
+
info = describe_llm()
|
|
153
|
+
print(
|
|
154
|
+
f"Note: LLM adapter active ({info['status']}); "
|
|
155
|
+
"citation format unchanged.",
|
|
156
|
+
file=sys.stderr,
|
|
157
|
+
)
|
|
158
|
+
hits = retriever.search(args.question, k=args.k)
|
|
159
|
+
result = generate_answer(args.question, hits, min_score=args.min_score)
|
|
160
|
+
if args.json:
|
|
161
|
+
print(json.dumps(format_json_payload(result), ensure_ascii=False))
|
|
162
|
+
else:
|
|
163
|
+
print(format_answer_text(result, k=args.k, min_score=args.min_score))
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def cmd_eval(args: argparse.Namespace) -> int:
|
|
168
|
+
_check_k(args.k)
|
|
169
|
+
_check_min_score(args.min_score)
|
|
170
|
+
_check_gate(args.min_recall, "--min-recall")
|
|
171
|
+
_check_gate(args.min_mrr, "--min-mrr")
|
|
172
|
+
retriever = _load_retriever(args.index)
|
|
173
|
+
if retriever is None:
|
|
174
|
+
return 1
|
|
175
|
+
try:
|
|
176
|
+
metrics = run_eval(
|
|
177
|
+
retriever, args.golden, k=args.k, min_score=args.min_score
|
|
178
|
+
)
|
|
179
|
+
except (FileNotFoundError, ValueError, OSError) as exc:
|
|
180
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
181
|
+
return 1
|
|
182
|
+
if args.json:
|
|
183
|
+
print(json.dumps(metrics, ensure_ascii=False))
|
|
184
|
+
else:
|
|
185
|
+
print(format_eval_text(metrics))
|
|
186
|
+
failures = []
|
|
187
|
+
if args.min_recall is not None and metrics["recall_at_k"] < args.min_recall:
|
|
188
|
+
failures.append(
|
|
189
|
+
f"recall_at_k={metrics['recall_at_k']:.4f} "
|
|
190
|
+
f"below --min-recall={args.min_recall}"
|
|
191
|
+
)
|
|
192
|
+
if args.min_mrr is not None and metrics["mrr"] < args.min_mrr:
|
|
193
|
+
failures.append(
|
|
194
|
+
f"mrr={metrics['mrr']:.4f} below --min-mrr={args.min_mrr}"
|
|
195
|
+
)
|
|
196
|
+
if failures:
|
|
197
|
+
for failure in failures:
|
|
198
|
+
print(f"Error: {failure}", file=sys.stderr)
|
|
199
|
+
return 1
|
|
200
|
+
return 0
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def main(argv: list[str] | None = None) -> int:
|
|
204
|
+
parser = build_parser()
|
|
205
|
+
args = parser.parse_args(argv)
|
|
206
|
+
if args.command == "index":
|
|
207
|
+
return cmd_index(args)
|
|
208
|
+
if args.command == "ask":
|
|
209
|
+
return cmd_ask(args)
|
|
210
|
+
if args.command == "eval":
|
|
211
|
+
return cmd_eval(args)
|
|
212
|
+
parser.print_usage(sys.stderr)
|
|
213
|
+
return 2
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
if __name__ == "__main__":
|
|
217
|
+
raise SystemExit(main())
|