cited-rag 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cited_rag/__init__.py +4 -0
- cited_rag/chunking.py +147 -0
- cited_rag/cli.py +217 -0
- cited_rag/eval.py +104 -0
- cited_rag/generate.py +190 -0
- cited_rag/index.py +224 -0
- cited_rag/llm_adapter.py +28 -0
- cited_rag/retrieval.py +207 -0
- cited_rag/tokenize.py +96 -0
- cited_rag-0.1.0.dist-info/METADATA +130 -0
- cited_rag-0.1.0.dist-info/RECORD +15 -0
- cited_rag-0.1.0.dist-info/WHEEL +5 -0
- cited_rag-0.1.0.dist-info/entry_points.txt +2 -0
- cited_rag-0.1.0.dist-info/licenses/LICENSE +21 -0
- cited_rag-0.1.0.dist-info/top_level.txt +1 -0
cited_rag/__init__.py
ADDED
cited_rag/chunking.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Traceable line-based chunking with security confinement (stdlib only).
|
|
2
|
+
|
|
3
|
+
Every chunk records ``file`` (path relative to the indexed root),
|
|
4
|
+
1-indexed ``start_line``/``end_line`` offsets, and a SHA-256 ``hash`` of
|
|
5
|
+
its exact text, so any ``[file:L1-L2]`` citation is verifiable verbatim.
|
|
6
|
+
|
|
7
|
+
Rules (PRODUCT_BRIEF C1 / DESIGN):
|
|
8
|
+
- line-based: 40 lines per chunk, 10-line fixed overlap;
|
|
9
|
+
- a markdown heading is never orphaned as the last line of a chunk;
|
|
10
|
+
- only ``.txt``/``.md`` files under the declared root are ingested;
|
|
11
|
+
- symlinks are never followed; absolute paths / ``..`` escapes rejected;
|
|
12
|
+
- key-like secret patterns emit warnings (never redaction).
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import os
|
|
18
|
+
import re
|
|
19
|
+
|
|
20
|
+
CHUNK_LINES = 40
|
|
21
|
+
OVERLAP = 10
|
|
22
|
+
ACCEPTED_SUFFIXES = (".txt", ".md")
|
|
23
|
+
|
|
24
|
+
# Key-like patterns worth warning about (index stores them as-is;
|
|
25
|
+
# the operator must scrub corpora -- see README "Limitations").
|
|
26
|
+
SECRET_PATTERNS = [
|
|
27
|
+
re.compile(r"AKIA[0-9A-Z]{16}"),
|
|
28
|
+
re.compile(r"ghp_[A-Za-z0-9]{36}"),
|
|
29
|
+
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
30
|
+
re.compile(r"xox[bap]-[A-Za-z0-9-]+"),
|
|
31
|
+
re.compile(r"sk-(live|test)-[A-Za-z0-9]{16,}"),
|
|
32
|
+
re.compile(r"-----BEGIN (?:RSA )?PRIVATE KEY-----"),
|
|
33
|
+
re.compile(r"(?i)\bapi[_-]?key\b\s*[:=]\s*['\"]?[A-Za-z0-9_\-]{16,}"),
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def is_heading(line: str) -> bool:
|
|
38
|
+
"""True for ATX markdown headings (``# ...`` up to ``###### ...``)."""
|
|
39
|
+
stripped = line.lstrip()
|
|
40
|
+
if not stripped.startswith("#"):
|
|
41
|
+
return False
|
|
42
|
+
hashes = stripped.split(" ", 1)[0] if " " in stripped else stripped
|
|
43
|
+
hashes = hashes.rstrip("\n")
|
|
44
|
+
return bool(hashes) and all(c == "#" for c in hashes) and len(hashes) <= 6
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def chunk_text_lines(
|
|
48
|
+
lines: list[str],
|
|
49
|
+
source: str,
|
|
50
|
+
chunk_lines: int = CHUNK_LINES,
|
|
51
|
+
overlap: int = OVERLAP,
|
|
52
|
+
) -> list[dict]:
|
|
53
|
+
"""Split pre-read lines into traceable chunk dicts.
|
|
54
|
+
|
|
55
|
+
``lines``: file lines *without* trailing newlines. Offsets are
|
|
56
|
+
1-indexed and inclusive on both ends.
|
|
57
|
+
"""
|
|
58
|
+
if chunk_lines <= 0:
|
|
59
|
+
raise ValueError("chunk_lines must be > 0")
|
|
60
|
+
if not 0 <= overlap < chunk_lines:
|
|
61
|
+
raise ValueError("overlap must satisfy 0 <= overlap < chunk_lines")
|
|
62
|
+
|
|
63
|
+
chunks: list[dict] = []
|
|
64
|
+
n = len(lines)
|
|
65
|
+
start = 0
|
|
66
|
+
while start < n:
|
|
67
|
+
end = min(start + chunk_lines, n)
|
|
68
|
+
# Never orphan a heading as the chunk's last line: extend the
|
|
69
|
+
# boundary so the heading stays with the line that follows it.
|
|
70
|
+
while end < n and is_heading(lines[end - 1]):
|
|
71
|
+
end += 1
|
|
72
|
+
text = "\n".join(lines[start:end])
|
|
73
|
+
digest = hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
74
|
+
chunks.append(
|
|
75
|
+
{
|
|
76
|
+
"file": source,
|
|
77
|
+
"start_line": start + 1,
|
|
78
|
+
"end_line": end,
|
|
79
|
+
"hash": digest,
|
|
80
|
+
"text": text,
|
|
81
|
+
}
|
|
82
|
+
)
|
|
83
|
+
if end >= n:
|
|
84
|
+
break
|
|
85
|
+
start = max(end - overlap, start + 1)
|
|
86
|
+
return chunks
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def chunk_file(
|
|
90
|
+
abs_path: str,
|
|
91
|
+
rel_path: str,
|
|
92
|
+
chunk_lines: int = CHUNK_LINES,
|
|
93
|
+
overlap: int = OVERLAP,
|
|
94
|
+
) -> list[dict]:
|
|
95
|
+
"""Read a file and chunk it. Returns [] for empty files."""
|
|
96
|
+
with open(abs_path, "r", encoding="utf-8", errors="replace") as fh:
|
|
97
|
+
content = fh.read()
|
|
98
|
+
lines = content.splitlines()
|
|
99
|
+
if not lines:
|
|
100
|
+
return []
|
|
101
|
+
return chunk_text_lines(lines, rel_path, chunk_lines, overlap)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def iter_corpus_files(root: str) -> list[tuple[str, str]]:
|
|
105
|
+
"""List ``(rel_path, abs_path)`` of ingestible files under ``root``.
|
|
106
|
+
|
|
107
|
+
Sorted for determinism. Symlinks (files and dirs) are skipped,
|
|
108
|
+
and any path escaping the root (traversal) is rejected.
|
|
109
|
+
"""
|
|
110
|
+
if not os.path.isdir(root):
|
|
111
|
+
raise FileNotFoundError(f"{root} not found")
|
|
112
|
+
root_real = os.path.realpath(root)
|
|
113
|
+
found: list[tuple[str, str]] = []
|
|
114
|
+
for dirpath, dirnames, filenames in os.walk(root, followlinks=False):
|
|
115
|
+
# Never descend into symlinked directories.
|
|
116
|
+
dirnames[:] = sorted(
|
|
117
|
+
d
|
|
118
|
+
for d in dirnames
|
|
119
|
+
if not os.path.islink(os.path.join(dirpath, d))
|
|
120
|
+
)
|
|
121
|
+
for name in sorted(filenames):
|
|
122
|
+
abs_path = os.path.join(dirpath, name)
|
|
123
|
+
if os.path.islink(abs_path):
|
|
124
|
+
continue
|
|
125
|
+
if not name.lower().endswith(ACCEPTED_SUFFIXES):
|
|
126
|
+
continue
|
|
127
|
+
real = os.path.realpath(abs_path)
|
|
128
|
+
try:
|
|
129
|
+
common = os.path.commonpath([root_real, real])
|
|
130
|
+
except ValueError:
|
|
131
|
+
continue
|
|
132
|
+
if common != root_real:
|
|
133
|
+
continue
|
|
134
|
+
rel = os.path.relpath(abs_path, root)
|
|
135
|
+
if rel.startswith("..") or os.path.isabs(rel):
|
|
136
|
+
continue
|
|
137
|
+
found.append((rel, abs_path))
|
|
138
|
+
return sorted(found)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def find_secrets(text: str) -> list[str]:
|
|
142
|
+
"""Return the matched secret-like substrings (for warnings)."""
|
|
143
|
+
hits: list[str] = []
|
|
144
|
+
for pattern in SECRET_PATTERNS:
|
|
145
|
+
for match in pattern.finditer(text):
|
|
146
|
+
hits.append(match.group(0)[:64])
|
|
147
|
+
return hits
|
cited_rag/cli.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""CLI: ``cited-rag index|ask|eval`` (stdlib only, argparse).
|
|
2
|
+
|
|
3
|
+
Exit codes (DESIGN sect. 5):
|
|
4
|
+
- 0: success OR nominal ``Insufficient evidence`` refusal;
|
|
5
|
+
- 1: runtime failure (missing/corrupt index, unreadable golden);
|
|
6
|
+
- 2: invalid usage (bad flag, missing dir, ``--k`` outside [1, 20]).
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
from . import __version__
|
|
16
|
+
from .eval import format_text as format_eval_text
|
|
17
|
+
from .eval import run_eval
|
|
18
|
+
from .generate import (
|
|
19
|
+
format_json_payload,
|
|
20
|
+
format_text as format_answer_text,
|
|
21
|
+
generate_answer,
|
|
22
|
+
)
|
|
23
|
+
from .index import index_corpus, load_index
|
|
24
|
+
from .llm_adapter import describe as describe_llm
|
|
25
|
+
from .llm_adapter import is_enabled as llm_enabled
|
|
26
|
+
from .retrieval import Retriever
|
|
27
|
+
|
|
28
|
+
DEFAULT_INDEX = os.path.join(".cited-rag", "index.json")
|
|
29
|
+
DEFAULT_GOLDEN = os.path.join("eval", "golden.jsonl")
|
|
30
|
+
DEFAULT_K = 5
|
|
31
|
+
DEFAULT_MIN_SCORE = 0.35
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _add_index_arg(parser: argparse.ArgumentParser) -> None:
|
|
35
|
+
parser.add_argument(
|
|
36
|
+
"--index",
|
|
37
|
+
"--db",
|
|
38
|
+
dest="index",
|
|
39
|
+
default=DEFAULT_INDEX,
|
|
40
|
+
help=f"path to the local index file (default: {DEFAULT_INDEX})",
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
45
|
+
parser = argparse.ArgumentParser(
|
|
46
|
+
prog="cited-rag",
|
|
47
|
+
description="Local RAG you can trust: every claim cited, or refused.",
|
|
48
|
+
)
|
|
49
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
50
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
51
|
+
|
|
52
|
+
p_index = sub.add_parser("index", help="build the local index from docs")
|
|
53
|
+
p_index.add_argument("docs", help="directory containing .txt/.md files")
|
|
54
|
+
_add_index_arg(p_index)
|
|
55
|
+
|
|
56
|
+
p_ask = sub.add_parser("ask", help="answer a question with citations")
|
|
57
|
+
p_ask.add_argument("question", help="question to answer")
|
|
58
|
+
p_ask.add_argument("--k", type=int, default=DEFAULT_K,
|
|
59
|
+
help="top-k chunks used (1-20, default: 5)")
|
|
60
|
+
p_ask.add_argument("--min-score", type=float, default=DEFAULT_MIN_SCORE,
|
|
61
|
+
dest="min_score",
|
|
62
|
+
help="refuse below this max_score (default: 0.35)")
|
|
63
|
+
p_ask.add_argument("--json", action="store_true",
|
|
64
|
+
help="emit machine-readable JSON")
|
|
65
|
+
p_ask.add_argument("--llm", action="store_true", default=False,
|
|
66
|
+
help="opt-in to the external LLM adapter (OFF by default)")
|
|
67
|
+
p_ask.add_argument("--llm-endpoint", default=None, dest="llm_endpoint",
|
|
68
|
+
help="external LLM endpoint URL (requires --llm)")
|
|
69
|
+
_add_index_arg(p_ask)
|
|
70
|
+
|
|
71
|
+
p_eval = sub.add_parser("eval", help="benchmark on a golden Q/A file")
|
|
72
|
+
p_eval.add_argument("--golden", default=DEFAULT_GOLDEN,
|
|
73
|
+
help=f"golden JSONL file (default: {DEFAULT_GOLDEN})")
|
|
74
|
+
p_eval.add_argument("--k", type=int, default=DEFAULT_K,
|
|
75
|
+
help="retrieval depth for Recall@k (1-20, default: 5)")
|
|
76
|
+
p_eval.add_argument("--min-score", type=float, default=DEFAULT_MIN_SCORE,
|
|
77
|
+
dest="min_score",
|
|
78
|
+
help="refusal gate used during eval (default: 0.35)")
|
|
79
|
+
p_eval.add_argument("--min-recall", type=float, default=None,
|
|
80
|
+
dest="min_recall",
|
|
81
|
+
help="fail (exit 1) if Recall@k is below this (0-1, default: no gate)")
|
|
82
|
+
p_eval.add_argument("--min-mrr", type=float, default=None,
|
|
83
|
+
dest="min_mrr",
|
|
84
|
+
help="fail (exit 1) if MRR is below this (0-1, default: no gate)")
|
|
85
|
+
p_eval.add_argument("--json", action="store_true",
|
|
86
|
+
help="emit machine-readable JSON")
|
|
87
|
+
_add_index_arg(p_eval)
|
|
88
|
+
return parser
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _check_k(k: int) -> int:
|
|
92
|
+
if k is None or not 1 <= k <= 20:
|
|
93
|
+
print("Error: --k must be in [1, 20]", file=sys.stderr)
|
|
94
|
+
raise SystemExit(2)
|
|
95
|
+
return k
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _check_gate(value: float | None, name: str) -> float | None:
|
|
99
|
+
if value is None:
|
|
100
|
+
return None
|
|
101
|
+
if not 0.0 <= value <= 1.0:
|
|
102
|
+
print(f"Error: {name} must be in [0, 1]", file=sys.stderr)
|
|
103
|
+
raise SystemExit(2)
|
|
104
|
+
return value
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _check_min_score(value: float) -> float:
|
|
108
|
+
if value is None or not 0.0 <= value <= 1.0:
|
|
109
|
+
print("Error: --min-score must be in [0, 1]", file=sys.stderr)
|
|
110
|
+
raise SystemExit(2)
|
|
111
|
+
return value
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def cmd_index(args: argparse.Namespace) -> int:
|
|
115
|
+
if not os.path.isdir(args.docs):
|
|
116
|
+
print(f"Error: {args.docs} not found", file=sys.stderr)
|
|
117
|
+
return 2
|
|
118
|
+
try:
|
|
119
|
+
summary = index_corpus(args.docs, args.index)
|
|
120
|
+
except (OSError, ValueError) as exc:
|
|
121
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
122
|
+
return 1
|
|
123
|
+
size_mb = summary["size_bytes"] / (1024 * 1024)
|
|
124
|
+
print(
|
|
125
|
+
f"Indexed {summary['num_files']} files -> "
|
|
126
|
+
f"{summary['num_chunks']} chunks -> {summary['index_path']} "
|
|
127
|
+
f"({size_mb:.1f} MB) in {summary['elapsed_s']:.1f}s"
|
|
128
|
+
)
|
|
129
|
+
for warning in summary["warnings"]:
|
|
130
|
+
print(f"Warning: {warning}", file=sys.stderr)
|
|
131
|
+
return 0
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _load_retriever(index_path: str) -> Retriever | None:
|
|
135
|
+
try:
|
|
136
|
+
return Retriever(load_index(index_path))
|
|
137
|
+
except FileNotFoundError as exc:
|
|
138
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
139
|
+
return None
|
|
140
|
+
except (ValueError, OSError, KeyError) as exc:
|
|
141
|
+
print(f"Error: corrupt index: {index_path} ({exc})", file=sys.stderr)
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def cmd_ask(args: argparse.Namespace) -> int:
|
|
146
|
+
_check_k(args.k)
|
|
147
|
+
_check_min_score(args.min_score)
|
|
148
|
+
retriever = _load_retriever(args.index)
|
|
149
|
+
if retriever is None:
|
|
150
|
+
return 1
|
|
151
|
+
if llm_enabled(args):
|
|
152
|
+
info = describe_llm()
|
|
153
|
+
print(
|
|
154
|
+
f"Note: LLM adapter active ({info['status']}); "
|
|
155
|
+
"citation format unchanged.",
|
|
156
|
+
file=sys.stderr,
|
|
157
|
+
)
|
|
158
|
+
hits = retriever.search(args.question, k=args.k)
|
|
159
|
+
result = generate_answer(args.question, hits, min_score=args.min_score)
|
|
160
|
+
if args.json:
|
|
161
|
+
print(json.dumps(format_json_payload(result), ensure_ascii=False))
|
|
162
|
+
else:
|
|
163
|
+
print(format_answer_text(result, k=args.k, min_score=args.min_score))
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def cmd_eval(args: argparse.Namespace) -> int:
|
|
168
|
+
_check_k(args.k)
|
|
169
|
+
_check_min_score(args.min_score)
|
|
170
|
+
_check_gate(args.min_recall, "--min-recall")
|
|
171
|
+
_check_gate(args.min_mrr, "--min-mrr")
|
|
172
|
+
retriever = _load_retriever(args.index)
|
|
173
|
+
if retriever is None:
|
|
174
|
+
return 1
|
|
175
|
+
try:
|
|
176
|
+
metrics = run_eval(
|
|
177
|
+
retriever, args.golden, k=args.k, min_score=args.min_score
|
|
178
|
+
)
|
|
179
|
+
except (FileNotFoundError, ValueError, OSError) as exc:
|
|
180
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
181
|
+
return 1
|
|
182
|
+
if args.json:
|
|
183
|
+
print(json.dumps(metrics, ensure_ascii=False))
|
|
184
|
+
else:
|
|
185
|
+
print(format_eval_text(metrics))
|
|
186
|
+
failures = []
|
|
187
|
+
if args.min_recall is not None and metrics["recall_at_k"] < args.min_recall:
|
|
188
|
+
failures.append(
|
|
189
|
+
f"recall_at_k={metrics['recall_at_k']:.4f} "
|
|
190
|
+
f"below --min-recall={args.min_recall}"
|
|
191
|
+
)
|
|
192
|
+
if args.min_mrr is not None and metrics["mrr"] < args.min_mrr:
|
|
193
|
+
failures.append(
|
|
194
|
+
f"mrr={metrics['mrr']:.4f} below --min-mrr={args.min_mrr}"
|
|
195
|
+
)
|
|
196
|
+
if failures:
|
|
197
|
+
for failure in failures:
|
|
198
|
+
print(f"Error: {failure}", file=sys.stderr)
|
|
199
|
+
return 1
|
|
200
|
+
return 0
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def main(argv: list[str] | None = None) -> int:
|
|
204
|
+
parser = build_parser()
|
|
205
|
+
args = parser.parse_args(argv)
|
|
206
|
+
if args.command == "index":
|
|
207
|
+
return cmd_index(args)
|
|
208
|
+
if args.command == "ask":
|
|
209
|
+
return cmd_ask(args)
|
|
210
|
+
if args.command == "eval":
|
|
211
|
+
return cmd_eval(args)
|
|
212
|
+
parser.print_usage(sys.stderr)
|
|
213
|
+
return 2
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
if __name__ == "__main__":
|
|
217
|
+
raise SystemExit(main())
|
cited_rag/eval.py
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""One-command benchmark over eval/golden.jsonl (stdlib only).
|
|
2
|
+
|
|
3
|
+
Metrics:
|
|
4
|
+
- Recall@k: fraction of answerable questions whose ``expected_doc``
|
|
5
|
+
(basename match) appears in the top-k retrieved chunk files.
|
|
6
|
+
- MRR: mean of 1/rank of the first relevant chunk (answerable only).
|
|
7
|
+
- correct-refusal rate: fraction of ``should_refuse`` questions for which
|
|
8
|
+
the full ask pipeline (search + generate + min_score gate) refuses.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
|
|
15
|
+
from .generate import generate_answer
|
|
16
|
+
from .retrieval import Retriever
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def load_golden(path: str) -> list[dict]:
|
|
20
|
+
if not os.path.exists(path):
|
|
21
|
+
raise FileNotFoundError(f"golden file not found: {path}")
|
|
22
|
+
records = []
|
|
23
|
+
with open(path, "r", encoding="utf-8") as fh:
|
|
24
|
+
for lineno, line in enumerate(fh, 1):
|
|
25
|
+
line = line.strip()
|
|
26
|
+
if not line:
|
|
27
|
+
continue
|
|
28
|
+
try:
|
|
29
|
+
rec = json.loads(line)
|
|
30
|
+
except json.JSONDecodeError as exc:
|
|
31
|
+
raise ValueError(f"{path}:{lineno}: invalid JSON ({exc})") from exc
|
|
32
|
+
if "question" not in rec or "should_refuse" not in rec:
|
|
33
|
+
raise ValueError(f"{path}:{lineno}: missing question/should_refuse")
|
|
34
|
+
records.append(rec)
|
|
35
|
+
if not records:
|
|
36
|
+
raise ValueError(f"golden file is empty: {path}")
|
|
37
|
+
return records
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _basename(path: str | None) -> str | None:
|
|
41
|
+
if not path:
|
|
42
|
+
return None
|
|
43
|
+
return os.path.basename(path)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def run_eval(
|
|
47
|
+
retriever: Retriever,
|
|
48
|
+
golden_path: str,
|
|
49
|
+
k: int = 5,
|
|
50
|
+
min_score: float = 0.35,
|
|
51
|
+
) -> dict:
|
|
52
|
+
records = load_golden(golden_path)
|
|
53
|
+
n_answerable = 0
|
|
54
|
+
n_refusal = 0
|
|
55
|
+
recall_hits = 0
|
|
56
|
+
reciprocal_sum = 0.0
|
|
57
|
+
correct_refusals = 0
|
|
58
|
+
|
|
59
|
+
for rec in records:
|
|
60
|
+
question = rec["question"]
|
|
61
|
+
if rec.get("should_refuse"):
|
|
62
|
+
n_refusal += 1
|
|
63
|
+
hits = retriever.search(question, k=k)
|
|
64
|
+
result = generate_answer(question, hits, min_score=min_score)
|
|
65
|
+
if result["refused"]:
|
|
66
|
+
correct_refusals += 1
|
|
67
|
+
else:
|
|
68
|
+
n_answerable += 1
|
|
69
|
+
expected = _basename(rec.get("expected_doc"))
|
|
70
|
+
hits = retriever.search(question, k=k)
|
|
71
|
+
rank = None
|
|
72
|
+
for i, hit in enumerate(hits, 1):
|
|
73
|
+
if _basename(hit["chunk"]["file"]) == expected:
|
|
74
|
+
rank = i
|
|
75
|
+
break
|
|
76
|
+
if rank is not None:
|
|
77
|
+
recall_hits += 1
|
|
78
|
+
reciprocal_sum += 1.0 / rank
|
|
79
|
+
|
|
80
|
+
recall = recall_hits / n_answerable if n_answerable else 0.0
|
|
81
|
+
mrr = reciprocal_sum / n_answerable if n_answerable else 0.0
|
|
82
|
+
refusal_rate = correct_refusals / n_refusal if n_refusal else 1.0
|
|
83
|
+
return {
|
|
84
|
+
"recall_at_k": round(recall, 4),
|
|
85
|
+
f"recall_at_{k}": round(recall, 4),
|
|
86
|
+
"mrr": round(mrr, 4),
|
|
87
|
+
"correct_refusal_rate": round(refusal_rate, 4),
|
|
88
|
+
"correct_refusals": correct_refusals,
|
|
89
|
+
"n_refusal": n_refusal,
|
|
90
|
+
"n_answerable": n_answerable,
|
|
91
|
+
"n": len(records),
|
|
92
|
+
"k": k,
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def format_text(metrics: dict) -> str:
|
|
97
|
+
k = metrics["k"]
|
|
98
|
+
recall = metrics["recall_at_k"]
|
|
99
|
+
return (
|
|
100
|
+
f"Recall@{k}={recall:.2f} MRR={metrics['mrr']:.2f} "
|
|
101
|
+
f"correct-refusal={metrics['correct_refusal_rate']:.2f} "
|
|
102
|
+
f"({metrics['correct_refusals']}/{metrics['n_refusal']}) "
|
|
103
|
+
f"n={metrics['n']}"
|
|
104
|
+
)
|
cited_rag/generate.py
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Extractive answer generation with exact citations + refusal gate.
|
|
2
|
+
|
|
3
|
+
Every affirmative sentence is a verbatim phrase from a retrieved chunk,
|
|
4
|
+
followed by ``[file:L1-L2]`` with the chunk's exact line offsets.
|
|
5
|
+
``fidelity`` = fraction of answer sentences found verbatim in their cited
|
|
6
|
+
chunk (1.0 by construction). Refusal ("Insufficient evidence") fires when
|
|
7
|
+
``max_score < min_score`` or fidelity < 0.5, carries zero citations, and
|
|
8
|
+
exits 0 (nominal outcome, not an error).
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import re
|
|
13
|
+
|
|
14
|
+
from .tokenize import normalize, tokenize
|
|
15
|
+
|
|
16
|
+
FIDELITY_THRESHOLD = 0.5
|
|
17
|
+
REFUSAL_TEXT = "Insufficient evidence"
|
|
18
|
+
CITATION_RE = re.compile(r"\[([^\[\]]+):L(\d+)-L(\d+)\]")
|
|
19
|
+
|
|
20
|
+
_SENT_SPLIT_RE = re.compile(r"(?<=[.!?])\s+(?=[\"'“”‘’(\[]?[A-Z0-9])")
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _collapse_ws(text: str) -> str:
|
|
24
|
+
return re.sub(r"\s+", " ", text).strip()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def split_sentences(text: str) -> list[str]:
|
|
28
|
+
"""Split chunk text into candidate verbatim sentences.
|
|
29
|
+
|
|
30
|
+
Non-heading blocks are joined with spaces first so sentences wrapped
|
|
31
|
+
across source lines stay whole; headings are dropped (titles, not claims).
|
|
32
|
+
"""
|
|
33
|
+
blocks = [
|
|
34
|
+
b.strip()
|
|
35
|
+
for b in text.split("\n")
|
|
36
|
+
if b.strip() and not b.lstrip().startswith("#")
|
|
37
|
+
]
|
|
38
|
+
if not blocks:
|
|
39
|
+
return []
|
|
40
|
+
parts = _SENT_SPLIT_RE.split(" ".join(blocks))
|
|
41
|
+
return [p.strip() for p in parts if p.strip()]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def format_citation(chunk: dict) -> str:
|
|
45
|
+
return f"[{chunk['file']}:L{chunk['start_line']}-L{chunk['end_line']}]"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def compute_fidelity(sentences: list[str], chunks: list[dict]) -> float:
|
|
49
|
+
"""Fraction of answer sentences found verbatim in their cited chunk.
|
|
50
|
+
|
|
51
|
+
Comparison is whitespace-insensitive (source line-wraps are joins, not
|
|
52
|
+
edits), so wrapped sentences still verify against exact line offsets.
|
|
53
|
+
"""
|
|
54
|
+
if not sentences:
|
|
55
|
+
return 0.0
|
|
56
|
+
normalized = [_collapse_ws(normalize(c["text"])) for c in chunks]
|
|
57
|
+
supported = 0
|
|
58
|
+
for sent in sentences:
|
|
59
|
+
needle = _collapse_ws(normalize(sent))
|
|
60
|
+
if needle and any(needle in body for body in normalized):
|
|
61
|
+
supported += 1
|
|
62
|
+
return supported / len(sentences)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def select_sentences(query: str, hits: list[dict], max_sentences: int = 3) -> list[tuple[str, dict]]:
|
|
66
|
+
"""Pick verbatim query-relevant sentences from ranked hits.
|
|
67
|
+
|
|
68
|
+
Returns ``(sentence, chunk)`` pairs, chunk-rank ordered, deduped.
|
|
69
|
+
Falls back to each hit's first sentence so an answer is never empty
|
|
70
|
+
when evidence exists.
|
|
71
|
+
"""
|
|
72
|
+
query_terms = set(tokenize(query))
|
|
73
|
+
picked: list[tuple[str, dict]] = []
|
|
74
|
+
seen: set[str] = set()
|
|
75
|
+
fallback: list[tuple[str, dict]] = []
|
|
76
|
+
for hit in hits:
|
|
77
|
+
chunk = hit["chunk"]
|
|
78
|
+
sentences = split_sentences(chunk["text"])
|
|
79
|
+
first_body = next(
|
|
80
|
+
(s for s in sentences if not s.lstrip().startswith("#")), None
|
|
81
|
+
)
|
|
82
|
+
if first_body and first_body not in seen:
|
|
83
|
+
fallback.append((first_body, chunk))
|
|
84
|
+
for sent in sentences:
|
|
85
|
+
if sent.lstrip().startswith("#"):
|
|
86
|
+
continue # headings are titles, not claims
|
|
87
|
+
key = normalize(sent)
|
|
88
|
+
if key in seen or len(picked) >= max_sentences:
|
|
89
|
+
continue
|
|
90
|
+
if query_terms and not (set(tokenize(sent)) & query_terms):
|
|
91
|
+
continue
|
|
92
|
+
seen.add(key)
|
|
93
|
+
picked.append((sent, chunk))
|
|
94
|
+
if len(picked) >= max_sentences:
|
|
95
|
+
break
|
|
96
|
+
if not picked:
|
|
97
|
+
for sent, chunk in fallback[:max_sentences]:
|
|
98
|
+
if normalize(sent) not in seen:
|
|
99
|
+
seen.add(normalize(sent))
|
|
100
|
+
picked.append((sent, chunk))
|
|
101
|
+
return picked[:max_sentences]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def generate_answer(
|
|
105
|
+
query: str, hits: list[dict], min_score: float = 0.35
|
|
106
|
+
) -> dict:
|
|
107
|
+
"""Build the answer payload (refused or cited)."""
|
|
108
|
+
max_score = max((h["score"] for h in hits), default=0.0)
|
|
109
|
+
if not hits or max_score < min_score:
|
|
110
|
+
return {
|
|
111
|
+
"answer": [],
|
|
112
|
+
"citations": [],
|
|
113
|
+
"fidelity": 0.0,
|
|
114
|
+
"max_score": max_score,
|
|
115
|
+
"refused": True,
|
|
116
|
+
}
|
|
117
|
+
pairs = select_sentences(query, hits)
|
|
118
|
+
sentences = [s for s, _ in pairs]
|
|
119
|
+
chunks = [c for _, c in pairs]
|
|
120
|
+
fidelity = compute_fidelity(sentences, chunks)
|
|
121
|
+
if fidelity < FIDELITY_THRESHOLD or not sentences:
|
|
122
|
+
return {
|
|
123
|
+
"answer": [],
|
|
124
|
+
"citations": [],
|
|
125
|
+
"fidelity": fidelity,
|
|
126
|
+
"max_score": max_score,
|
|
127
|
+
"refused": True,
|
|
128
|
+
}
|
|
129
|
+
citations = [
|
|
130
|
+
{
|
|
131
|
+
"file": chunk["file"],
|
|
132
|
+
"start_line": chunk["start_line"],
|
|
133
|
+
"end_line": chunk["end_line"],
|
|
134
|
+
"score": round(hit_score_for(hits, chunk), 4),
|
|
135
|
+
}
|
|
136
|
+
for (_, chunk) in pairs
|
|
137
|
+
]
|
|
138
|
+
return {
|
|
139
|
+
"answer": sentences,
|
|
140
|
+
"citations": citations,
|
|
141
|
+
"fidelity": round(fidelity, 4),
|
|
142
|
+
"max_score": round(max_score, 4),
|
|
143
|
+
"refused": False,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def hit_score_for(hits: list[dict], chunk: dict) -> float:
|
|
148
|
+
for hit in hits:
|
|
149
|
+
if hit["chunk"] is chunk or hit["chunk"] == chunk:
|
|
150
|
+
return hit["score"]
|
|
151
|
+
return 0.0
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def format_text(result: dict, k: int, min_score: float) -> str:
|
|
155
|
+
if result["refused"]:
|
|
156
|
+
return (
|
|
157
|
+
f"{REFUSAL_TEXT} (max_score={result['max_score']:.2f} "
|
|
158
|
+
f"< min-score={min_score:.2f}, k={k})."
|
|
159
|
+
)
|
|
160
|
+
lines = []
|
|
161
|
+
# Re-associate sentences with citations in order.
|
|
162
|
+
for sent, cite in zip(result["answer"], result["citations"]):
|
|
163
|
+
lines.append(
|
|
164
|
+
f"{sent} [{cite['file']}:L{cite['start_line']}-L{cite['end_line']}]"
|
|
165
|
+
)
|
|
166
|
+
total = len(result["answer"])
|
|
167
|
+
supported = round(result["fidelity"] * total)
|
|
168
|
+
lines.append(
|
|
169
|
+
f"Fidelity: {result['fidelity']:.2f} ({supported}/{total} supported) "
|
|
170
|
+
f"| max_score={result['max_score']:.2f} | k={k}"
|
|
171
|
+
)
|
|
172
|
+
return "\n".join(lines)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def format_json_payload(result: dict) -> dict:
|
|
176
|
+
if result["refused"]:
|
|
177
|
+
return {
|
|
178
|
+
"answer": REFUSAL_TEXT,
|
|
179
|
+
"citations": [],
|
|
180
|
+
"fidelity": 0.0,
|
|
181
|
+
"max_score": result["max_score"],
|
|
182
|
+
"refused": True,
|
|
183
|
+
}
|
|
184
|
+
return {
|
|
185
|
+
"answer": result["answer"],
|
|
186
|
+
"citations": result["citations"],
|
|
187
|
+
"fidelity": result["fidelity"],
|
|
188
|
+
"max_score": result["max_score"],
|
|
189
|
+
"refused": False,
|
|
190
|
+
}
|