MetacouplingLLM 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metacouplingllm/__init__.py +208 -0
- metacouplingllm/core.py +4542 -0
- metacouplingllm/data/Papers.zip +0 -0
- metacouplingllm/data/PeriTelecoupling_clean.csv +49953 -0
- metacouplingllm/data/PeriTelecoupling_subset.csv +11 -0
- metacouplingllm/data/chunk_embeddings.manifest.json +11 -0
- metacouplingllm/data/chunk_embeddings.npy +0 -0
- metacouplingllm/data/pericoupled_adm1_edge_list.csv +8291 -0
- metacouplingllm/data/telecoupling_literature.bib +3498 -0
- metacouplingllm/knowledge/__init__.py +1 -0
- metacouplingllm/knowledge/adm1_pericoupling.py +591 -0
- metacouplingllm/knowledge/citations.py +109 -0
- metacouplingllm/knowledge/countries.py +557 -0
- metacouplingllm/knowledge/examples.py +713 -0
- metacouplingllm/knowledge/framework.py +717 -0
- metacouplingllm/knowledge/literature.py +518 -0
- metacouplingllm/knowledge/pericoupling.py +241 -0
- metacouplingllm/knowledge/rag.py +1791 -0
- metacouplingllm/knowledge/references.py +112 -0
- metacouplingllm/knowledge/websearch.py +1842 -0
- metacouplingllm/llm/__init__.py +1 -0
- metacouplingllm/llm/client.py +636 -0
- metacouplingllm/llm/parser.py +868 -0
- metacouplingllm/output/__init__.py +1 -0
- metacouplingllm/output/formatter.py +276 -0
- metacouplingllm/prompts/__init__.py +1 -0
- metacouplingllm/prompts/builder.py +477 -0
- metacouplingllm/prompts/templates.py +556 -0
- metacouplingllm/visualization/__init__.py +114 -0
- metacouplingllm/visualization/adm1_map.py +884 -0
- metacouplingllm/visualization/worldmap.py +1372 -0
- metacouplingllm-0.1.0.dist-info/METADATA +440 -0
- metacouplingllm-0.1.0.dist-info/RECORD +35 -0
- metacouplingllm-0.1.0.dist-info/WHEEL +4 -0
- metacouplingllm-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1791 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Retrieval-Augmented Generation (RAG) for metacoupling paper evidence.
|
|
3
|
+
|
|
4
|
+
Loads full-text markdown papers, matches them against the BibTeX literature
|
|
5
|
+
database, chunks them into passages, and retrieves relevant passages given
|
|
6
|
+
a query (typically derived from an LLM analysis).
|
|
7
|
+
|
|
8
|
+
Two retrieval backends are available:
|
|
9
|
+
|
|
10
|
+
- :class:`EmbeddingRetriever` (default): semantic retrieval using
|
|
11
|
+
``fastembed`` + pre-computed corpus embeddings bundled with the
|
|
12
|
+
package. Catches synonyms, paraphrases, and related concepts.
|
|
13
|
+
- :class:`TfIdfIndex` (fallback): lexical retrieval using TF-IDF +
|
|
14
|
+
cosine similarity. Fast, no external dependencies, but misses
|
|
15
|
+
semantic matches. Used when ``fastembed`` is not installed or the
|
|
16
|
+
pre-computed embedding file is missing.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import math
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
from collections import Counter
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import TYPE_CHECKING
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
from metacouplingllm.knowledge.literature import Paper
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
logger = logging.getLogger(__name__)
|
|
37
|
+
|
|
38
|
+
# ---------------------------------------------------------------------------
|
|
39
|
+
# Bundled papers extraction
|
|
40
|
+
# ---------------------------------------------------------------------------
|
|
41
|
+
|
|
42
|
+
_BUNDLED_ZIP = Path(__file__).parent / ".." / "data" / "Papers.zip"
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _extract_bundled_papers(verbose: bool = False) -> Path:
|
|
46
|
+
"""Extract the bundled ``data/Papers.zip`` to a cache directory.
|
|
47
|
+
|
|
48
|
+
Returns the path to the extracted ``Papers/`` directory. If the
|
|
49
|
+
files have already been extracted (same zip mtime), the cached
|
|
50
|
+
version is reused.
|
|
51
|
+
"""
|
|
52
|
+
import shutil
|
|
53
|
+
import tempfile
|
|
54
|
+
import zipfile
|
|
55
|
+
|
|
56
|
+
zip_path = _BUNDLED_ZIP.resolve()
|
|
57
|
+
if not zip_path.exists():
|
|
58
|
+
return Path("__nonexistent__")
|
|
59
|
+
|
|
60
|
+
# Use a stable cache directory based on the package location
|
|
61
|
+
cache_root = Path(tempfile.gettempdir()) / "metacoupling_rag_cache"
|
|
62
|
+
papers_dir = cache_root / "Papers"
|
|
63
|
+
stamp_file = cache_root / ".extracted_stamp"
|
|
64
|
+
|
|
65
|
+
# Check if already extracted and up-to-date
|
|
66
|
+
zip_mtime = zip_path.stat().st_mtime
|
|
67
|
+
if papers_dir.is_dir() and stamp_file.exists():
|
|
68
|
+
try:
|
|
69
|
+
cached_mtime = float(stamp_file.read_text().strip())
|
|
70
|
+
if cached_mtime == zip_mtime:
|
|
71
|
+
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
72
|
+
expected_files = {
|
|
73
|
+
Path(name).name
|
|
74
|
+
for name in zf.namelist()
|
|
75
|
+
if name.startswith("Papers/") and name.endswith(".md")
|
|
76
|
+
}
|
|
77
|
+
extracted_files = {
|
|
78
|
+
path.name for path in papers_dir.glob("*.md")
|
|
79
|
+
}
|
|
80
|
+
if expected_files == extracted_files:
|
|
81
|
+
if verbose:
|
|
82
|
+
print(
|
|
83
|
+
f"[RAG] Using cached bundled papers: {papers_dir}"
|
|
84
|
+
)
|
|
85
|
+
return papers_dir
|
|
86
|
+
if verbose:
|
|
87
|
+
print("[RAG] Refreshing bundled papers cache: file list changed.")
|
|
88
|
+
except (ValueError, OSError):
|
|
89
|
+
pass
|
|
90
|
+
|
|
91
|
+
# Extract
|
|
92
|
+
if verbose:
|
|
93
|
+
print(f"[RAG] Extracting bundled Papers.zip → {cache_root}")
|
|
94
|
+
|
|
95
|
+
cache_root.mkdir(parents=True, exist_ok=True)
|
|
96
|
+
if papers_dir.exists():
|
|
97
|
+
shutil.rmtree(papers_dir, ignore_errors=True)
|
|
98
|
+
with zipfile.ZipFile(zip_path, "r") as zf:
|
|
99
|
+
zf.extractall(cache_root)
|
|
100
|
+
|
|
101
|
+
stamp_file.write_text(str(zip_mtime))
|
|
102
|
+
|
|
103
|
+
if verbose:
|
|
104
|
+
md_count = sum(1 for f in papers_dir.iterdir() if f.suffix == ".md")
|
|
105
|
+
print(f"[RAG] Extracted {md_count} bundled paper files.")
|
|
106
|
+
|
|
107
|
+
return papers_dir
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
# ---------------------------------------------------------------------------
|
|
111
|
+
# Data structures
|
|
112
|
+
# ---------------------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class TextChunk:
|
|
117
|
+
"""A passage extracted from a full-text paper.
|
|
118
|
+
|
|
119
|
+
Attributes
|
|
120
|
+
----------
|
|
121
|
+
paper_key:
|
|
122
|
+
BibTeX citation key of the matched paper.
|
|
123
|
+
paper_title:
|
|
124
|
+
Title of the paper.
|
|
125
|
+
authors:
|
|
126
|
+
Author string.
|
|
127
|
+
year:
|
|
128
|
+
Publication year.
|
|
129
|
+
section:
|
|
130
|
+
Section heading the passage comes from (e.g., "INTRODUCTION").
|
|
131
|
+
text:
|
|
132
|
+
The actual passage text.
|
|
133
|
+
chunk_index:
|
|
134
|
+
Position of this chunk within the paper (0-indexed).
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
paper_key: str = ""
|
|
138
|
+
paper_title: str = ""
|
|
139
|
+
authors: str = ""
|
|
140
|
+
year: int = 0
|
|
141
|
+
section: str = ""
|
|
142
|
+
text: str = ""
|
|
143
|
+
chunk_index: int = 0
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
@dataclass
|
|
147
|
+
class RetrievalResult:
|
|
148
|
+
"""A retrieved passage with its relevance score.
|
|
149
|
+
|
|
150
|
+
Attributes
|
|
151
|
+
----------
|
|
152
|
+
chunk:
|
|
153
|
+
The text chunk that was retrieved.
|
|
154
|
+
score:
|
|
155
|
+
Cosine similarity score (0–1).
|
|
156
|
+
"""
|
|
157
|
+
|
|
158
|
+
chunk: TextChunk
|
|
159
|
+
score: float = 0.0
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
# ---------------------------------------------------------------------------
|
|
163
|
+
# Paper matching: filename → BibTeX entry
|
|
164
|
+
# ---------------------------------------------------------------------------
|
|
165
|
+
|
|
166
|
+
# Pattern for Zotero-style markdown filenames:
|
|
167
|
+
# "Author et al. - YYYY - Title.md"
|
|
168
|
+
# "Author and Author2 - YYYY - Title.md"
|
|
169
|
+
_FILENAME_RE = re.compile(
|
|
170
|
+
r"^(?P<authors>.+?)\s*-\s*(?P<year>\d{4})\s*-\s*(?P<title>.+)\.md$",
|
|
171
|
+
re.IGNORECASE,
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _normalise_for_match(text: str) -> str:
|
|
176
|
+
"""Normalise text for fuzzy matching: lowercase, collapse whitespace, strip punctuation."""
|
|
177
|
+
text = text.lower()
|
|
178
|
+
text = re.sub(r"[^\w\s]", "", text)
|
|
179
|
+
text = re.sub(r"\s+", " ", text).strip()
|
|
180
|
+
return text
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _match_paper_to_db(
|
|
184
|
+
filename: str, db_papers: list[Paper],
|
|
185
|
+
) -> Paper | None:
|
|
186
|
+
"""Match a markdown filename to a BibTeX Paper entry.
|
|
187
|
+
|
|
188
|
+
Uses a two-step approach:
|
|
189
|
+
1. Extract year from filename and filter candidates.
|
|
190
|
+
2. Fuzzy-match the title from the filename against candidate titles.
|
|
191
|
+
|
|
192
|
+
Returns the best matching Paper, or None if no match found.
|
|
193
|
+
"""
|
|
194
|
+
m = _FILENAME_RE.match(filename)
|
|
195
|
+
if not m:
|
|
196
|
+
return None
|
|
197
|
+
|
|
198
|
+
file_year = int(m.group("year"))
|
|
199
|
+
file_title = _normalise_for_match(m.group("title"))
|
|
200
|
+
file_authors = _normalise_for_match(m.group("authors"))
|
|
201
|
+
|
|
202
|
+
# Filter by year (allow ±1 year tolerance for pre-prints)
|
|
203
|
+
candidates = [p for p in db_papers if abs(p.year - file_year) <= 1]
|
|
204
|
+
if not candidates:
|
|
205
|
+
return None
|
|
206
|
+
|
|
207
|
+
# Score each candidate by title word overlap
|
|
208
|
+
best_score = 0.0
|
|
209
|
+
best_paper: Paper | None = None
|
|
210
|
+
|
|
211
|
+
file_title_words = set(file_title.split())
|
|
212
|
+
file_author_words = set(file_authors.split())
|
|
213
|
+
|
|
214
|
+
for paper in candidates:
|
|
215
|
+
paper_title = _normalise_for_match(paper.title)
|
|
216
|
+
paper_title_words = set(paper_title.split())
|
|
217
|
+
|
|
218
|
+
if not paper_title_words:
|
|
219
|
+
continue
|
|
220
|
+
|
|
221
|
+
# Jaccard similarity on title words
|
|
222
|
+
intersection = file_title_words & paper_title_words
|
|
223
|
+
union = file_title_words | paper_title_words
|
|
224
|
+
title_score = len(intersection) / len(union) if union else 0.0
|
|
225
|
+
|
|
226
|
+
# Bonus for author match
|
|
227
|
+
paper_author_norm = _normalise_for_match(paper.authors)
|
|
228
|
+
paper_author_words = set(paper_author_norm.split())
|
|
229
|
+
author_overlap = len(file_author_words & paper_author_words)
|
|
230
|
+
author_bonus = min(0.2, author_overlap * 0.05)
|
|
231
|
+
|
|
232
|
+
score = title_score + author_bonus
|
|
233
|
+
|
|
234
|
+
if score > best_score:
|
|
235
|
+
best_score = score
|
|
236
|
+
best_paper = paper
|
|
237
|
+
|
|
238
|
+
# Require at least 30% title overlap
|
|
239
|
+
if best_score < 0.3:
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
return best_paper
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _paper_from_filename(filename: str) -> Paper | None:
|
|
246
|
+
"""Create fallback paper metadata from a Zotero-style markdown filename."""
|
|
247
|
+
from metacouplingllm.knowledge.literature import Paper
|
|
248
|
+
|
|
249
|
+
m = _FILENAME_RE.match(filename)
|
|
250
|
+
if m:
|
|
251
|
+
title = re.sub(r"\s+", " ", m.group("title")).strip()
|
|
252
|
+
authors = re.sub(r"\s+", " ", m.group("authors")).strip()
|
|
253
|
+
try:
|
|
254
|
+
year = int(m.group("year"))
|
|
255
|
+
except ValueError:
|
|
256
|
+
year = 0
|
|
257
|
+
else:
|
|
258
|
+
stem = Path(filename).stem
|
|
259
|
+
year_match = re.search(r"\b((?:18|19|20)\d{2})\b", stem)
|
|
260
|
+
year = int(year_match.group(1)) if year_match else 0
|
|
261
|
+
if " - " in stem:
|
|
262
|
+
authors, title = stem.split(" - ", 1)
|
|
263
|
+
elif "_" in stem:
|
|
264
|
+
authors, title = stem.split("_", 1)
|
|
265
|
+
else:
|
|
266
|
+
authors, title = "Unknown", stem
|
|
267
|
+
authors = re.sub(r"\s+", " ", authors).strip() or "Unknown"
|
|
268
|
+
title = re.sub(r"[_\s]+", " ", title).strip() or stem
|
|
269
|
+
if not title:
|
|
270
|
+
return None
|
|
271
|
+
if not title:
|
|
272
|
+
return None
|
|
273
|
+
if not authors:
|
|
274
|
+
authors = "Unknown"
|
|
275
|
+
if not isinstance(year, int):
|
|
276
|
+
year = 0
|
|
277
|
+
key_source = f"{authors}_{year}_{title}".lower()
|
|
278
|
+
key = re.sub(r"[^a-z0-9]+", "_", key_source).strip("_")[:80]
|
|
279
|
+
return Paper(
|
|
280
|
+
key=key or _normalise_for_match(filename).replace(" ", "_")[:80],
|
|
281
|
+
title=title,
|
|
282
|
+
authors=authors,
|
|
283
|
+
year=year,
|
|
284
|
+
keywords={
|
|
285
|
+
"telecoupling",
|
|
286
|
+
"metacoupling",
|
|
287
|
+
"rag-fallback-metadata",
|
|
288
|
+
},
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
# ---------------------------------------------------------------------------
|
|
293
|
+
# Text chunking
|
|
294
|
+
# ---------------------------------------------------------------------------
|
|
295
|
+
|
|
296
|
+
# Section heading patterns in markdown
|
|
297
|
+
_HEADING_RE = re.compile(r"^#{1,3}\s+(.+)$", re.MULTILINE)
|
|
298
|
+
_REFERENCE_SECTION_RE = re.compile(
|
|
299
|
+
r"(?im)^(?:#{1,6}\s*)?"
|
|
300
|
+
r"(references|bibliography|literature cited)\s*$"
|
|
301
|
+
)
|
|
302
|
+
_BIBLIOGRAPHY_HEADING_RE = re.compile(
|
|
303
|
+
r"^(?:(?:18|19|20)\d{2}|\d{3,})\."
|
|
304
|
+
)
|
|
305
|
+
_REFERENCE_LEAD_RE = re.compile(
|
|
306
|
+
r"^(?:(?:[A-Z]\.\s*)?(?:18|19|20)\d{2}\.|"
|
|
307
|
+
r"\d{3,}\.\s+[A-Z])"
|
|
308
|
+
)
|
|
309
|
+
_REFERENCE_AUTHOR_RE = re.compile(
|
|
310
|
+
r"\b[A-Z][A-Za-z'`.-]+,\s*(?:[A-Z]\.\s*){1,3}"
|
|
311
|
+
)
|
|
312
|
+
_REFERENCE_YEAR_RE = re.compile(r"(?:\(|\b)(?:18|19|20)\d{2}(?:\)|\.)")
|
|
313
|
+
_REFERENCE_TERM_RE = re.compile(
|
|
314
|
+
r"\b(?:doi|proceedings|thesis|dissertation|editors?|"
|
|
315
|
+
r"university|press|journal|rome|fao|pages?)\b",
|
|
316
|
+
re.IGNORECASE,
|
|
317
|
+
)
|
|
318
|
+
_REFERENCE_URL_RE = re.compile(r"https?://|doi[:/]", re.IGNORECASE)
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
def _truncate_at_references(text: str) -> str:
|
|
322
|
+
"""Drop everything from the first references-style section onward."""
|
|
323
|
+
match = _REFERENCE_SECTION_RE.search(text)
|
|
324
|
+
if not match:
|
|
325
|
+
return text
|
|
326
|
+
return text[:match.start()].rstrip()
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _is_reference_heading(heading: str) -> bool:
|
|
330
|
+
"""Return True when *heading* looks like bibliography content."""
|
|
331
|
+
heading = heading.strip()
|
|
332
|
+
heading_lower = heading.lower()
|
|
333
|
+
if heading_lower in (
|
|
334
|
+
"references", "bibliography", "literature cited",
|
|
335
|
+
"acknowledgments", "acknowledgements", "appendix",
|
|
336
|
+
"supplementary material", "supporting information",
|
|
337
|
+
):
|
|
338
|
+
return True
|
|
339
|
+
return bool(_BIBLIOGRAPHY_HEADING_RE.match(heading))
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _looks_like_reference_chunk(text: str) -> bool:
|
|
343
|
+
"""Heuristically reject chunks that read like bibliography entries."""
|
|
344
|
+
compact = re.sub(r"\s+", " ", text).strip()
|
|
345
|
+
if not compact:
|
|
346
|
+
return False
|
|
347
|
+
|
|
348
|
+
year_count = len(_REFERENCE_YEAR_RE.findall(compact))
|
|
349
|
+
author_count = len(_REFERENCE_AUTHOR_RE.findall(compact))
|
|
350
|
+
term_count = len(_REFERENCE_TERM_RE.findall(compact))
|
|
351
|
+
has_url = bool(_REFERENCE_URL_RE.search(compact))
|
|
352
|
+
has_page_ref = bool(re.search(r"\bpages?\s+\d", compact, re.IGNORECASE))
|
|
353
|
+
has_reference_lead = bool(_REFERENCE_LEAD_RE.match(compact))
|
|
354
|
+
|
|
355
|
+
if term_count >= 3:
|
|
356
|
+
return True
|
|
357
|
+
if has_reference_lead and (term_count >= 1 or has_page_ref or year_count >= 1):
|
|
358
|
+
return True
|
|
359
|
+
if has_page_ref and term_count >= 2:
|
|
360
|
+
return True
|
|
361
|
+
if author_count >= 2 and (year_count >= 1 or term_count >= 1 or has_page_ref):
|
|
362
|
+
return True
|
|
363
|
+
if year_count >= 2 and (term_count >= 1 or has_url or has_page_ref):
|
|
364
|
+
return True
|
|
365
|
+
return False
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _chunk_markdown(
|
|
369
|
+
text: str,
|
|
370
|
+
paper: Paper,
|
|
371
|
+
max_chunk_words: int = 250,
|
|
372
|
+
overlap_words: int = 50,
|
|
373
|
+
) -> list[TextChunk]:
|
|
374
|
+
"""Split a markdown document into overlapping text chunks.
|
|
375
|
+
|
|
376
|
+
Strategy:
|
|
377
|
+
1. Split by section headings (## or ###).
|
|
378
|
+
2. Within each section, split into ~max_chunk_words chunks with overlap.
|
|
379
|
+
3. Filter out very short chunks (< 30 words) and non-text chunks.
|
|
380
|
+
"""
|
|
381
|
+
# Remove image placeholders and citation artifacts
|
|
382
|
+
text = re.sub(r"<!--\s*image\s*-->", "", text)
|
|
383
|
+
text = re.sub(r"\|\s*#\s*\|.*?\n", "", text) # table headers
|
|
384
|
+
text = _truncate_at_references(text)
|
|
385
|
+
|
|
386
|
+
# Split into sections by headings
|
|
387
|
+
sections: list[tuple[str, str]] = []
|
|
388
|
+
parts = _HEADING_RE.split(text)
|
|
389
|
+
|
|
390
|
+
# parts alternates: [pre-heading text, heading1, text1, heading2, text2, ...]
|
|
391
|
+
current_heading = "HEADER"
|
|
392
|
+
i = 0
|
|
393
|
+
if len(parts) > 0 and not _HEADING_RE.match(parts[0] if parts[0] else ""):
|
|
394
|
+
sections.append((current_heading, parts[0]))
|
|
395
|
+
i = 1
|
|
396
|
+
|
|
397
|
+
while i < len(parts) - 1:
|
|
398
|
+
heading = parts[i].strip()
|
|
399
|
+
body = parts[i + 1] if i + 1 < len(parts) else ""
|
|
400
|
+
sections.append((heading, body))
|
|
401
|
+
i += 2
|
|
402
|
+
|
|
403
|
+
chunks: list[TextChunk] = []
|
|
404
|
+
chunk_idx = 0
|
|
405
|
+
|
|
406
|
+
for heading, body in sections:
|
|
407
|
+
if _is_reference_heading(heading):
|
|
408
|
+
continue
|
|
409
|
+
|
|
410
|
+
# Clean the body text
|
|
411
|
+
body = re.sub(r"\s+", " ", body).strip()
|
|
412
|
+
if not body:
|
|
413
|
+
continue
|
|
414
|
+
|
|
415
|
+
words = body.split()
|
|
416
|
+
if len(words) < 30:
|
|
417
|
+
continue
|
|
418
|
+
|
|
419
|
+
# Split into chunks with overlap
|
|
420
|
+
start = 0
|
|
421
|
+
while start < len(words):
|
|
422
|
+
end = min(start + max_chunk_words, len(words))
|
|
423
|
+
chunk_words = words[start:end]
|
|
424
|
+
chunk_text = " ".join(chunk_words)
|
|
425
|
+
|
|
426
|
+
if len(chunk_words) >= 30 and not _looks_like_reference_chunk(
|
|
427
|
+
chunk_text
|
|
428
|
+
):
|
|
429
|
+
chunks.append(TextChunk(
|
|
430
|
+
paper_key=paper.key,
|
|
431
|
+
paper_title=paper.title,
|
|
432
|
+
authors=paper.authors,
|
|
433
|
+
year=paper.year,
|
|
434
|
+
section=heading,
|
|
435
|
+
text=chunk_text,
|
|
436
|
+
chunk_index=chunk_idx,
|
|
437
|
+
))
|
|
438
|
+
chunk_idx += 1
|
|
439
|
+
|
|
440
|
+
# Advance with overlap
|
|
441
|
+
if end >= len(words):
|
|
442
|
+
break
|
|
443
|
+
start = end - overlap_words
|
|
444
|
+
|
|
445
|
+
return chunks
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
# ---------------------------------------------------------------------------
|
|
449
|
+
# TF-IDF retrieval engine
|
|
450
|
+
# ---------------------------------------------------------------------------
|
|
451
|
+
|
|
452
|
+
# Stopwords for TF-IDF
|
|
453
|
+
_STOPWORDS = frozenset({
|
|
454
|
+
"a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for",
|
|
455
|
+
"of", "with", "by", "from", "is", "it", "as", "be", "was", "were",
|
|
456
|
+
"are", "been", "being", "has", "had", "have", "do", "does", "did",
|
|
457
|
+
"will", "would", "could", "should", "may", "might", "can", "shall",
|
|
458
|
+
"not", "no", "nor", "so", "if", "then", "than", "that", "this",
|
|
459
|
+
"these", "those", "their", "its", "our", "his", "her", "my", "your",
|
|
460
|
+
"we", "they", "he", "she", "you", "i", "me", "us", "them", "him",
|
|
461
|
+
"also", "more", "most", "much", "many", "some", "any", "all", "each",
|
|
462
|
+
"every", "both", "few", "several", "other", "another", "such", "only",
|
|
463
|
+
"just", "very", "too", "even", "still", "already", "about", "over",
|
|
464
|
+
"after", "before", "between", "through", "during", "into", "out",
|
|
465
|
+
"up", "down", "which", "who", "whom", "what", "where", "when", "how",
|
|
466
|
+
"while", "there", "here", "et", "al", "fig", "figure", "table",
|
|
467
|
+
"however", "thus", "therefore", "although", "because", "since",
|
|
468
|
+
"whether", "either", "neither", "per", "via",
|
|
469
|
+
})
|
|
470
|
+
|
|
471
|
+
_WORD_RE = re.compile(r"[a-z][a-z0-9-]+")
|
|
472
|
+
|
|
473
|
+
|
|
474
|
+
def _tokenise(text: str) -> list[str]:
|
|
475
|
+
"""Tokenise text into lowercase words, removing stopwords."""
|
|
476
|
+
words = _WORD_RE.findall(text.lower())
|
|
477
|
+
return [w for w in words if w not in _STOPWORDS and len(w) > 2]
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
_GENERIC_CITATION_TOKENS = frozenset({
|
|
481
|
+
"agent", "agents", "cause", "causes", "coupled", "coupling",
|
|
482
|
+
"effect", "effects", "flow", "flows", "framework", "human",
|
|
483
|
+
"humannature", "impact", "impacts", "information", "interaction",
|
|
484
|
+
"interactions", "intracoupling", "metacoupling", "nature",
|
|
485
|
+
"pericoupling", "receiving", "sending", "spillover", "system",
|
|
486
|
+
"systems", "telecoupling",
|
|
487
|
+
})
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def _specific_tokens(tokens: set[str] | list[str]) -> set[str]:
|
|
491
|
+
"""Return the non-generic subset of *tokens* for citation matching."""
|
|
492
|
+
return {t for t in tokens if t not in _GENERIC_CITATION_TOKENS}
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _citation_policy(section: str) -> tuple[int, float, int, int]:
|
|
496
|
+
"""Return citation thresholds for the current output section.
|
|
497
|
+
|
|
498
|
+
Returns
|
|
499
|
+
-------
|
|
500
|
+
``(min_overlap, min_ratio, min_specific_overlap, max_citations)``
|
|
501
|
+
"""
|
|
502
|
+
if section in {"CAUSES", "EFFECTS", "RESEARCH GAPS"}:
|
|
503
|
+
return 4, 0.35, 2, 2
|
|
504
|
+
return 3, 0.20, 1, 3
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def _best_excerpt(
|
|
508
|
+
text: str,
|
|
509
|
+
anchor_text: str = "",
|
|
510
|
+
max_chars: int = 300,
|
|
511
|
+
) -> str:
|
|
512
|
+
"""Extract a short excerpt centered on the most relevant passage."""
|
|
513
|
+
compact = re.sub(r"\s+", " ", text).strip()
|
|
514
|
+
if len(compact) <= max_chars:
|
|
515
|
+
return compact
|
|
516
|
+
|
|
517
|
+
anchor_tokens = _specific_tokens(set(_tokenise(anchor_text)))
|
|
518
|
+
if not anchor_tokens:
|
|
519
|
+
return compact[:max_chars].rstrip() + "..."
|
|
520
|
+
|
|
521
|
+
words = compact.split()
|
|
522
|
+
window_words = max(30, min(60, max_chars // 6))
|
|
523
|
+
step = max(10, window_words // 3)
|
|
524
|
+
best_range = (0, min(len(words), window_words))
|
|
525
|
+
best_score = (-1.0, -1.0, -1.0)
|
|
526
|
+
|
|
527
|
+
for start in range(0, len(words), step):
|
|
528
|
+
end = min(len(words), start + window_words)
|
|
529
|
+
window_text = " ".join(words[start:end])
|
|
530
|
+
window_tokens = _specific_tokens(set(_tokenise(window_text)))
|
|
531
|
+
overlap = anchor_tokens & window_tokens
|
|
532
|
+
if not overlap:
|
|
533
|
+
continue
|
|
534
|
+
|
|
535
|
+
overlap_ratio = len(overlap) / len(anchor_tokens)
|
|
536
|
+
density = len(overlap) / max(1, len(window_tokens))
|
|
537
|
+
score = (float(len(overlap)), overlap_ratio, density)
|
|
538
|
+
if score > best_score:
|
|
539
|
+
best_score = score
|
|
540
|
+
best_range = (start, end)
|
|
541
|
+
|
|
542
|
+
start, end = best_range
|
|
543
|
+
excerpt = " ".join(words[start:end]).strip()
|
|
544
|
+
if len(excerpt) > max_chars:
|
|
545
|
+
excerpt = excerpt[:max_chars].rsplit(" ", 1)[0].rstrip()
|
|
546
|
+
if start > 0:
|
|
547
|
+
excerpt = "..." + excerpt
|
|
548
|
+
if end < len(words):
|
|
549
|
+
excerpt = excerpt.rstrip() + "..."
|
|
550
|
+
return excerpt
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
# ---------------------------------------------------------------------------
|
|
554
|
+
# Embedding retrieval (default backend)
|
|
555
|
+
# ---------------------------------------------------------------------------
|
|
556
|
+
|
|
557
|
+
# Default embedding model. BGE-base improves semantic recall over BGE-small
|
|
558
|
+
# while keeping the model download at runtime rather than bundling it.
|
|
559
|
+
DEFAULT_EMBEDDING_MODEL = "BAAI/bge-base-en-v1.5"
|
|
560
|
+
DEFAULT_EMBEDDING_DIMS = 768
|
|
561
|
+
|
|
562
|
+
# Path to the pre-computed corpus embeddings bundled with the package.
|
|
563
|
+
_BUNDLED_EMBEDDINGS_PATH = (
|
|
564
|
+
Path(__file__).parent / ".." / "data" / "chunk_embeddings.npy"
|
|
565
|
+
)
|
|
566
|
+
|
|
567
|
+
# Path to the sidecar manifest that records the chunk-order fingerprint the
|
|
568
|
+
# .npy was built against. Used to detect silent chunk/embedding mismatches
|
|
569
|
+
# that can occur when the .npy is built on one platform and loaded on
|
|
570
|
+
# another — see ``compute_chunk_fingerprint`` below for rationale.
|
|
571
|
+
_BUNDLED_MANIFEST_PATH = (
|
|
572
|
+
Path(__file__).parent / ".." / "data" / "chunk_embeddings.manifest.json"
|
|
573
|
+
)
|
|
574
|
+
|
|
575
|
+
|
|
576
|
+
def compute_chunk_fingerprint(chunks: "list[TextChunk]") -> str:
|
|
577
|
+
"""Return a deterministic SHA-256 fingerprint of a chunk list's order.
|
|
578
|
+
|
|
579
|
+
The fingerprint is computed over ``paper_key:chunk_index`` pairs
|
|
580
|
+
(joined with ``|``) and is independent of chunk text content. Two
|
|
581
|
+
chunk lists produce the same fingerprint iff they reference the
|
|
582
|
+
same papers in the same order with the same per-paper chunking.
|
|
583
|
+
|
|
584
|
+
This is used by :class:`EmbeddingRetriever` to verify that the
|
|
585
|
+
runtime chunks are in the same order as the chunks that were used
|
|
586
|
+
to build ``chunk_embeddings.npy``. A mismatch means the pre-
|
|
587
|
+
computed embeddings at index ``k`` correspond to a different chunk
|
|
588
|
+
than the runtime chunk at the same index — every retrieval result
|
|
589
|
+
would silently return the wrong passage.
|
|
590
|
+
|
|
591
|
+
Parameters
|
|
592
|
+
----------
|
|
593
|
+
chunks:
|
|
594
|
+
Ordered list of :class:`TextChunk`. Normally obtained from
|
|
595
|
+
:meth:`RAGEngine.load`.
|
|
596
|
+
|
|
597
|
+
Returns
|
|
598
|
+
-------
|
|
599
|
+
Hex-digest SHA-256 string.
|
|
600
|
+
"""
|
|
601
|
+
joined = b"|".join(
|
|
602
|
+
f"{c.paper_key}:{c.chunk_index}".encode("utf-8") for c in chunks
|
|
603
|
+
)
|
|
604
|
+
return hashlib.sha256(joined).hexdigest()
|
|
605
|
+
|
|
606
|
+
|
|
607
|
+
def _load_precomputed_embeddings() -> "np.ndarray | None":
|
|
608
|
+
"""Load the pre-computed corpus embeddings shipped with the package.
|
|
609
|
+
|
|
610
|
+
Returns ``None`` when the file is missing (e.g., in development
|
|
611
|
+
before the build step has been run) so callers can fall back to
|
|
612
|
+
regenerating or using a different backend.
|
|
613
|
+
"""
|
|
614
|
+
try:
|
|
615
|
+
import numpy as np # type: ignore[import-not-found]
|
|
616
|
+
except ImportError:
|
|
617
|
+
return None
|
|
618
|
+
|
|
619
|
+
if not _BUNDLED_EMBEDDINGS_PATH.exists():
|
|
620
|
+
return None
|
|
621
|
+
try:
|
|
622
|
+
return np.load(_BUNDLED_EMBEDDINGS_PATH)
|
|
623
|
+
except Exception as exc:
|
|
624
|
+
print(f"[RAG] Failed to load pre-computed embeddings: {exc}")
|
|
625
|
+
return None
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _load_chunk_manifest() -> "dict | None":
|
|
629
|
+
"""Load the sidecar chunk-order manifest, if it exists.
|
|
630
|
+
|
|
631
|
+
Returns ``None`` when the file is missing (legacy installs that
|
|
632
|
+
shipped before the manifest was introduced — the loader warns but
|
|
633
|
+
proceeds) or when the file is malformed.
|
|
634
|
+
"""
|
|
635
|
+
if not _BUNDLED_MANIFEST_PATH.exists():
|
|
636
|
+
return None
|
|
637
|
+
try:
|
|
638
|
+
with _BUNDLED_MANIFEST_PATH.open("r", encoding="utf-8") as fh:
|
|
639
|
+
return json.load(fh)
|
|
640
|
+
except (OSError, ValueError) as exc:
|
|
641
|
+
logger.warning(
|
|
642
|
+
"Failed to read chunk-embeddings manifest at %s: %s",
|
|
643
|
+
_BUNDLED_MANIFEST_PATH.resolve(), exc,
|
|
644
|
+
)
|
|
645
|
+
return None
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
class EmbeddingRetriever:
|
|
649
|
+
"""Semantic retrieval over pre-computed corpus embeddings.
|
|
650
|
+
|
|
651
|
+
Unlike :class:`TfIdfIndex`, this retriever uses dense vector
|
|
652
|
+
embeddings from a pre-trained transformer model (via ``fastembed``
|
|
653
|
+
+ ONNX Runtime). Queries and chunks are compared via cosine
|
|
654
|
+
similarity on 768-dimensional vectors, which captures semantic
|
|
655
|
+
similarity — synonyms, paraphrases, and related concepts that
|
|
656
|
+
TF-IDF misses.
|
|
657
|
+
|
|
658
|
+
The corpus embeddings are **pre-computed and shipped with the
|
|
659
|
+
package** as a ``.npy`` file (see ``scripts/build_embeddings.py``).
|
|
660
|
+
At runtime the retriever only needs to embed the query, which
|
|
661
|
+
takes ~40 ms on CPU after a one-time ~2 s model load.
|
|
662
|
+
|
|
663
|
+
Parameters
|
|
664
|
+
----------
|
|
665
|
+
chunks:
|
|
666
|
+
List of :class:`TextChunk` objects (must be in the SAME order
|
|
667
|
+
used when the pre-computed embeddings were generated).
|
|
668
|
+
model_name:
|
|
669
|
+
Name of the embedding model. Must match the model used to
|
|
670
|
+
produce the pre-computed corpus embeddings. Defaults to
|
|
671
|
+
``"BAAI/bge-base-en-v1.5"``.
|
|
672
|
+
precomputed_embeddings:
|
|
673
|
+
Optional pre-computed ``(n_chunks, dims)`` numpy array. When
|
|
674
|
+
``None``, the retriever tries to load the bundled file. When
|
|
675
|
+
the shape doesn't match ``len(chunks)``, the embeddings are
|
|
676
|
+
discarded and a warning is printed (falling back to re-encoding
|
|
677
|
+
is caller's responsibility).
|
|
678
|
+
|
|
679
|
+
Raises
|
|
680
|
+
------
|
|
681
|
+
ImportError
|
|
682
|
+
If ``fastembed`` is not installed.
|
|
683
|
+
RuntimeError
|
|
684
|
+
If no pre-computed embeddings are available AND
|
|
685
|
+
``allow_rebuild`` is False (the default).
|
|
686
|
+
"""
|
|
687
|
+
|
|
688
|
+
def __init__(
|
|
689
|
+
self,
|
|
690
|
+
chunks: "list[TextChunk]",
|
|
691
|
+
*,
|
|
692
|
+
model_name: str = DEFAULT_EMBEDDING_MODEL,
|
|
693
|
+
precomputed_embeddings: "np.ndarray | None" = None,
|
|
694
|
+
allow_rebuild: bool = False,
|
|
695
|
+
verbose: bool = False,
|
|
696
|
+
) -> None:
|
|
697
|
+
try:
|
|
698
|
+
import numpy as np # type: ignore[import-not-found]
|
|
699
|
+
except ImportError as exc: # pragma: no cover
|
|
700
|
+
raise ImportError(
|
|
701
|
+
"numpy is required for EmbeddingRetriever. "
|
|
702
|
+
"Install with `pip install numpy`."
|
|
703
|
+
) from exc
|
|
704
|
+
|
|
705
|
+
try:
|
|
706
|
+
from fastembed import TextEmbedding # type: ignore[import-not-found]
|
|
707
|
+
except ImportError as exc:
|
|
708
|
+
raise ImportError(
|
|
709
|
+
"fastembed is required for EmbeddingRetriever. "
|
|
710
|
+
"Install with `pip install fastembed`, or use the "
|
|
711
|
+
"TF-IDF backend by passing rag_backend='tfidf' to "
|
|
712
|
+
"MetacouplingAssistant."
|
|
713
|
+
) from exc
|
|
714
|
+
|
|
715
|
+
self._chunks = chunks
|
|
716
|
+
self._n_docs = len(chunks)
|
|
717
|
+
self._model_name = model_name
|
|
718
|
+
self._verbose = verbose
|
|
719
|
+
self._np = np
|
|
720
|
+
self._TextEmbedding = TextEmbedding
|
|
721
|
+
|
|
722
|
+
# Lazily initialised embedder (first query triggers download +
|
|
723
|
+
# model load). Keeping this lazy avoids a ~2 s startup cost
|
|
724
|
+
# when the retriever is constructed but never used.
|
|
725
|
+
self._embedder: object | None = None
|
|
726
|
+
|
|
727
|
+
# Load or validate the corpus embeddings.
|
|
728
|
+
embeddings = precomputed_embeddings
|
|
729
|
+
if embeddings is None:
|
|
730
|
+
embeddings = _load_precomputed_embeddings()
|
|
731
|
+
|
|
732
|
+
if embeddings is None:
|
|
733
|
+
if not allow_rebuild:
|
|
734
|
+
raise RuntimeError(
|
|
735
|
+
"No pre-computed embeddings found at "
|
|
736
|
+
f"{_BUNDLED_EMBEDDINGS_PATH.resolve()}. Run "
|
|
737
|
+
"`python scripts/build_embeddings.py` to generate "
|
|
738
|
+
"the file, or pass allow_rebuild=True to build it "
|
|
739
|
+
"at runtime (~60 s on CPU)."
|
|
740
|
+
)
|
|
741
|
+
embeddings = self._build_corpus_embeddings()
|
|
742
|
+
|
|
743
|
+
if embeddings.shape[0] != self._n_docs:
|
|
744
|
+
raise RuntimeError(
|
|
745
|
+
f"Pre-computed embedding count ({embeddings.shape[0]}) "
|
|
746
|
+
f"does not match chunk count ({self._n_docs}). The "
|
|
747
|
+
"embeddings file is stale — re-run "
|
|
748
|
+
"`python scripts/build_embeddings.py`."
|
|
749
|
+
)
|
|
750
|
+
|
|
751
|
+
# Verify chunk-order fingerprint against the bundled manifest.
|
|
752
|
+
# The shape check above only guarantees counts match. We ALSO
|
|
753
|
+
# need to verify the chunks are in the SAME ORDER they were in
|
|
754
|
+
# when the .npy was built — otherwise embedding [k] points to
|
|
755
|
+
# the wrong chunk. This matters because Python's ``Path.__lt__``
|
|
756
|
+
# sorts case-INsensitively on Windows and case-sensitively on
|
|
757
|
+
# Linux/macOS, so building the .npy on one OS and loading it on
|
|
758
|
+
# another can silently corrupt the mapping whenever the corpus
|
|
759
|
+
# has mixed-case filenames.
|
|
760
|
+
#
|
|
761
|
+
# Only run the manifest check when the BUNDLED embeddings were
|
|
762
|
+
# loaded (precomputed_embeddings was None on entry). When a
|
|
763
|
+
# caller passes their own embeddings, the bundled manifest
|
|
764
|
+
# doesn't apply — it's the caller's responsibility to ensure
|
|
765
|
+
# the order matches their custom embeddings.
|
|
766
|
+
using_bundled = precomputed_embeddings is None
|
|
767
|
+
manifest = _load_chunk_manifest() if using_bundled else None
|
|
768
|
+
if manifest is not None:
|
|
769
|
+
stored_hash = manifest.get("paper_key_order_sha256", "")
|
|
770
|
+
runtime_hash = compute_chunk_fingerprint(chunks)
|
|
771
|
+
if stored_hash and stored_hash != runtime_hash:
|
|
772
|
+
raise RuntimeError(
|
|
773
|
+
"Chunk-order fingerprint mismatch between the "
|
|
774
|
+
"pre-computed embeddings and the chunks loaded at "
|
|
775
|
+
"runtime. The chunks in chunk_embeddings.npy were "
|
|
776
|
+
"built in a different order than the one produced "
|
|
777
|
+
"by RAGEngine.load() on this machine — every "
|
|
778
|
+
"retrieval result would silently return the wrong "
|
|
779
|
+
"passage. This usually happens when the package "
|
|
780
|
+
"was built on one OS and installed on another, or "
|
|
781
|
+
"when Papers.zip was updated without rebuilding "
|
|
782
|
+
"the embeddings.\n\n"
|
|
783
|
+
"Fix: run `python scripts/build_embeddings.py "
|
|
784
|
+
"--force` on this machine to regenerate the .npy "
|
|
785
|
+
"and manifest, or install a newer package version.\n\n"
|
|
786
|
+
f" expected (manifest): {stored_hash[:16]}...\n"
|
|
787
|
+
f" actual (runtime): {runtime_hash[:16]}..."
|
|
788
|
+
)
|
|
789
|
+
elif self._verbose:
|
|
790
|
+
# Legacy install without a manifest — warn but proceed.
|
|
791
|
+
logger.warning(
|
|
792
|
+
"No chunk-embeddings manifest found at %s. Cannot "
|
|
793
|
+
"verify chunk-order integrity; retrieval may silently "
|
|
794
|
+
"return wrong passages if the .npy was built on a "
|
|
795
|
+
"different platform. Consider re-running "
|
|
796
|
+
"`python scripts/build_embeddings.py --force`.",
|
|
797
|
+
_BUNDLED_MANIFEST_PATH.resolve(),
|
|
798
|
+
)
|
|
799
|
+
|
|
800
|
+
# L2-normalize for cosine similarity via dot product. Bundled
|
|
801
|
+
# embeddings may be stored as float16 to keep the wheel small;
|
|
802
|
+
# scoring is always done in float32.
|
|
803
|
+
embeddings = np.asarray(embeddings, dtype=np.float32)
|
|
804
|
+
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
|
|
805
|
+
norms = np.where(norms == 0, 1.0, norms)
|
|
806
|
+
self._chunk_vecs = (embeddings / norms).astype(np.float32)
|
|
807
|
+
|
|
808
|
+
# ------------------------------------------------------------------
|
|
809
|
+
# Internal helpers
|
|
810
|
+
# ------------------------------------------------------------------
|
|
811
|
+
|
|
812
|
+
def _ensure_embedder(self) -> object:
|
|
813
|
+
"""Lazily initialise the fastembed encoder."""
|
|
814
|
+
if self._embedder is None:
|
|
815
|
+
if self._verbose:
|
|
816
|
+
print(
|
|
817
|
+
f"[RAG] Loading embedding model "
|
|
818
|
+
f"{self._model_name} (first use may take ~10 s "
|
|
819
|
+
"while the ONNX model is downloaded/cached)..."
|
|
820
|
+
)
|
|
821
|
+
self._embedder = self._TextEmbedding(
|
|
822
|
+
model_name=self._model_name,
|
|
823
|
+
)
|
|
824
|
+
return self._embedder
|
|
825
|
+
|
|
826
|
+
def _build_corpus_embeddings(self) -> "np.ndarray":
|
|
827
|
+
"""Encode the chunks with the active model (runtime fallback).
|
|
828
|
+
|
|
829
|
+
Only called when the pre-computed file is missing AND the user
|
|
830
|
+
opts in via ``allow_rebuild=True``. Prefer pre-computing once
|
|
831
|
+
and shipping the .npy file with the package.
|
|
832
|
+
"""
|
|
833
|
+
embedder = self._ensure_embedder()
|
|
834
|
+
texts = [chunk.text for chunk in self._chunks]
|
|
835
|
+
if self._verbose:
|
|
836
|
+
print(
|
|
837
|
+
f"[RAG] Encoding {len(texts)} chunks with "
|
|
838
|
+
f"{self._model_name} (this takes ~60 s on CPU)..."
|
|
839
|
+
)
|
|
840
|
+
vectors = self._np.array(
|
|
841
|
+
list(embedder.embed(texts)),
|
|
842
|
+
dtype=self._np.float32,
|
|
843
|
+
)
|
|
844
|
+
return vectors
|
|
845
|
+
|
|
846
|
+
# ------------------------------------------------------------------
|
|
847
|
+
# Public API
|
|
848
|
+
# ------------------------------------------------------------------
|
|
849
|
+
|
|
850
|
+
def query(
|
|
851
|
+
self,
|
|
852
|
+
query_text: str,
|
|
853
|
+
top_k: int = 5,
|
|
854
|
+
min_score: float = 0.3,
|
|
855
|
+
max_chunks_per_paper: int = 3,
|
|
856
|
+
) -> list[RetrievalResult]:
|
|
857
|
+
"""Retrieve the top-k most relevant chunks for a query.
|
|
858
|
+
|
|
859
|
+
Parameters
|
|
860
|
+
----------
|
|
861
|
+
query_text:
|
|
862
|
+
The search query text.
|
|
863
|
+
top_k:
|
|
864
|
+
Maximum number of results (chunks) to return. Note: this
|
|
865
|
+
counts individual chunks, not unique papers — multiple
|
|
866
|
+
chunks from the same paper may appear when that paper's
|
|
867
|
+
most relevant content is spread across different sections.
|
|
868
|
+
min_score:
|
|
869
|
+
Minimum cosine similarity threshold. BGE-small cosine
|
|
870
|
+
similarity typically ranges 0.3–0.9 for relevant results,
|
|
871
|
+
so this default is stricter than the TF-IDF default.
|
|
872
|
+
max_chunks_per_paper:
|
|
873
|
+
Maximum number of chunks from the same paper that can
|
|
874
|
+
appear in the result set. Default ``3`` lets strongly-
|
|
875
|
+
matching papers contribute several high-scoring chunks
|
|
876
|
+
rather than a single chunk that may not cover all of the
|
|
877
|
+
paper's relevant content. Set to ``1`` for the legacy
|
|
878
|
+
one-chunk-per-paper behavior, or raise to ``5+`` when a
|
|
879
|
+
single paper is expected to be highly relevant and its
|
|
880
|
+
evidence is spread across many sections.
|
|
881
|
+
|
|
882
|
+
Returns
|
|
883
|
+
-------
|
|
884
|
+
List of :class:`RetrievalResult` sorted by score descending,
|
|
885
|
+
with at most ``max_chunks_per_paper`` chunks from any single
|
|
886
|
+
paper.
|
|
887
|
+
"""
|
|
888
|
+
if not self._chunks:
|
|
889
|
+
return []
|
|
890
|
+
if not query_text or not query_text.strip():
|
|
891
|
+
return []
|
|
892
|
+
if max_chunks_per_paper < 1:
|
|
893
|
+
max_chunks_per_paper = 1
|
|
894
|
+
|
|
895
|
+
embedder = self._ensure_embedder()
|
|
896
|
+
query_vec = next(iter(embedder.embed([query_text])))
|
|
897
|
+
query_vec = self._np.asarray(query_vec, dtype=self._np.float32)
|
|
898
|
+
q_norm = float(self._np.linalg.norm(query_vec))
|
|
899
|
+
if q_norm == 0.0:
|
|
900
|
+
return []
|
|
901
|
+
query_vec = query_vec / q_norm
|
|
902
|
+
|
|
903
|
+
# Cosine similarity = dot product of L2-normalized vectors.
|
|
904
|
+
scores = self._chunk_vecs @ query_vec # shape (n_chunks,)
|
|
905
|
+
|
|
906
|
+
# Two-pass selection:
|
|
907
|
+
# Pass 1 prefers chunks from sections we have NOT yet seen
|
|
908
|
+
# for a given paper — this buys section diversity first
|
|
909
|
+
# (Introduction + Results + Discussion when all score highly).
|
|
910
|
+
# Chunks from sections we've already seen get deferred.
|
|
911
|
+
# Pass 2 fills any remaining slots (per-paper cap and
|
|
912
|
+
# top_k) from the deferred list — so long sections that
|
|
913
|
+
# the chunker split into several sub-topic chunks (e.g.,
|
|
914
|
+
# "4. Results" with inbound vs outbound sub-parts) can
|
|
915
|
+
# still contribute multiple chunks when the paper has
|
|
916
|
+
# budget to spare.
|
|
917
|
+
order = self._np.argsort(-scores)
|
|
918
|
+
deduped: list[RetrievalResult] = []
|
|
919
|
+
paper_counts: dict[str, int] = {}
|
|
920
|
+
paper_sections_seen: dict[str, set[str]] = {}
|
|
921
|
+
deferred: list[tuple[int, float]] = []
|
|
922
|
+
for idx in order:
|
|
923
|
+
score = float(scores[idx])
|
|
924
|
+
if score < min_score:
|
|
925
|
+
break
|
|
926
|
+
chunk = self._chunks[int(idx)]
|
|
927
|
+
paper_key = chunk.paper_key
|
|
928
|
+
if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
|
|
929
|
+
continue
|
|
930
|
+
sections = paper_sections_seen.setdefault(paper_key, set())
|
|
931
|
+
if chunk.section in sections:
|
|
932
|
+
deferred.append((int(idx), score))
|
|
933
|
+
continue
|
|
934
|
+
sections.add(chunk.section)
|
|
935
|
+
paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
|
|
936
|
+
deduped.append(RetrievalResult(chunk=chunk, score=score))
|
|
937
|
+
if len(deduped) >= top_k:
|
|
938
|
+
return deduped
|
|
939
|
+
# Pass 2 — use remaining per-paper budget on deferred chunks,
|
|
940
|
+
# in score-desc order (``deferred`` is already score-desc
|
|
941
|
+
# because ``order`` was).
|
|
942
|
+
for idx, score in deferred:
|
|
943
|
+
chunk = self._chunks[int(idx)]
|
|
944
|
+
paper_key = chunk.paper_key
|
|
945
|
+
if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
|
|
946
|
+
continue
|
|
947
|
+
paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
|
|
948
|
+
deduped.append(RetrievalResult(chunk=chunk, score=score))
|
|
949
|
+
if len(deduped) >= top_k:
|
|
950
|
+
break
|
|
951
|
+
return deduped
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
class TfIdfIndex:
|
|
955
|
+
"""In-memory TF-IDF index for text chunks.
|
|
956
|
+
|
|
957
|
+
Parameters
|
|
958
|
+
----------
|
|
959
|
+
chunks:
|
|
960
|
+
List of :class:`TextChunk` objects to index.
|
|
961
|
+
"""
|
|
962
|
+
|
|
963
|
+
def __init__(self, chunks: list[TextChunk]) -> None:
|
|
964
|
+
self._chunks = chunks
|
|
965
|
+
self._n_docs = len(chunks)
|
|
966
|
+
self._vocab: dict[str, int] = {} # word → doc-frequency
|
|
967
|
+
self._tf_vectors: list[dict[str, float]] = []
|
|
968
|
+
|
|
969
|
+
self._build_index()
|
|
970
|
+
|
|
971
|
+
def _build_index(self) -> None:
|
|
972
|
+
"""Compute TF and DF for all chunks."""
|
|
973
|
+
doc_freq: Counter[str] = Counter()
|
|
974
|
+
tf_raw: list[Counter[str]] = []
|
|
975
|
+
|
|
976
|
+
for chunk in self._chunks:
|
|
977
|
+
tokens = _tokenise(chunk.text)
|
|
978
|
+
tf = Counter(tokens)
|
|
979
|
+
tf_raw.append(tf)
|
|
980
|
+
# Document frequency: count each term once per document
|
|
981
|
+
doc_freq.update(tf.keys())
|
|
982
|
+
|
|
983
|
+
self._vocab = dict(doc_freq)
|
|
984
|
+
|
|
985
|
+
# Normalise TF to TF log-normalised: 1 + log(tf)
|
|
986
|
+
for tf in tf_raw:
|
|
987
|
+
total = sum(tf.values()) or 1
|
|
988
|
+
normed: dict[str, float] = {}
|
|
989
|
+
for word, count in tf.items():
|
|
990
|
+
normed[word] = (1 + math.log(count)) / total
|
|
991
|
+
self._tf_vectors.append(normed)
|
|
992
|
+
|
|
993
|
+
def query(
|
|
994
|
+
self,
|
|
995
|
+
query_text: str,
|
|
996
|
+
top_k: int = 5,
|
|
997
|
+
min_score: float = 0.01,
|
|
998
|
+
max_chunks_per_paper: int = 3,
|
|
999
|
+
) -> list[RetrievalResult]:
|
|
1000
|
+
"""Retrieve the top-k most relevant chunks for a query.
|
|
1001
|
+
|
|
1002
|
+
Parameters
|
|
1003
|
+
----------
|
|
1004
|
+
query_text:
|
|
1005
|
+
The search query text.
|
|
1006
|
+
top_k:
|
|
1007
|
+
Maximum number of results (chunks) to return. Note: this
|
|
1008
|
+
counts individual chunks, not unique papers.
|
|
1009
|
+
min_score:
|
|
1010
|
+
Minimum cosine similarity threshold.
|
|
1011
|
+
max_chunks_per_paper:
|
|
1012
|
+
Maximum number of chunks from the same paper that can
|
|
1013
|
+
appear in the result set. Default ``3``. Set to ``1``
|
|
1014
|
+
for the legacy one-chunk-per-paper behavior.
|
|
1015
|
+
|
|
1016
|
+
Returns
|
|
1017
|
+
-------
|
|
1018
|
+
List of :class:`RetrievalResult` sorted by score descending,
|
|
1019
|
+
with at most ``max_chunks_per_paper`` chunks per paper.
|
|
1020
|
+
"""
|
|
1021
|
+
if not self._chunks:
|
|
1022
|
+
return []
|
|
1023
|
+
if max_chunks_per_paper < 1:
|
|
1024
|
+
max_chunks_per_paper = 1
|
|
1025
|
+
|
|
1026
|
+
query_tokens = _tokenise(query_text)
|
|
1027
|
+
if not query_tokens:
|
|
1028
|
+
return []
|
|
1029
|
+
|
|
1030
|
+
# Build query TF-IDF vector
|
|
1031
|
+
query_tf = Counter(query_tokens)
|
|
1032
|
+
query_vec: dict[str, float] = {}
|
|
1033
|
+
for word, count in query_tf.items():
|
|
1034
|
+
df = self._vocab.get(word, 0)
|
|
1035
|
+
if df == 0:
|
|
1036
|
+
continue
|
|
1037
|
+
idf = math.log((self._n_docs + 1) / (df + 1)) + 1
|
|
1038
|
+
query_vec[word] = (1 + math.log(count)) * idf
|
|
1039
|
+
|
|
1040
|
+
if not query_vec:
|
|
1041
|
+
return []
|
|
1042
|
+
|
|
1043
|
+
# Compute query vector magnitude
|
|
1044
|
+
q_mag = math.sqrt(sum(v * v for v in query_vec.values()))
|
|
1045
|
+
if q_mag == 0:
|
|
1046
|
+
return []
|
|
1047
|
+
|
|
1048
|
+
# Score each document
|
|
1049
|
+
results: list[tuple[float, int]] = []
|
|
1050
|
+
for idx, doc_tf in enumerate(self._tf_vectors):
|
|
1051
|
+
dot = 0.0
|
|
1052
|
+
for word, q_val in query_vec.items():
|
|
1053
|
+
d_val = doc_tf.get(word, 0.0)
|
|
1054
|
+
if d_val > 0:
|
|
1055
|
+
df = self._vocab.get(word, 0)
|
|
1056
|
+
idf = math.log((self._n_docs + 1) / (df + 1)) + 1
|
|
1057
|
+
dot += q_val * d_val * idf
|
|
1058
|
+
|
|
1059
|
+
# Document magnitude (approximation — use cached if needed)
|
|
1060
|
+
d_mag = math.sqrt(
|
|
1061
|
+
sum(
|
|
1062
|
+
(v * (math.log((self._n_docs + 1) / (self._vocab.get(w, 1) + 1)) + 1)) ** 2
|
|
1063
|
+
for w, v in doc_tf.items()
|
|
1064
|
+
)
|
|
1065
|
+
)
|
|
1066
|
+
if d_mag == 0:
|
|
1067
|
+
continue
|
|
1068
|
+
|
|
1069
|
+
cosine = dot / (q_mag * d_mag)
|
|
1070
|
+
if cosine >= min_score:
|
|
1071
|
+
results.append((cosine, idx))
|
|
1072
|
+
|
|
1073
|
+
# Sort by score descending
|
|
1074
|
+
results.sort(key=lambda x: x[0], reverse=True)
|
|
1075
|
+
|
|
1076
|
+
# Two-pass selection (see EmbeddingRetriever.query() for the
|
|
1077
|
+
# rationale): Pass 1 prefers chunks from distinct sections per
|
|
1078
|
+
# paper; Pass 2 fills remaining per-paper budget with deferred
|
|
1079
|
+
# chunks from sections we already took from, so long sections
|
|
1080
|
+
# split into several sub-topic chunks can still contribute
|
|
1081
|
+
# multiple chunks when budget allows.
|
|
1082
|
+
paper_counts: dict[str, int] = {}
|
|
1083
|
+
paper_sections_seen: dict[str, set[str]] = {}
|
|
1084
|
+
deferred: list[tuple[float, int]] = []
|
|
1085
|
+
deduped: list[RetrievalResult] = []
|
|
1086
|
+
for score, idx in results:
|
|
1087
|
+
chunk = self._chunks[idx]
|
|
1088
|
+
paper_key = chunk.paper_key
|
|
1089
|
+
if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
|
|
1090
|
+
continue
|
|
1091
|
+
sections = paper_sections_seen.setdefault(paper_key, set())
|
|
1092
|
+
if chunk.section in sections:
|
|
1093
|
+
deferred.append((score, idx))
|
|
1094
|
+
continue
|
|
1095
|
+
sections.add(chunk.section)
|
|
1096
|
+
paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
|
|
1097
|
+
deduped.append(RetrievalResult(chunk=chunk, score=score))
|
|
1098
|
+
if len(deduped) >= top_k:
|
|
1099
|
+
return deduped
|
|
1100
|
+
for score, idx in deferred:
|
|
1101
|
+
chunk = self._chunks[idx]
|
|
1102
|
+
paper_key = chunk.paper_key
|
|
1103
|
+
if paper_counts.get(paper_key, 0) >= max_chunks_per_paper:
|
|
1104
|
+
continue
|
|
1105
|
+
paper_counts[paper_key] = paper_counts.get(paper_key, 0) + 1
|
|
1106
|
+
deduped.append(RetrievalResult(chunk=chunk, score=score))
|
|
1107
|
+
if len(deduped) >= top_k:
|
|
1108
|
+
break
|
|
1109
|
+
|
|
1110
|
+
return deduped
|
|
1111
|
+
|
|
1112
|
+
|
|
1113
|
+
# ---------------------------------------------------------------------------
|
|
1114
|
+
# RAG Engine — high-level API
|
|
1115
|
+
# ---------------------------------------------------------------------------
|
|
1116
|
+
|
|
1117
|
+
|
|
1118
|
+
class RAGEngine:
|
|
1119
|
+
"""Retrieval-Augmented Generation engine for metacoupling papers.
|
|
1120
|
+
|
|
1121
|
+
Loads full-text papers from a directory, matches them against the
|
|
1122
|
+
BibTeX literature database, chunks them, and builds a retrieval
|
|
1123
|
+
index.
|
|
1124
|
+
|
|
1125
|
+
Two retrieval backends are available:
|
|
1126
|
+
|
|
1127
|
+
- ``"embeddings"`` (default): semantic retrieval using fastembed +
|
|
1128
|
+
pre-computed corpus embeddings bundled with the package. Catches
|
|
1129
|
+
synonyms, paraphrases, and related concepts.
|
|
1130
|
+
- ``"tfidf"``: lexical retrieval using TF-IDF + cosine similarity.
|
|
1131
|
+
Fallback when fastembed is unavailable.
|
|
1132
|
+
- ``"auto"``: try embeddings first, fall back to TF-IDF on failure.
|
|
1133
|
+
|
|
1134
|
+
Parameters
|
|
1135
|
+
----------
|
|
1136
|
+
papers_dir:
|
|
1137
|
+
Path to directory containing full-text markdown papers.
|
|
1138
|
+
max_chunk_words:
|
|
1139
|
+
Maximum words per text chunk.
|
|
1140
|
+
overlap_words:
|
|
1141
|
+
Number of overlapping words between consecutive chunks.
|
|
1142
|
+
verbose:
|
|
1143
|
+
Print progress information during loading.
|
|
1144
|
+
backend:
|
|
1145
|
+
Retrieval backend: ``"embeddings"`` (default), ``"tfidf"``,
|
|
1146
|
+
or ``"auto"``. ``"auto"`` tries embeddings first and
|
|
1147
|
+
transparently falls back to TF-IDF if fastembed or the
|
|
1148
|
+
pre-computed embedding file are missing.
|
|
1149
|
+
"""
|
|
1150
|
+
|
|
1151
|
+
def __init__(
|
|
1152
|
+
self,
|
|
1153
|
+
papers_dir: str | Path,
|
|
1154
|
+
max_chunk_words: int = 250,
|
|
1155
|
+
overlap_words: int = 50,
|
|
1156
|
+
verbose: bool = False,
|
|
1157
|
+
backend: str = "auto",
|
|
1158
|
+
) -> None:
|
|
1159
|
+
self._papers_dir = Path(papers_dir)
|
|
1160
|
+
self._max_chunk_words = max_chunk_words
|
|
1161
|
+
self._overlap_words = overlap_words
|
|
1162
|
+
self._verbose = verbose
|
|
1163
|
+
if backend not in ("embeddings", "tfidf", "auto"):
|
|
1164
|
+
raise ValueError(
|
|
1165
|
+
f"Unknown backend: {backend!r}. "
|
|
1166
|
+
"Expected 'embeddings', 'tfidf', or 'auto'."
|
|
1167
|
+
)
|
|
1168
|
+
self._backend = backend
|
|
1169
|
+
self._active_backend: str | None = None
|
|
1170
|
+
self._index: TfIdfIndex | EmbeddingRetriever | None = None
|
|
1171
|
+
self._matched_count: int = 0
|
|
1172
|
+
self._total_chunks: int = 0
|
|
1173
|
+
self._total_files: int = 0
|
|
1174
|
+
|
|
1175
|
+
@property
|
|
1176
|
+
def is_loaded(self) -> bool:
|
|
1177
|
+
"""Return True if the index has been built."""
|
|
1178
|
+
return self._index is not None
|
|
1179
|
+
|
|
1180
|
+
@property
|
|
1181
|
+
def backend(self) -> str | None:
|
|
1182
|
+
"""Return the name of the active retrieval backend.
|
|
1183
|
+
|
|
1184
|
+
One of ``"embeddings"``, ``"tfidf"``, or ``None`` if
|
|
1185
|
+
:meth:`load` has not been called yet.
|
|
1186
|
+
"""
|
|
1187
|
+
return self._active_backend
|
|
1188
|
+
|
|
1189
|
+
@property
|
|
1190
|
+
def matched_papers(self) -> int:
|
|
1191
|
+
"""Number of papers matched to the literature database."""
|
|
1192
|
+
return self._matched_count
|
|
1193
|
+
|
|
1194
|
+
@property
|
|
1195
|
+
def total_chunks(self) -> int:
|
|
1196
|
+
"""Total number of text chunks in the index."""
|
|
1197
|
+
return self._total_chunks
|
|
1198
|
+
|
|
1199
|
+
@property
|
|
1200
|
+
def total_files(self) -> int:
|
|
1201
|
+
"""Total number of markdown files found."""
|
|
1202
|
+
return self._total_files
|
|
1203
|
+
|
|
1204
|
+
def load(self) -> None:
|
|
1205
|
+
"""Load papers, match to database, chunk, and build index.
|
|
1206
|
+
|
|
1207
|
+
This should be called once before querying. It may take a few
|
|
1208
|
+
seconds depending on the number of papers.
|
|
1209
|
+
|
|
1210
|
+
If the configured ``papers_dir`` does not exist, the engine
|
|
1211
|
+
automatically falls back to the bundled ``data/Papers.zip``
|
|
1212
|
+
shipped with the package.
|
|
1213
|
+
"""
|
|
1214
|
+
from metacouplingllm.knowledge.literature import _get_database
|
|
1215
|
+
|
|
1216
|
+
# Validate papers directory exists; fall back to bundled zip
|
|
1217
|
+
if not self._papers_dir.exists() or not self._papers_dir.is_dir():
|
|
1218
|
+
self._papers_dir = _extract_bundled_papers(self._verbose)
|
|
1219
|
+
|
|
1220
|
+
if not self._papers_dir.exists() or not self._papers_dir.is_dir():
|
|
1221
|
+
msg = (
|
|
1222
|
+
f"[RAG] Papers directory does not exist: "
|
|
1223
|
+
f"{self._papers_dir.resolve()}"
|
|
1224
|
+
)
|
|
1225
|
+
print(msg)
|
|
1226
|
+
self._index = TfIdfIndex([])
|
|
1227
|
+
return
|
|
1228
|
+
|
|
1229
|
+
db_papers = _get_database()
|
|
1230
|
+
if not db_papers:
|
|
1231
|
+
print("[RAG] WARNING: No literature database loaded. "
|
|
1232
|
+
"RAG will not produce evidence.")
|
|
1233
|
+
self._index = TfIdfIndex([])
|
|
1234
|
+
return
|
|
1235
|
+
|
|
1236
|
+
# Collect markdown files.
|
|
1237
|
+
#
|
|
1238
|
+
# IMPORTANT: sort by the filename STRING, not by the Path object.
|
|
1239
|
+
# Python's ``Path.__lt__`` is platform-dependent: on Windows it
|
|
1240
|
+
# uses case-INsensitive comparison (via ``_str_normcase``), but
|
|
1241
|
+
# on Linux/macOS it uses case-sensitive Unicode codepoint order.
|
|
1242
|
+
# Sorting ``Path`` objects directly would produce a different
|
|
1243
|
+
# chunk order on Windows vs Linux for any corpus that contains
|
|
1244
|
+
# filenames with mixed leading case (e.g., "da Silva..." vs
|
|
1245
|
+
# "Dandan..."). That in turn corrupts the chunks-to-embeddings
|
|
1246
|
+
# mapping for the pre-computed ``chunk_embeddings.npy`` file
|
|
1247
|
+
# whenever it's built on one OS and loaded on another. Using
|
|
1248
|
+
# ``key=lambda p: p.name`` forces pure-string (case-sensitive)
|
|
1249
|
+
# comparison on every platform, so the chunk order is
|
|
1250
|
+
# deterministic across OSes. The manifest integrity check in
|
|
1251
|
+
# ``EmbeddingRetriever.__init__`` catches mismatches if this
|
|
1252
|
+
# ever drifts again.
|
|
1253
|
+
md_files = sorted(
|
|
1254
|
+
self._papers_dir.glob("*.md"), key=lambda p: p.name,
|
|
1255
|
+
)
|
|
1256
|
+
self._total_files = len(md_files)
|
|
1257
|
+
|
|
1258
|
+
if self._total_files == 0:
|
|
1259
|
+
print(
|
|
1260
|
+
f"[RAG] WARNING: No .md files found in "
|
|
1261
|
+
f"{self._papers_dir.resolve()}"
|
|
1262
|
+
)
|
|
1263
|
+
self._index = TfIdfIndex([])
|
|
1264
|
+
return
|
|
1265
|
+
|
|
1266
|
+
if self._verbose:
|
|
1267
|
+
print(f"[RAG] Found {self._total_files} markdown files.")
|
|
1268
|
+
|
|
1269
|
+
all_chunks: list[TextChunk] = []
|
|
1270
|
+
matched = 0
|
|
1271
|
+
fallback_metadata = 0
|
|
1272
|
+
|
|
1273
|
+
for md_path in md_files:
|
|
1274
|
+
# Match to database
|
|
1275
|
+
paper = _match_paper_to_db(md_path.name, db_papers)
|
|
1276
|
+
if paper is None:
|
|
1277
|
+
paper = _paper_from_filename(md_path.name)
|
|
1278
|
+
if paper is None:
|
|
1279
|
+
continue
|
|
1280
|
+
fallback_metadata += 1
|
|
1281
|
+
else:
|
|
1282
|
+
matched += 1
|
|
1283
|
+
|
|
1284
|
+
# Read and chunk the file
|
|
1285
|
+
try:
|
|
1286
|
+
text = md_path.read_text(encoding="utf-8", errors="replace")
|
|
1287
|
+
except Exception:
|
|
1288
|
+
continue
|
|
1289
|
+
|
|
1290
|
+
chunks = _chunk_markdown(
|
|
1291
|
+
text, paper,
|
|
1292
|
+
max_chunk_words=self._max_chunk_words,
|
|
1293
|
+
overlap_words=self._overlap_words,
|
|
1294
|
+
)
|
|
1295
|
+
all_chunks.extend(chunks)
|
|
1296
|
+
|
|
1297
|
+
self._matched_count = matched
|
|
1298
|
+
self._total_chunks = len(all_chunks)
|
|
1299
|
+
|
|
1300
|
+
indexed_papers = matched + fallback_metadata
|
|
1301
|
+
|
|
1302
|
+
# Always print summary so user knows RAG loaded. Report indexed
|
|
1303
|
+
# papers first; BibTeX matching is metadata quality, not corpus
|
|
1304
|
+
# coverage.
|
|
1305
|
+
print(
|
|
1306
|
+
f"[RAG] Indexed {indexed_papers}/{self._total_files} papers "
|
|
1307
|
+
f"-> {len(all_chunks)} chunks."
|
|
1308
|
+
)
|
|
1309
|
+
if matched == 0 and fallback_metadata == 0:
|
|
1310
|
+
print(
|
|
1311
|
+
"[RAG] WARNING: No papers matched. Ensure filenames follow "
|
|
1312
|
+
"Zotero format: 'Author - YYYY - Title.md'"
|
|
1313
|
+
)
|
|
1314
|
+
|
|
1315
|
+
self._index = self._build_retriever(all_chunks)
|
|
1316
|
+
|
|
1317
|
+
def _build_retriever(
|
|
1318
|
+
self, chunks: list[TextChunk],
|
|
1319
|
+
) -> "TfIdfIndex | EmbeddingRetriever":
|
|
1320
|
+
"""Build the configured retriever for the given chunks.
|
|
1321
|
+
|
|
1322
|
+
Handles the embeddings → TF-IDF fallback logic when
|
|
1323
|
+
``backend="auto"``. Prints the selected backend so users
|
|
1324
|
+
know which retriever is active.
|
|
1325
|
+
"""
|
|
1326
|
+
if self._backend == "tfidf":
|
|
1327
|
+
self._active_backend = "tfidf"
|
|
1328
|
+
if self._verbose:
|
|
1329
|
+
print("[RAG] Using TF-IDF retrieval backend.")
|
|
1330
|
+
return TfIdfIndex(chunks)
|
|
1331
|
+
|
|
1332
|
+
# "embeddings" or "auto"
|
|
1333
|
+
try:
|
|
1334
|
+
retriever = EmbeddingRetriever(
|
|
1335
|
+
chunks,
|
|
1336
|
+
verbose=self._verbose,
|
|
1337
|
+
)
|
|
1338
|
+
self._active_backend = "embeddings"
|
|
1339
|
+
print(
|
|
1340
|
+
f"[RAG] Using embedding retrieval backend "
|
|
1341
|
+
f"({DEFAULT_EMBEDDING_MODEL})."
|
|
1342
|
+
)
|
|
1343
|
+
return retriever
|
|
1344
|
+
except (ImportError, RuntimeError) as exc:
|
|
1345
|
+
if self._backend == "embeddings":
|
|
1346
|
+
# User explicitly asked for embeddings — surface the
|
|
1347
|
+
# error loudly but still fall back to keep the
|
|
1348
|
+
# package usable.
|
|
1349
|
+
print(
|
|
1350
|
+
f"[RAG] WARNING: embedding backend unavailable: "
|
|
1351
|
+
f"{exc}. Falling back to TF-IDF."
|
|
1352
|
+
)
|
|
1353
|
+
elif self._verbose:
|
|
1354
|
+
print(
|
|
1355
|
+
f"[RAG] Embedding backend unavailable ({exc}). "
|
|
1356
|
+
"Using TF-IDF fallback."
|
|
1357
|
+
)
|
|
1358
|
+
self._active_backend = "tfidf"
|
|
1359
|
+
return TfIdfIndex(chunks)
|
|
1360
|
+
|
|
1361
|
+
def retrieve(
|
|
1362
|
+
self,
|
|
1363
|
+
query: str,
|
|
1364
|
+
top_k: int = 5,
|
|
1365
|
+
min_score: float | None = None,
|
|
1366
|
+
max_chunks_per_paper: int = 3,
|
|
1367
|
+
) -> list[RetrievalResult]:
|
|
1368
|
+
"""Retrieve relevant passages for a query string.
|
|
1369
|
+
|
|
1370
|
+
Parameters
|
|
1371
|
+
----------
|
|
1372
|
+
query:
|
|
1373
|
+
Free-text query (e.g., research description or analysis text).
|
|
1374
|
+
top_k:
|
|
1375
|
+
Maximum number of passages (chunks) to return. Multiple
|
|
1376
|
+
chunks may come from the same paper when that paper's
|
|
1377
|
+
most relevant content is spread across different sections.
|
|
1378
|
+
min_score:
|
|
1379
|
+
Minimum similarity threshold. When ``None`` (default), a
|
|
1380
|
+
backend-appropriate value is used: 0.01 for TF-IDF,
|
|
1381
|
+
0.3 for embedding cosine similarity.
|
|
1382
|
+
max_chunks_per_paper:
|
|
1383
|
+
Cap on how many chunks from the same paper may appear in
|
|
1384
|
+
the result set (default ``3``). Set to ``1`` for the
|
|
1385
|
+
legacy one-chunk-per-paper behavior. Useful when a single
|
|
1386
|
+
paper is highly relevant but its key evidence is scattered
|
|
1387
|
+
across Introduction / Results / Discussion sections that
|
|
1388
|
+
a single chunk can't cover on its own.
|
|
1389
|
+
|
|
1390
|
+
Returns
|
|
1391
|
+
-------
|
|
1392
|
+
List of :class:`RetrievalResult` sorted by relevance.
|
|
1393
|
+
|
|
1394
|
+
Raises
|
|
1395
|
+
------
|
|
1396
|
+
RuntimeError
|
|
1397
|
+
If :meth:`load` has not been called.
|
|
1398
|
+
"""
|
|
1399
|
+
if self._index is None:
|
|
1400
|
+
raise RuntimeError("RAG engine not loaded. Call load() first.")
|
|
1401
|
+
if min_score is None:
|
|
1402
|
+
min_score = (
|
|
1403
|
+
0.3 if self._active_backend == "embeddings" else 0.01
|
|
1404
|
+
)
|
|
1405
|
+
return self._index.query(
|
|
1406
|
+
query,
|
|
1407
|
+
top_k=top_k,
|
|
1408
|
+
min_score=min_score,
|
|
1409
|
+
max_chunks_per_paper=max_chunks_per_paper,
|
|
1410
|
+
)
|
|
1411
|
+
|
|
1412
|
+
def retrieve_for_analysis(
|
|
1413
|
+
self,
|
|
1414
|
+
parsed: object, # ParsedAnalysis, but avoid circular import
|
|
1415
|
+
top_k: int = 5,
|
|
1416
|
+
min_score: float | None = None,
|
|
1417
|
+
max_chunks_per_paper: int = 3,
|
|
1418
|
+
) -> list[RetrievalResult]:
|
|
1419
|
+
"""Retrieve passages relevant to a parsed LLM analysis.
|
|
1420
|
+
|
|
1421
|
+
Extracts query terms from the parsed analysis and retrieves
|
|
1422
|
+
matching passages from the full-text paper index.
|
|
1423
|
+
|
|
1424
|
+
Parameters
|
|
1425
|
+
----------
|
|
1426
|
+
parsed:
|
|
1427
|
+
A :class:`~metacouplingllm.llm.parser.ParsedAnalysis` object.
|
|
1428
|
+
top_k:
|
|
1429
|
+
Maximum number of results.
|
|
1430
|
+
min_score:
|
|
1431
|
+
Minimum similarity score.
|
|
1432
|
+
max_chunks_per_paper:
|
|
1433
|
+
See :meth:`retrieve`. Default ``3``.
|
|
1434
|
+
|
|
1435
|
+
Returns
|
|
1436
|
+
-------
|
|
1437
|
+
List of :class:`RetrievalResult`.
|
|
1438
|
+
"""
|
|
1439
|
+
query = _build_query_from_analysis(parsed)
|
|
1440
|
+
if not query:
|
|
1441
|
+
return []
|
|
1442
|
+
return self.retrieve(
|
|
1443
|
+
query,
|
|
1444
|
+
top_k=top_k,
|
|
1445
|
+
min_score=min_score,
|
|
1446
|
+
max_chunks_per_paper=max_chunks_per_paper,
|
|
1447
|
+
)
|
|
1448
|
+
|
|
1449
|
+
|
|
1450
|
+
# ---------------------------------------------------------------------------
|
|
1451
|
+
# Query builder from ParsedAnalysis
|
|
1452
|
+
# ---------------------------------------------------------------------------
|
|
1453
|
+
|
|
1454
|
+
|
|
1455
|
+
def _build_query_from_analysis(parsed: object) -> str:
|
|
1456
|
+
"""Build a search query from a ParsedAnalysis object.
|
|
1457
|
+
|
|
1458
|
+
Extracts key terms from classification, systems, flows, and causes.
|
|
1459
|
+
"""
|
|
1460
|
+
parts: list[str] = []
|
|
1461
|
+
|
|
1462
|
+
# Coupling classification
|
|
1463
|
+
cc = getattr(parsed, "coupling_classification", "")
|
|
1464
|
+
if cc:
|
|
1465
|
+
parts.append(cc)
|
|
1466
|
+
|
|
1467
|
+
# Systems
|
|
1468
|
+
systems = getattr(parsed, "systems", {})
|
|
1469
|
+
for role in ("sending", "receiving", "spillover"):
|
|
1470
|
+
val = systems.get(role, "")
|
|
1471
|
+
if isinstance(val, dict):
|
|
1472
|
+
for key in ("name", "geographic_scope", "description"):
|
|
1473
|
+
v = val.get(key, "")
|
|
1474
|
+
if v:
|
|
1475
|
+
parts.append(v)
|
|
1476
|
+
elif isinstance(val, str) and val:
|
|
1477
|
+
parts.append(val)
|
|
1478
|
+
|
|
1479
|
+
# Flows
|
|
1480
|
+
flows = getattr(parsed, "flows", [])
|
|
1481
|
+
for flow in flows:
|
|
1482
|
+
if isinstance(flow, dict):
|
|
1483
|
+
desc = flow.get("description", "")
|
|
1484
|
+
if desc:
|
|
1485
|
+
parts.append(desc)
|
|
1486
|
+
cat = flow.get("category", "")
|
|
1487
|
+
if cat:
|
|
1488
|
+
parts.append(cat)
|
|
1489
|
+
elif isinstance(flow, str) and flow:
|
|
1490
|
+
parts.append(flow)
|
|
1491
|
+
|
|
1492
|
+
# Causes — can be a list of strings, a dict of lists, or a dict of strings
|
|
1493
|
+
causes = getattr(parsed, "causes", [])
|
|
1494
|
+
if isinstance(causes, dict):
|
|
1495
|
+
for key, val in causes.items():
|
|
1496
|
+
if isinstance(val, list):
|
|
1497
|
+
for item in val:
|
|
1498
|
+
if isinstance(item, str) and item:
|
|
1499
|
+
parts.append(item)
|
|
1500
|
+
elif isinstance(val, str) and val:
|
|
1501
|
+
parts.append(val)
|
|
1502
|
+
elif isinstance(causes, list):
|
|
1503
|
+
for cat in causes:
|
|
1504
|
+
if isinstance(cat, str) and cat:
|
|
1505
|
+
parts.append(cat)
|
|
1506
|
+
elif isinstance(cat, dict):
|
|
1507
|
+
desc = cat.get("description", "")
|
|
1508
|
+
if desc:
|
|
1509
|
+
parts.append(desc)
|
|
1510
|
+
|
|
1511
|
+
# Effects — can be a list of strings, a dict of lists, or a dict of strings
|
|
1512
|
+
effects = getattr(parsed, "effects", [])
|
|
1513
|
+
if isinstance(effects, dict):
|
|
1514
|
+
for key, val in effects.items():
|
|
1515
|
+
if isinstance(val, list):
|
|
1516
|
+
for item in val:
|
|
1517
|
+
if isinstance(item, str) and item:
|
|
1518
|
+
parts.append(item)
|
|
1519
|
+
elif isinstance(val, str) and val:
|
|
1520
|
+
parts.append(val)
|
|
1521
|
+
elif isinstance(effects, list):
|
|
1522
|
+
for eff in effects:
|
|
1523
|
+
if isinstance(eff, dict):
|
|
1524
|
+
desc = eff.get("description", "")
|
|
1525
|
+
if desc:
|
|
1526
|
+
parts.append(desc)
|
|
1527
|
+
elif isinstance(eff, str) and eff:
|
|
1528
|
+
parts.append(eff)
|
|
1529
|
+
|
|
1530
|
+
return " ".join(parts)
|
|
1531
|
+
|
|
1532
|
+
|
|
1533
|
+
# ---------------------------------------------------------------------------
|
|
1534
|
+
# Formatting
|
|
1535
|
+
# ---------------------------------------------------------------------------
|
|
1536
|
+
|
|
1537
|
+
|
|
1538
|
+
def format_evidence(
|
|
1539
|
+
results: list[RetrievalResult],
|
|
1540
|
+
anchor_text: str = "",
|
|
1541
|
+
backend: str = "tfidf",
|
|
1542
|
+
) -> str:
|
|
1543
|
+
"""Format retrieved passages as a human-readable evidence block.
|
|
1544
|
+
|
|
1545
|
+
Parameters
|
|
1546
|
+
----------
|
|
1547
|
+
results:
|
|
1548
|
+
List of :class:`RetrievalResult` from retrieval.
|
|
1549
|
+
anchor_text:
|
|
1550
|
+
Optional query or analysis text used to choose a more relevant
|
|
1551
|
+
excerpt from each evidence chunk.
|
|
1552
|
+
backend:
|
|
1553
|
+
Retrieval backend that produced the scores (``"tfidf"`` or
|
|
1554
|
+
``"embeddings"``). Used to pick appropriate High/Medium/Low
|
|
1555
|
+
thresholds since the two backends have very different score
|
|
1556
|
+
ranges.
|
|
1557
|
+
|
|
1558
|
+
Returns
|
|
1559
|
+
-------
|
|
1560
|
+
A formatted string ready for display.
|
|
1561
|
+
"""
|
|
1562
|
+
if not results:
|
|
1563
|
+
return "No supporting evidence found in the full-text paper database."
|
|
1564
|
+
|
|
1565
|
+
lines: list[str] = []
|
|
1566
|
+
lines.append("=" * 70)
|
|
1567
|
+
lines.append(" SUPPORTING EVIDENCE FROM LITERATURE")
|
|
1568
|
+
lines.append("=" * 70)
|
|
1569
|
+
lines.append("")
|
|
1570
|
+
|
|
1571
|
+
for i, r in enumerate(results, 1):
|
|
1572
|
+
chunk = r.chunk
|
|
1573
|
+
# Shorten author list
|
|
1574
|
+
authors = chunk.authors
|
|
1575
|
+
if len(authors) > 60:
|
|
1576
|
+
parts = authors.split(" and ")
|
|
1577
|
+
if len(parts) > 2:
|
|
1578
|
+
authors = parts[0] + " et al."
|
|
1579
|
+
|
|
1580
|
+
confidence = _score_to_confidence(r.score, backend=backend)
|
|
1581
|
+
|
|
1582
|
+
lines.append(f" [{i}] {chunk.paper_title}")
|
|
1583
|
+
lines.append(f" {authors} ({chunk.year})")
|
|
1584
|
+
lines.append(f" Section: {chunk.section}")
|
|
1585
|
+
lines.append(f" Confidence: {confidence} (score: {r.score:.3f})")
|
|
1586
|
+
lines.append(f" ---")
|
|
1587
|
+
|
|
1588
|
+
# Show a short excerpt from the most relevant part of the chunk.
|
|
1589
|
+
excerpt = _best_excerpt(chunk.text, anchor_text=anchor_text)
|
|
1590
|
+
lines.append(f" \"{excerpt}\"")
|
|
1591
|
+
lines.append("")
|
|
1592
|
+
|
|
1593
|
+
lines.append("=" * 70)
|
|
1594
|
+
return "\n".join(lines)
|
|
1595
|
+
|
|
1596
|
+
|
|
1597
|
+
def annotate_citations(
|
|
1598
|
+
formatted: str,
|
|
1599
|
+
results: list[RetrievalResult],
|
|
1600
|
+
min_keyword_overlap: int = 3,
|
|
1601
|
+
min_overlap_ratio: float = 0.20,
|
|
1602
|
+
) -> str:
|
|
1603
|
+
"""Add inline citation numbers ``[N]`` to formatted analysis lines.
|
|
1604
|
+
|
|
1605
|
+
For each content line in the formatted output, checks keyword overlap
|
|
1606
|
+
with each RAG evidence passage. If a sufficient number of distinctive
|
|
1607
|
+
keywords match, appends ``[N]`` citation tags to the line.
|
|
1608
|
+
|
|
1609
|
+
Parameters
|
|
1610
|
+
----------
|
|
1611
|
+
formatted:
|
|
1612
|
+
The formatted analysis text (from :class:`AnalysisFormatter`).
|
|
1613
|
+
results:
|
|
1614
|
+
List of :class:`RetrievalResult` from RAG retrieval.
|
|
1615
|
+
min_keyword_overlap:
|
|
1616
|
+
Minimum number of shared keywords between a line and an evidence
|
|
1617
|
+
passage for a citation to be added.
|
|
1618
|
+
min_overlap_ratio:
|
|
1619
|
+
Minimum fraction of the line's keywords that must appear in the
|
|
1620
|
+
evidence passage.
|
|
1621
|
+
|
|
1622
|
+
Returns
|
|
1623
|
+
-------
|
|
1624
|
+
The formatted text with ``[N]`` citations appended to matching lines.
|
|
1625
|
+
"""
|
|
1626
|
+
if not results:
|
|
1627
|
+
return formatted
|
|
1628
|
+
|
|
1629
|
+
# Build keyword sets for each evidence passage
|
|
1630
|
+
evidence_keywords: list[set[str]] = []
|
|
1631
|
+
evidence_specific_keywords: list[set[str]] = []
|
|
1632
|
+
for r in results:
|
|
1633
|
+
tokens = set(_tokenise(r.chunk.text))
|
|
1634
|
+
evidence_keywords.append(tokens)
|
|
1635
|
+
evidence_specific_keywords.append(_specific_tokens(tokens))
|
|
1636
|
+
|
|
1637
|
+
# Section headers should not be annotated
|
|
1638
|
+
# Note: separator-only lines (all "=" or "-") are caught separately below
|
|
1639
|
+
_skip_prefixes = (
|
|
1640
|
+
"COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
|
|
1641
|
+
"FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
|
|
1642
|
+
"RESEARCH GAPS", "PERICOUPLING DATABASE",
|
|
1643
|
+
"METACOUPLING FRAMEWORK", "SUPPORTING EVIDENCE",
|
|
1644
|
+
"RECOMMENDED LITERATURE", "Note:",
|
|
1645
|
+
)
|
|
1646
|
+
_section_headers = (
|
|
1647
|
+
"COUPLING CLASSIFICATION", "SYSTEMS IDENTIFICATION",
|
|
1648
|
+
"FLOWS ANALYSIS", "AGENTS", "CAUSES", "EFFECTS",
|
|
1649
|
+
"RESEARCH GAPS",
|
|
1650
|
+
)
|
|
1651
|
+
|
|
1652
|
+
lines = formatted.split("\n")
|
|
1653
|
+
annotated: list[str] = []
|
|
1654
|
+
|
|
1655
|
+
# Track which section we're in to avoid annotating non-analysis content
|
|
1656
|
+
in_evidence_block = False
|
|
1657
|
+
current_section = ""
|
|
1658
|
+
|
|
1659
|
+
for line in lines:
|
|
1660
|
+
stripped = line.strip()
|
|
1661
|
+
|
|
1662
|
+
matched_header = next(
|
|
1663
|
+
(header for header in _section_headers if stripped.startswith(header)),
|
|
1664
|
+
"",
|
|
1665
|
+
)
|
|
1666
|
+
if matched_header:
|
|
1667
|
+
current_section = matched_header
|
|
1668
|
+
|
|
1669
|
+
# Detect evidence block boundaries — never annotate evidence itself
|
|
1670
|
+
if "SUPPORTING EVIDENCE FROM LITERATURE" in stripped:
|
|
1671
|
+
in_evidence_block = True
|
|
1672
|
+
if in_evidence_block:
|
|
1673
|
+
annotated.append(line)
|
|
1674
|
+
continue
|
|
1675
|
+
|
|
1676
|
+
# Skip empty lines, separators, and section headers
|
|
1677
|
+
if (
|
|
1678
|
+
not stripped
|
|
1679
|
+
or stripped.startswith(tuple(_skip_prefixes))
|
|
1680
|
+
or set(stripped) <= {"=", "-", " "}
|
|
1681
|
+
):
|
|
1682
|
+
annotated.append(line)
|
|
1683
|
+
continue
|
|
1684
|
+
|
|
1685
|
+
# Skip sub-headings like " [Sending]", " Ecological:", etc.
|
|
1686
|
+
if stripped.startswith("[") and stripped.endswith("]"):
|
|
1687
|
+
annotated.append(line)
|
|
1688
|
+
continue
|
|
1689
|
+
# Skip category headings like "Socioeconomic:" or " **Ecological**"
|
|
1690
|
+
if stripped.endswith(":") and len(stripped.split()) <= 3:
|
|
1691
|
+
annotated.append(line)
|
|
1692
|
+
continue
|
|
1693
|
+
|
|
1694
|
+
# Tokenize the line content
|
|
1695
|
+
line_tokens = set(_tokenise(stripped))
|
|
1696
|
+
if len(line_tokens) < 3:
|
|
1697
|
+
annotated.append(line)
|
|
1698
|
+
continue
|
|
1699
|
+
line_specific_tokens = _specific_tokens(line_tokens)
|
|
1700
|
+
section_min_overlap, section_min_ratio, min_specific_overlap, max_citations = (
|
|
1701
|
+
_citation_policy(current_section)
|
|
1702
|
+
)
|
|
1703
|
+
min_overlap = max(min_keyword_overlap, section_min_overlap)
|
|
1704
|
+
min_ratio = max(min_overlap_ratio, section_min_ratio)
|
|
1705
|
+
|
|
1706
|
+
# Find matching evidence passages
|
|
1707
|
+
candidates: list[tuple[tuple[float, float, float, float], int]] = []
|
|
1708
|
+
for idx, ev_kw in enumerate(evidence_keywords):
|
|
1709
|
+
overlap = line_tokens & ev_kw
|
|
1710
|
+
n_overlap = len(overlap)
|
|
1711
|
+
specific_overlap = line_specific_tokens & evidence_specific_keywords[idx]
|
|
1712
|
+
effective_overlap = (
|
|
1713
|
+
len(specific_overlap) if line_specific_tokens else n_overlap
|
|
1714
|
+
)
|
|
1715
|
+
effective_base = (
|
|
1716
|
+
len(line_specific_tokens) if line_specific_tokens else len(line_tokens)
|
|
1717
|
+
)
|
|
1718
|
+
ratio = effective_overlap / effective_base if effective_base else 0
|
|
1719
|
+
|
|
1720
|
+
if n_overlap < min_overlap or ratio < min_ratio:
|
|
1721
|
+
continue
|
|
1722
|
+
if line_specific_tokens and len(specific_overlap) < min_specific_overlap:
|
|
1723
|
+
continue
|
|
1724
|
+
|
|
1725
|
+
score = (
|
|
1726
|
+
float(len(specific_overlap)),
|
|
1727
|
+
ratio,
|
|
1728
|
+
float(n_overlap),
|
|
1729
|
+
results[idx].score,
|
|
1730
|
+
)
|
|
1731
|
+
candidates.append((score, idx + 1)) # 1-indexed
|
|
1732
|
+
|
|
1733
|
+
if candidates:
|
|
1734
|
+
candidates.sort(reverse=True)
|
|
1735
|
+
citations = sorted({
|
|
1736
|
+
citation
|
|
1737
|
+
for _, citation in candidates[:max_citations]
|
|
1738
|
+
})
|
|
1739
|
+
|
|
1740
|
+
existing = {
|
|
1741
|
+
int(match.group(1))
|
|
1742
|
+
for match in re.finditer(r"\[(\d+)\]", line)
|
|
1743
|
+
}
|
|
1744
|
+
new_citations = [c for c in citations if c not in existing]
|
|
1745
|
+
if not new_citations:
|
|
1746
|
+
annotated.append(line)
|
|
1747
|
+
continue
|
|
1748
|
+
|
|
1749
|
+
cite_str = " " + " ".join(f"[{n}]" for n in new_citations)
|
|
1750
|
+
annotated.append(f"{line}{cite_str}")
|
|
1751
|
+
else:
|
|
1752
|
+
annotated.append(line)
|
|
1753
|
+
|
|
1754
|
+
return "\n".join(annotated)
|
|
1755
|
+
|
|
1756
|
+
|
|
1757
|
+
def _score_to_confidence(score: float, backend: str = "tfidf") -> str:
|
|
1758
|
+
"""Convert a similarity score to a human-readable confidence label.
|
|
1759
|
+
|
|
1760
|
+
Parameters
|
|
1761
|
+
----------
|
|
1762
|
+
score:
|
|
1763
|
+
Cosine similarity score.
|
|
1764
|
+
backend:
|
|
1765
|
+
Either ``"tfidf"`` or ``"embeddings"``. The two backends
|
|
1766
|
+
produce scores in very different ranges, so we use separate
|
|
1767
|
+
threshold sets:
|
|
1768
|
+
|
|
1769
|
+
- TF-IDF: typical range 0.01–0.4 for relevant chunks.
|
|
1770
|
+
Thresholds: 0.15 / 0.08 / 0.03.
|
|
1771
|
+
- Embeddings (BGE-small cosine): typical range 0.5–0.9 for
|
|
1772
|
+
relevant chunks. Thresholds: 0.7 / 0.6 / 0.5.
|
|
1773
|
+
"""
|
|
1774
|
+
if backend == "embeddings":
|
|
1775
|
+
if score >= 0.70:
|
|
1776
|
+
return "High"
|
|
1777
|
+
elif score >= 0.60:
|
|
1778
|
+
return "Medium"
|
|
1779
|
+
elif score >= 0.50:
|
|
1780
|
+
return "Low"
|
|
1781
|
+
else:
|
|
1782
|
+
return "Very Low"
|
|
1783
|
+
# TF-IDF (default)
|
|
1784
|
+
if score >= 0.15:
|
|
1785
|
+
return "High"
|
|
1786
|
+
elif score >= 0.08:
|
|
1787
|
+
return "Medium"
|
|
1788
|
+
elif score >= 0.03:
|
|
1789
|
+
return "Low"
|
|
1790
|
+
else:
|
|
1791
|
+
return "Very Low"
|